Merge branch 'master' into FSDP

Lightning-AI · Nov 26, 2024 · 51f2b86 · 51f2b86
2 parents 088b972 + 87565cb
commit 51f2b86
Show file tree

Hide file tree

Showing 288 changed files with 3,101 additions and 1,709 deletions.
diff --git a/.actions/assistant.py b/.actions/assistant.py
@@ -18,10 +18,11 @@
 import shutil
 import tempfile
 import urllib.request
+from collections.abc import Iterable, Iterator, Sequence
 from itertools import chain
 from os.path import dirname, isfile
 from pathlib import Path
-from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple
+from typing import Any, Optional
 
 from packaging.requirements import Requirement
 from packaging.version import Version
@@ -127,7 +128,7 @@ def _parse_requirements(lines: Iterable[str]) -> Iterator[_RequirementWithCommen
         pip_argument = None
 
 
-def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str = "all") -> List[str]:
+def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str = "all") -> list[str]:
     """Loading requirements from a file.
 
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
@@ -222,7 +223,7 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
         fp.writelines([ln + os.linesep for ln in requires] + [os.linesep])
 
 
-def _retrieve_files(directory: str, *ext: str) -> List[str]:
+def _retrieve_files(directory: str, *ext: str) -> list[str]:
     all_files = []
     for root, _, files in os.walk(directory):
         for fname in files:
@@ -232,7 +233,7 @@ def _retrieve_files(directory: str, *ext: str) -> List[str]:
     return all_files
 
 
-def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]], lightning_by: str = "") -> List[str]:
+def _replace_imports(lines: list[str], mapping: list[tuple[str, str]], lightning_by: str = "") -> list[str]:
     """Replace imports of standalone package to lightning.
 
     >>> lns = [
@@ -320,7 +321,7 @@ def copy_replace_imports(
             fo.writelines(lines)
 
 
-def create_mirror_package(source_dir: str, package_mapping: Dict[str, str]) -> None:
+def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> None:
     """Create a mirror package with adjusted imports."""
     # replace imports and copy the code
     mapping = package_mapping.copy()

diff --git a/.azure/gpu-benchmarks.yml b/.azure/gpu-benchmarks.yml
@@ -46,7 +46,7 @@ jobs:
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.4-cuda12.1.0"
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
       options: "--gpus=all --shm-size=32g"
     strategy:
       matrix:

diff --git a/.azure/gpu-tests-fabric.yml b/.azure/gpu-tests-fabric.yml
@@ -60,7 +60,7 @@ jobs:
           image: "pytorchlightning/pytorch_lightning:base-cuda-py3.11-torch2.3-cuda12.1.0"
           PACKAGE_NAME: "fabric"
         "Lightning | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.4-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
           PACKAGE_NAME: "lightning"
     workspace:
       clean: all
@@ -134,13 +134,13 @@ jobs:
         condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
         displayName: "Adjust tests & examples"
 
-      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest . -v --durations=50
-        workingDirectory: tests/tests_fabric/
+      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest tests_fabric/ -v --durations=50
+        workingDirectory: tests/
         displayName: "Testing: fabric standard"
         timeoutInMinutes: "10"
 
-      - bash: bash ../run_standalone_tests.sh "."
-        workingDirectory: tests/tests_fabric/
+      - bash: bash ./run_standalone_tests.sh "tests_fabric"
+        workingDirectory: tests/
         env:
           PL_STANDALONE_TESTS_SOURCE: $(COVERAGE_SOURCE)
         displayName: "Testing: fabric standalone"
@@ -157,7 +157,7 @@ jobs:
           ./codecov --token=$(CODECOV_TOKEN) --commit=$(Build.SourceVersion) \
             --flags=gpu,pytest,${COVERAGE_SOURCE} --name="GPU-coverage" --env=linux,azure
           ls -l
-        workingDirectory: tests/tests_fabric/
+        workingDirectory: tests/
         displayName: "Statistics"
 
       - script: |

diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml
@@ -53,7 +53,7 @@ jobs:
           image: "pytorchlightning/pytorch_lightning:base-cuda-py3.11-torch2.3-cuda12.1.0"
           PACKAGE_NAME: "pytorch"
         "Lightning | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.4-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
           PACKAGE_NAME: "lightning"
     pool: lit-rtx-3090
     variables:
@@ -155,13 +155,13 @@ jobs:
           ls -l checkpoints/
         displayName: "Get legacy checkpoints"
 
-      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest -v --durations=50
-        workingDirectory: tests/tests_pytorch
+      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest tests_pytorch/ -v --durations=50
+        workingDirectory: tests/
         displayName: "Testing: PyTorch standard"
         timeoutInMinutes: "35"
 
-      - bash: bash ../run_standalone_tests.sh "."
-        workingDirectory: tests/tests_pytorch
+      - bash: bash ./run_standalone_tests.sh "tests_pytorch"
+        workingDirectory: tests/
         env:
           PL_USE_MOCKED_MNIST: "1"
           PL_STANDALONE_TESTS_SOURCE: $(COVERAGE_SOURCE)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -5,39 +5,16 @@
 # the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
-* @lantiga @borda @tchaton @awaelchli @justusschock
-
-# CI/CD and configs
-/.actions/                  @borda @ethanwharris @justusschock
-/.github/                   @borda @ethanwharris @justusschock
-/.azure/                    @borda @ethanwharris @justusschock
-/dockers/                   @borda @ethanwharris @justusschock
-*.yml                       @borda @ethanwharris @justusschock
+* @lantiga @borda @tchaton @justusschock @ethanwharris
 
 # Docs
-/docs/                                      @lantiga @borda @awaelchli
-/docs/*/conf.py                             @borda @awaelchli
 /.github/*.md                               @williamfalcon @lantiga @borda
-/.github/ISSUE_TEMPLATE/                    @borda @tchaton @awaelchli
-/docs/source-fabric/conf.py                 @borda @awaelchli
-/docs/source-fabric/index.rst               @awaelchli @lantiga
-/docs/source-pytorch/conf.py                @borda @awaelchli
+/docs/source-fabric/index.rst               @williamfalcon @lantiga
 /docs/source-pytorch/index.rst              @williamfalcon @lantiga
 /docs/source-pytorch/levels                 @williamfalcon @lantiga
 
-# PyTorch Lightning
-/src/lightning/pytorch                      @lantiga @borda @tchaton @awaelchli @justusschock
-
-# Lightning Data
-/src/lightning/data/                        @tchaton @lantiga
-
-# Lightning Fabric
-/src/lightning/fabric                       @lantiga @borda @tchaton @awaelchli @justusschock
-
 /.github/CODEOWNERS                  @williamfalcon
 /SECURITY.md                         @williamfalcon @lantiga
 /README.md                           @williamfalcon @lantiga
-/setup.py                            @williamfalcon @borda
-/src/pytorch_lightning/__about__.py  @williamfalcon @borda
-/src/lightning_fabric/__about__.py   @williamfalcon @borda @awaelchli
-/src/*/__setup__.py                  @borda @justusschock
+/src/pytorch_lightning/__about__.py  @williamfalcon @lantiga @borda
+/src/lightning_fabric/__about__.py   @williamfalcon @lantiga @borda
diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml
@@ -19,27 +19,30 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "pl-cpu (macOS-13, lightning, 3.9, 2.1, oldest)"
+      - "pl-cpu (macOS-14, lightning, 3.9, 2.1, oldest)"
       - "pl-cpu (macOS-14, lightning, 3.10, 2.1)"
-      - "pl-cpu (macOS-14, lightning, 3.11, 2.2)"
+      - "pl-cpu (macOS-14, lightning, 3.11, 2.2.2)"
       - "pl-cpu (macOS-14, lightning, 3.11, 2.3)"
-      - "pl-cpu (macOS-14, lightning, 3.12, 2.4)"
+      - "pl-cpu (macOS-14, lightning, 3.12, 2.4.1)"
+      - "pl-cpu (macOS-14, lightning, 3.12, 2.5.1)"
       - "pl-cpu (ubuntu-20.04, lightning, 3.9, 2.1, oldest)"
       - "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.11, 2.2)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.11, 2.2.2)"
       - "pl-cpu (ubuntu-20.04, lightning, 3.11, 2.3)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.12, 2.4)"
+      - "pl-cpu (ubuntu-22.04, lightning, 3.12, 2.4.1)"
+      - "pl-cpu (ubuntu-22.04, lightning, 3.12, 2.5.1)"
       - "pl-cpu (windows-2022, lightning, 3.9, 2.1, oldest)"
       - "pl-cpu (windows-2022, lightning, 3.10, 2.1)"
-      - "pl-cpu (windows-2022, lightning, 3.11, 2.2)"
+      - "pl-cpu (windows-2022, lightning, 3.11, 2.2.2)"
       - "pl-cpu (windows-2022, lightning, 3.11, 2.3)"
-      - "pl-cpu (windows-2022, lightning, 3.12, 2.4)"
+      - "pl-cpu (windows-2022, lightning, 3.12, 2.4.1)"
+      - "pl-cpu (windows-2022, lightning, 3.12, 2.5.1)"
       - "pl-cpu (macOS-14, pytorch, 3.9, 2.1)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 2.1)"
       - "pl-cpu (windows-2022, pytorch, 3.9, 2.1)"
-      - "pl-cpu (macOS-12, pytorch, 3.10, 2.1)"
-      - "pl-cpu (ubuntu-22.04, pytorch, 3.10, 2.1)"
-      - "pl-cpu (windows-2022, pytorch, 3.10, 2.1)"
+      - "pl-cpu (macOS-14, pytorch, 3.12, 2.5.1)"
+      - "pl-cpu (ubuntu-22.04, pytorch, 3.12, 2.5.1)"
+      - "pl-cpu (windows-2022, pytorch, 3.12, 2.5.1)"
 
   - id: "pytorch_lightning: Azure GPU"
     paths:
@@ -86,14 +89,15 @@ subprojects:
     checks:
       - "lightning.Benchmarks"
 
-  - id: "pytorch-lightning: TPU workflow"
-    paths:
-      # tpu CI availability is very limited, so we only require tpu tests
-      # to pass when their configurations are modified
-      - ".github/workflows/tpu-tests.yml"
-      - "tests/tests_pytorch/run_tpu_tests.sh"
-    checks:
-      - "test-on-tpus (pytorch, pjrt, v4-8)"
+  # Temporarily disabled
+  #  - id: "pytorch-lightning: TPU workflow"
+  #    paths:
+  #      # tpu CI availability is very limited, so we only require tpu tests
+  #      # to pass when their configurations are modified
+  #      - ".github/workflows/tpu-tests.yml"
+  #      - "tests/tests_pytorch/run_tpu_tests.sh"
+  #    checks:
+  #      - "test-on-tpus (pytorch, pjrt, v4-8)"
 
   - id: "fabric: Docs"
     paths:
@@ -141,15 +145,17 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "build-cuda (3.11, 2.1, 12.1.0)"
-      - "build-cuda (3.11, 2.2, 12.1.0)"
-      - "build-cuda (3.11, 2.3, 12.1.0)"
-      - "build-cuda (3.12, 2.4, 12.1.0)"
+      - "build-cuda (3.10, 2.1.2, 12.1.0)"
+      - "build-cuda (3.11, 2.2.2, 12.1.0)"
+      - "build-cuda (3.11, 2.3.1, 12.1.0)"
+      - "build-cuda (3.11, 2.4.1, 12.1.0)"
+      - "build-cuda (3.12, 2.5.1, 12.1.0)"
       #- "build-NGC"
-      - "build-pl (3.11, 2.1, 12.1.0)"
+      - "build-pl (3.10, 2.1, 12.1.0)"
       - "build-pl (3.11, 2.2, 12.1.0)"
       - "build-pl (3.11, 2.3, 12.1.0)"
-      - "build-pl (3.12, 2.4, 12.1.0)"
+      - "build-pl (3.11, 2.4, 12.1.0)"
+      - "build-pl (3.12, 2.5, 12.1.0)"
 
   # SECTION: lightning_fabric
 
@@ -166,27 +172,30 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "fabric-cpu (macOS-13, lightning, 3.9, 2.1, oldest)"
+      - "fabric-cpu (macOS-14, lightning, 3.9, 2.1, oldest)"
       - "fabric-cpu (macOS-14, lightning, 3.10, 2.1)"
-      - "fabric-cpu (macOS-14, lightning, 3.11, 2.2)"
+      - "fabric-cpu (macOS-14, lightning, 3.11, 2.2.2)"
       - "fabric-cpu (macOS-14, lightning, 3.11, 2.3)"
-      - "fabric-cpu (macOS-14, lightning, 3.12, 2.4)"
+      - "fabric-cpu (macOS-14, lightning, 3.12, 2.4.1)"
+      - "fabric-cpu (macOS-14, lightning, 3.12, 2.5.1)"
       - "fabric-cpu (ubuntu-20.04, lightning, 3.9, 2.1, oldest)"
       - "fabric-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.2)"
+      - "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.2.2)"
       - "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.3)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.12, 2.4)"
+      - "fabric-cpu (ubuntu-22.04, lightning, 3.12, 2.4.1)"
+      - "fabric-cpu (ubuntu-22.04, lightning, 3.12, 2.5.1)"
       - "fabric-cpu (windows-2022, lightning, 3.9, 2.1, oldest)"
       - "fabric-cpu (windows-2022, lightning, 3.10, 2.1)"
-      - "fabric-cpu (windows-2022, lightning, 3.11, 2.2)"
+      - "fabric-cpu (windows-2022, lightning, 3.11, 2.2.2)"
       - "fabric-cpu (windows-2022, lightning, 3.11, 2.3)"
-      - "fabric-cpu (windows-2022, lightning, 3.12, 2.4)"
+      - "fabric-cpu (windows-2022, lightning, 3.12, 2.4.1)"
+      - "fabric-cpu (windows-2022, lightning, 3.12, 2.5.1)"
       - "fabric-cpu (macOS-14, fabric, 3.9, 2.1)"
       - "fabric-cpu (ubuntu-20.04, fabric, 3.9, 2.1)"
       - "fabric-cpu (windows-2022, fabric, 3.9, 2.1)"
-      - "fabric-cpu (macOS-12, fabric, 3.10, 2.1)"
-      - "fabric-cpu (ubuntu-22.04, fabric, 3.10, 2.1)"
-      - "fabric-cpu (windows-2022, fabric, 3.10, 2.1)"
+      - "fabric-cpu (macOS-14, fabric, 3.12, 2.5.1)"
+      - "fabric-cpu (ubuntu-22.04, fabric, 3.12, 2.5.1)"
+      - "fabric-cpu (windows-2022, fabric, 3.12, 2.5.1)"
 
   - id: "lightning_fabric: Azure GPU"
     paths:
@@ -258,14 +267,14 @@ subprojects:
       - "install-pkg (ubuntu-22.04, lightning, 3.11)"
       - "install-pkg (ubuntu-22.04, notset, 3.9)"
       - "install-pkg (ubuntu-22.04, notset, 3.11)"
-      - "install-pkg (macOS-12, fabric, 3.9)"
-      - "install-pkg (macOS-12, fabric, 3.11)"
-      - "install-pkg (macOS-12, pytorch, 3.9)"
-      - "install-pkg (macOS-12, pytorch, 3.11)"
-      - "install-pkg (macOS-12, lightning, 3.9)"
-      - "install-pkg (macOS-12, lightning, 3.11)"
-      - "install-pkg (macOS-12, notset, 3.9)"
-      - "install-pkg (macOS-12, notset, 3.11)"
+      - "install-pkg (macOS-14, fabric, 3.9)"
+      - "install-pkg (macOS-14, fabric, 3.11)"
+      - "install-pkg (macOS-14, pytorch, 3.9)"
+      - "install-pkg (macOS-14, pytorch, 3.11)"
+      - "install-pkg (macOS-14, lightning, 3.9)"
+      - "install-pkg (macOS-14, lightning, 3.11)"
+      - "install-pkg (macOS-14, notset, 3.9)"
+      - "install-pkg (macOS-14, notset, 3.11)"
       - "install-pkg (windows-2022, fabric, 3.9)"
       - "install-pkg (windows-2022, fabric, 3.11)"
       - "install-pkg (windows-2022, pytorch, 3.9)"

diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml
@@ -60,7 +60,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           # Python version here needs to be supported by all PL versions listed in back-compatible-versions.txt.
-          python-version: 3.8
+          python-version: "3.9"
 
       - name: Install PL from source
         env:

diff --git a/.github/workflows/call-clear-cache.yml b/.github/workflows/call-clear-cache.yml
@@ -23,18 +23,18 @@ on:
 jobs:
   cron-clear:
     if: github.event_name == 'schedule' || github.event_name == 'pull_request'
-    uses: Lightning-AI/utilities/.github/workflows/[email protected].7
+    uses: Lightning-AI/utilities/.github/workflows/[email protected].9
     with:
-      scripts-ref: v0.11.7
+      scripts-ref: v0.11.8
       dry-run: ${{ github.event_name == 'pull_request' }}
       pattern: "latest|docs"
       age-days: 7
 
   direct-clear:
     if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
-    uses: Lightning-AI/utilities/.github/workflows/[email protected].7
+    uses: Lightning-AI/utilities/.github/workflows/[email protected].9
     with:
-      scripts-ref: v0.11.7
+      scripts-ref: v0.11.8
       dry-run: ${{ github.event_name == 'pull_request' }}
       pattern: ${{ inputs.pattern || 'pypi_wheels' }} # setting str in case of PR / debugging
       age-days: ${{ fromJSON(inputs.age-days) || 0 }} # setting 0 in case of PR / debugging
diff --git a/.github/workflows/ci-check-md-links.yml b/.github/workflows/ci-check-md-links.yml
@@ -14,7 +14,7 @@ on:
 
 jobs:
   check-md-links:
-    uses: Lightning-AI/utilities/.github/workflows/[email protected].7
+    uses: Lightning-AI/utilities/.github/workflows/[email protected].9
     with:
       config-file: ".github/markdown-links-config.json"
       base-branch: "master"
diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml
@@ -42,7 +42,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "macOS-12", "windows-2022"]
+        os: ["ubuntu-22.04", "macOS-14", "windows-2022"]
         pkg-name: ["fabric", "pytorch", "lightning", "notset"]
         python-version: ["3.9", "3.11"]
     steps:

diff --git a/.github/workflows/ci-schema.yml b/.github/workflows/ci-schema.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   check:
-    uses: Lightning-AI/utilities/.github/workflows/[email protected].7
+    uses: Lightning-AI/utilities/.github/workflows/[email protected].9
     with:
       # skip azure due to the wrong schema file by MSFT
       # https://github.com/Lightning-AI/lightning-flash/pull/1455#issuecomment-1244793607