Merge branch 'main' into mask-issue

Author: amoghrajesh

.asf.yaml

@@ -123,6 +123,7 @@ github:
     # https://github.com/apache/infrastructure-asfyaml/blob/main/README.md#assigning-the-github-triage-role-to-external-collaborators
     - aritra24
     - dirrao
+    - omkar-foss
     - rawwar
     - nathadfield
     - sunank200
@@ -133,3 +134,7 @@ github:
 
 notifications:
   jobs: [email protected]
+  commits: [email protected]
+  issues: [email protected]
+  pullrequests: [email protected]
+  discussions: [email protected]

.github/SECURITY.md

@@ -99,10 +99,11 @@ do not apply to Airflow, or have a different severity than some generic scoring
 
 The Airflow Security Team will get back to you after assessing the report. You will usually get
 confirmation that the issue is being worked (or that we quickly assessed it as invalid) within several
-business days. Note that this is an Open-Source projects and members of the security team are volunteers
-so please make sure to be patient. If you do not get a response within a week or so, please send a
-kind reminder to the security team. We will usually let you know the CVE number that will be assigned
-to the issue and the severity of the issue as well as release the issue is scheduled to be fixed
+business days. Note that this is an Open-Source projects and members of the security team are volunteers,
+so please make sure to be patient. If you do not get a response within a week, please send a kind reminder
+to the security team about a lack of response; however, reminders should only be for the initial response
+and not for updates on the assessment or remediation. We will usually let you know the CVE number that will
+be assigned to the issue and the severity of the issue as well as release the issue is scheduled to be fixed
 after we assess the issue (which might take longer or shorter time depending on the issue complexity and
 potential impact, severity, whether we want to address a whole class issues in a single fix and a number
 of other factors). You should subscribe  and monitor the `[email protected]` mailing

.github/workflows/basic-tests.yml

@@ -359,6 +359,8 @@ jobs:
           --hook-stage manual
           update-build-dependencies
         if: always()
+        env:
+          SKIP_TROVE_CLASSIFIERS_ONLY: "true"
       - name: "Run automated upgrade for chart dependencies"
         run: >
           pre-commit run

.github/workflows/ci.yml

@@ -99,7 +99,7 @@ jobs:
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docs-build: ${{ steps.selective-checks.outputs.docs-build }}
-      mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
+      mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
       needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
       needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
       needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
@@ -298,7 +298,7 @@ jobs:
       runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
-      mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
+      mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -644,6 +644,24 @@ jobs:
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
       needs.build-info.outputs.needs-helm-tests == 'true')
 
+  tests-task-sdk:
+    name: "Task SDK tests"
+    uses: ./.github/workflows/task-sdk-tests.yml
+    needs: [build-info, wait-for-ci-images]
+    permissions:
+      contents: read
+      packages: read
+    secrets: inherit
+    with:
+      runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
+      image-tag: ${{ needs.build-info.outputs.image-tag }}
+      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
+      run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }}
+    if: >
+      ( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
+      needs.build-info.outputs.run-tests == 'true')
+
   finalize-tests:
     name: Finalize tests
     permissions:

.github/workflows/static-checks-mypy-docs.yml

@@ -36,7 +36,7 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to run mypy checks (true/false)"
         required: true
         type: string
-      mypy-folders:
+      mypy-checks:
         description: "List of folders to run mypy checks on"
         required: false
         type: string
@@ -148,7 +148,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mypy-folder: ${{ fromJSON(inputs.mypy-folders) }}
+        mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
       IMAGE_TAG: "${{ inputs.image-tag }}"
@@ -166,10 +166,10 @@ jobs:
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
         id: breeze
-      - name: "MyPy checks for ${{ matrix.mypy-folder }}"
+      - name: "MyPy checks for ${{ matrix.mypy-check }}"
         run: |
           pip install pre-commit
-          pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files
+          pre-commit run --color always --verbose --hook-stage manual ${{matrix.mypy-check}} --all-files
         env:
           VERBOSE: "false"
           COLUMNS: "250"

.github/workflows/task-sdk-tests.yml

@@ -28,10 +28,6 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
-      canary-run:
-        description: "Whether this is a canary run"
-        required: true
-        type: string
       default-python-version:
         description: "Which version of python should be used by default"
         required: true
@@ -40,6 +36,10 @@ on:  # yamllint disable-line rule:truthy
         description: "JSON-formatted array of Python versions to build images from"
         required: true
         type: string
+      run-task-sdk-tests:
+        description: "Whether to run Task SDK tests or not (true/false)"
+        required: true
+        type: string
 jobs:
   task-sdk-tests:
     timeout-minutes: 80
@@ -58,7 +58,6 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERBOSE: "true"
       CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
-    if: inputs.run-task-sdk-tests == 'true'
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -81,11 +80,6 @@ jobs:
           pipx uninstall twine || true
           pipx install twine && twine check dist/*.whl
       - name: >
-          Run provider unit tests on
-          Airflow Task SDK:Python ${{ matrix.python-version }}
-        if: matrix.run-tests == 'true'
+          Run unit tests for Airflow Task SDK:Python ${{ matrix.python-version }}
         run: >
-          breeze testing tests --run-in-parallel
-          --parallel-test-types TaskSDK
-          --use-packages-from-dist
-          --package-format wheel
+          breeze testing task-sdk-tests --python "${{ matrix.python-version }}"

.pre-commit-config.yaml

@@ -193,6 +193,7 @@ repos:
         files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_build_dependencies.py$
         pass_filenames: false
         require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
       - id: update-installers
         name: Update installers to latest (manual)
         entry: ./scripts/ci/pre_commit/update_installers.py
@@ -1291,6 +1292,7 @@ repos:
           ^.*/.*_vendor/ |
           ^airflow/migrations |
           ^providers/ |
+          ^task_sdk/ |
           ^dev |
           ^scripts |
           ^docs |
@@ -1343,6 +1345,23 @@ repos:
         files: ^.*\.py$
         require_serial: true
         additional_dependencies: ['rich>=12.4.4']
+      - id: mypy-task-sdk
+        name: Run mypy for Task SDK
+        language: python
+        entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages
+        files: ^task_sdk/src/airflow/sdk/.*\.py$|^task_sdk/tests//.*\.py$
+        exclude: ^.*/.*_vendor/
+        require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
+      - id: mypy-task-sdk
+        stages: ['manual']
+        name: Run mypy for Task SDK (manual)
+        language: python
+        entry: ./scripts/ci/pre_commit/mypy_folder.py task_sdk/src/airflow/sdk
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
       - id: check-provider-yaml-valid
         name: Validate provider.yaml files
         entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py

Dockerfile.ci

@@ -1385,6 +1385,7 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
 COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml
 COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml
 COPY task_sdk/pyproject.toml ${AIRFLOW_SOURCES}/task_sdk/pyproject.toml
+COPY task_sdk/README.md ${AIRFLOW_SOURCES}/task_sdk/README.md
 COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/
 COPY tests_common/ ${AIRFLOW_SOURCES}/tests_common/
 COPY generated/* ${AIRFLOW_SOURCES}/generated/

README.md

@@ -219,7 +219,7 @@ Those are - in the order of most common ways people install Airflow:
 - [PyPI releases](https://pypi.org/project/apache-airflow/) to install Airflow using standard `pip` tool
 - [Docker Images](https://hub.docker.com/r/apache/airflow) to install airflow via
   `docker` tool, use them in Kubernetes, Helm Charts, `docker-compose`, `docker swarm`, etc. You can
-  read more about using, customising, and extending the images in the
+  read more about using, customizing, and extending the images in the
   [Latest docs](https://airflow.apache.org/docs/docker-stack/index.html), and
   learn details on the internals in the [images](https://airflow.apache.org/docs/docker-stack/index.html) document.
 - [Tags in GitHub](https://github.com/apache/airflow/tags) to retrieve the git project sources that

airflow/__init__.py

@@ -15,7 +15,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import annotations
+
+# We do not use "from __future__ import annotations" here because it is not supported
+# by Pycharm when we want to make sure all imports in airflow work from namespace packages
+# Adding it automatically is excluded in pyproject.toml via I002 ruff rule exclusion
 
 # Make `airflow` a namespace package, supporting installing
 # airflow.providers.* in different locations (i.e. one in site, and one in user