google · Piyush-Ingale · Dec 26, 2025 · Dec 26, 2025 · Dec 27, 2025 · Dec 30, 2025
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.11"]
     steps:
     - name: Cancel previous
       uses: styfle/cancel-workflow-action@0.7.0
@@ -47,7 +47,7 @@ jobs:
       with:
         path: ~/conda_pkgs_dir
         key:
-          ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ matrix.python-version }}-${{ hashFiles('ci3.8.yml') }}
+          ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ matrix.python-version }}-${{ hashFiles('ci3.11.yml') }}
     - name: Setup conda environment
       uses: conda-incubator/setup-miniconda@v3
       with:
@@ -86,15 +86,15 @@ jobs:
           echo "dir=$(pip cache dir)" >> "$GITHUB_OUTPUT"
       - name: Install linter
         run: |
-          pip install ruff==0.1.2
+          pip install ruff==0.14.10
       - name: Lint project
         run: ruff check .
   type-check:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8"]
+        python-version: ["3.11"]
     steps:
       - name: Cancel previous
         uses: styfle/cancel-workflow-action@0.7.0
@@ -110,7 +110,7 @@ jobs:
         with:
           path: ~/conda_pkgs_dir
           key:
-            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ matrix.python-version }}-${{ hashFiles('ci3.8.yml') }}
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ matrix.python-version }}-${{ hashFiles('ci3.11.yml') }}
       - name: Setup conda environment
         uses: conda-incubator/setup-miniconda@v3
         with:

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -25,7 +25,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2.3.1
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install dependencies
         run: |
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/setup-python@v2.3.1
         name: Install Python
         with:
-          python-version: 3.8
+          python-version: 3.11
       - uses: actions/download-artifact@v4.1.7
         with:
           name: releases

diff --git a/Dockerfile b/Dockerfile
@@ -13,21 +13,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-ARG py_version=3.8
-FROM apache/beam_python${py_version}_sdk:2.40.0 as beam_sdk
+ARG py_version=3.11
+FROM apache/beam_python${py_version}_sdk:2.70.0 as beam_sdk
 FROM continuumio/miniconda3:latest
 
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y libeccodes-dev && \
+    rm -rf /var/lib/apt/lists/*
+
 # Add the mamba solver for faster builds
 RUN conda install -n base conda-libmamba-solver
 RUN conda config --set solver libmamba
 
 # Create conda env using environment.yml
-ARG weather_tools_git_rev=main
-RUN git clone https://github.com/google/weather-tools.git /weather
+ARG weather_tools_git_rev=update-weather-tools-dependencies
+RUN git clone  https://github.com/Piyush-Ingale/weather-tools.git /weather 
 WORKDIR /weather
 RUN git checkout "${weather_tools_git_rev}"
 RUN rm -r /weather/weather_*/test_data/
-RUN conda env create -f environment.yml --debug
+WORKDIR /weather
+RUN conda env create -f environment.yml --debug && conda clean --all -f --yes
 
 # Activate the conda env and update the PATH
 ARG CONDA_ENV_NAME=weather-tools
@@ -36,6 +41,9 @@ ENV PATH /opt/conda/envs/${CONDA_ENV_NAME}/bin:$PATH
 
 # Install gcloud alpha
 RUN apt-get update -y
+
+# Install curl
+RUN apt-get install curl -y
 RUN gcloud components install alpha --quiet
 
 # Copy files from official SDK image, including script/dependencies.

diff --git a/Runners.md b/Runners.md
@@ -23,6 +23,10 @@ weather-mv -i gs://netcdf_file.nc \
   --direct_num_workers 2
 ```
 
+**NOTE:** By default, for direct runner `job_server_timeout` is set to 5 mins, meaning If gRPC connection
+is idle for more than 5 mins, it will be cut off and a "DEADLINE_EXCEEDED" error will be issued. You can
+use the `--job_server_timeout <sec>` or `--job-server-timeout <sec>` flag to override the default deadline.
+
 For a full list of how to configure the direct runner, please review
 [this page](https://beam.apache.org/documentation/runners/direct/).
 

diff --git a/ci3.11.yml b/ci3.11.yml
@@ -0,0 +1,43 @@
+name: weather-tools
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.11
+  - apache-beam=2.70.0
+  - pytest=9.0.2
+  - pytest-subtests=0.15.0
+  - cfgrib=0.9.15.1
+  - dask=2025.12.0
+  - dataclasses=0.8
+  - distributed=2025.12.0
+  - eccodes=2.42.0
+  - requests=2.32.5
+  - netcdf4=1.7.2
+  - rioxarray=0.19.0
+  - xarray-beam=0.6.3
+  - ecmwf-api-client=1.6.5
+  - fsspec=2025.12.0
+  - gcsfs=2025.12.0
+  - gdal=3.10.0
+  - libgdal-grib=3.10.0
+  - pyproj=3.7.1
+  - geojson=3.2.0
+  - simplejson=3.20.2
+  - metview-batch=5.23.0
+  - numpy=2.2.6
+  - pandas=2.3.3
+  - pip=25.3
+  - pygrib=2.1.8
+  - google-cloud-sdk=550.0.0
+  - aria2=1.37.0
+  - xarray==2025.08.0
+  - ruff==0.14.10
+  - zarr=3.1.5
+  - google-cloud-monitoring=2.28.0
+  - pip:
+    - cython==3.2.3
+    - earthengine-api==1.7.4
+    - git+https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi # TODO([#474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl.
+    - pyparsing==3.3.1
+    - .[test]
diff --git a/ci3.9.yml b/ci3.9.yml
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,4 +1,4 @@
 # doc requirements
 myst-parser==0.13.7
 sphinx>=2.1
-Jinja2<3.1
+Jinja2<3.2
diff --git a/environment.yml b/environment.yml
@@ -2,37 +2,39 @@ name: weather-tools
 channels:
   - conda-forge
 dependencies:
-  - python=3.8.13
-  - apache-beam=2.40.0
-  - xarray-beam=0.6.2
-  - xarray=2023.1.0
-  - fsspec=2022.11.0
-  - gcsfs=2022.11.0
-  - rioxarray=0.13.4
-  - gdal=3.5.1
-  - pyproj=3.4.0
-  - ecmwf-api-client=1.6.3
-  - eccodes=2.27.0
-  - cfgrib=0.9.10.2
-  - pygrib=2.1.4
-  - metview-batch=5.17.0
-  - netcdf4=1.6.1
-  - geojson=2.5.0=py_0
-  - simplejson=3.17.6
-  - numpy=1.22.4
-  - pandas=1.5.1
-  - google-cloud-sdk=410.0.0
-  - aria2=1.36.0
-  - pip=22.3
-  - zarr=2.15.0
-  - google-cloud-monitoring=2.22.2
+  - python=3.11
+  - apache-beam=2.70.0
+  - xarray-beam=0.6.3
+  - xarray=2025.08.0
+  - fsspec=2025.12.0
+  - gcsfs=2025.12.0
+  - rioxarray=0.19.0
+  - gdal=3.10.0
+  - libgdal-grib=3.10.0
+  - pyproj=3.7.1
+  - ecmwf-api-client=1.6.5
+  - eccodes=2.42.0
+  - cfgrib=0.9.15.1
+  - pygrib=2.1.8
+  - metview-batch=5.23.0
+  - netcdf4=1.7.2
+  - geojson=3.2.0
+  - simplejson=3.20.2
+  - numpy=2.2.6
+  - pandas=2.3.3
+  - google-cloud-sdk=550.0.0
+  - aria2=1.37.0
+  - pip=25.3
+  - zarr=3.1.5
+  - google-cloud-monitoring=2.28.0
   - pip:
-    - cython==0.29.34
-    - earthengine-api==0.1.329
-    - firebase-admin==6.0.1
-    - setuptools==70.3.0
+    - cython==3.2.3
+    - earthengine-api==1.7.4
+    - firebase-admin==6.9.0
+    - setuptools==80.9.0
+    - apache-beam[dill]==2.70.0  # Need this to run metview on dataflow.
     - git+https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi # TODO([#474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl.
-    - pyparsing==3.1.4
+    - pyparsing==3.3.1
     - .
     - ./weather_dl
     - ./weather_mv

diff --git a/pyproject.toml b/pyproject.toml
@@ -37,9 +37,12 @@ line-length = 120
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 
-# Assume Python 3.10.
-target-version = "py310"
+# Assume Python 3.11.
+target-version = "py311"
 
 [tool.ruff.mccabe]
 # Unlike Flake8, default to a complexity level of 10.
 max-complexity = 10
+
+[tool.pytype]
+inputs = ["weather_dl", "weather_mv", "weather_sp"]
diff --git a/setup.py b/setup.py
@@ -15,20 +15,20 @@
 from setuptools import find_packages, setup
 
 beam_gcp_requirements = [
-    "google-cloud-bigquery==2.34.4",
-    "google-cloud-bigquery-storage==2.14.1",
-    "google-cloud-bigtable==1.7.2",
-    "google-cloud-core==1.7.3",
-    "google-cloud-datastore==1.15.5",
-    "google-cloud-dlp==3.8.0",
-    "google-cloud-language==1.3.2",
-    "google-cloud-pubsub==2.13.4",
-    "google-cloud-pubsublite==1.4.2",
-    "google-cloud-recommendations-ai==0.2.0",
-    "google-cloud-spanner==1.19.3",
-    "google-cloud-videointelligence==1.16.3",
-    "google-cloud-vision==1.0.2",
-    "apache-beam[gcp]==2.40.0",
+    "google-cloud-bigquery==3.39.0",
+    "google-cloud-bigquery-storage==2.36.0",
+    "google-cloud-bigtable==2.35.0",
+    "google-cloud-core==2.5.0",
+    "google-cloud-datastore==2.23.0",
+    "google-cloud-dlp==3.33.0",
+    "google-cloud-language==2.18.0",
+    "google-cloud-pubsub==2.34.0",
+    "google-cloud-pubsublite==1.13.0",
+    "google-cloud-recommendations-ai==0.10.18",
+    "google-cloud-spanner==3.61.0",
+    "google-cloud-videointelligence==2.17.0",
+    "google-cloud-vision==3.11.0",
+    "apache-beam[gcp]==2.70.0",
 ]
 
 weather_dl_requirements = [
@@ -39,7 +39,7 @@
     "requests>=2.24.0",
     "google-cloud-firestore",
     "firebase-admin",
-    "urllib3==1.26.5",
+    "urllib3==2.6.2",
 ]
 
 weather_mv_requirements = [
@@ -56,9 +56,9 @@
     "earthengine-api>=0.1.263",
     "pyproj",  # requires separate binary installation!
     "gdal",  # requires separate binary installation!
-    "xarray-beam==0.6.2",
-    "gcsfs==2022.11.0",
-    "zarr==2.15.0",
+    "xarray-beam==0.6.3",
+    "gcsfs==2025.12.0",
+    "zarr==3.1.5",
 ]
 
 weather_sp_requirements = [
@@ -69,8 +69,8 @@
 ]
 
 test_requirements = [
-    "pytype==2021.11.29",
-    "ruff==0.1.2",
+    "pytype==2024.02.09",
+    "ruff==0.14.10",
     "pytest",
     "pytest-subtests",
     "netcdf4",
@@ -110,21 +110,20 @@
         'Operating System :: MacOS :: MacOS X',
         # 'Operating System :: Microsoft :: Windows',  # TODO(#64): Fully support Windows.
         'Operating System :: POSIX',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Scientific/Engineering :: Atmospheric Science',
 
     ],
-    python_requires='>=3.8, <3.10',
-    install_requires=['apache-beam[gcp]==2.40.0', 'gcsfs==2022.11.0'],
+    python_requires='>=3.11',
+    install_requires=['apache-beam[gcp]==2.70.0', 'apache-beam[dill]==2.70.0', 'gcsfs==2025.12.0'],
     dependency_links=['https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi'],  # TODO([
         #474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl.
     use_scm_version=True,
     setup_requires=['setuptools_scm'],
     scripts=['weather_dl/weather-dl', 'weather_mv/weather-mv', 'weather_sp/weather-sp'],
     tests_require=test_requirements,
     extras_require={
-        'docs': ['tox', 'sphinx>=2.1', 'myst-parser', 'Jinja2<3.1'],
+        'docs': ['tox', 'sphinx>=2.1', 'myst-parser', 'Jinja2<3.2'],
         'test': all_test_requirements,
         'dev': ['google-weather-tools[docs,test]'],
         'regrid': ['metview']

diff --git a/weather_dl/download_pipeline/fetcher.py b/weather_dl/download_pipeline/fetcher.py
@@ -46,7 +46,7 @@ class Fetcher(beam.DoFn):
     """
 
     client_name: str
-    manifest: Manifest = NoOpManifest(Location('noop://in-memory'))
+    manifest: Manifest = dataclasses.field(default_factory=NoOpManifest(Location('noop://in-memory')))
     store: t.Optional[Store] = None
     log_level: t.Optional[int] = logging.INFO