Update to pydantic v2

ohrite · ohrite · commit 5ddf1c451f4d · 2025-11-13T17:30:28.000-08:00
diff --git a/airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py b/airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py
@@ -19,7 +19,7 @@
     GTFSDownloadConfigExtract,
     get_fs,
 )
-from pydantic import ValidationError
+from pydantic.v1 import ValidationError
 
 
 def gtfs_datasets_to_extract_configs(
diff --git a/airflow/dags/download_gtfs_schedule_v2/download_schedule_feeds.py b/airflow/dags/download_gtfs_schedule_v2/download_schedule_feeds.py
@@ -24,7 +24,7 @@
     get_fs,
     get_latest,
 )
-from pydantic import validator
+from pydantic.v1 import validator
 from requests.exceptions import HTTPError
 
 GTFS_FEED_LIST_ERROR_THRESHOLD = 0.95
diff --git a/airflow/dags/sync_ntd_data_xlsx/scrape_ntd_xlsx_urls.py b/airflow/dags/sync_ntd_data_xlsx/scrape_ntd_xlsx_urls.py
@@ -7,7 +7,7 @@
 
 import requests
 from bs4 import BeautifulSoup
-from pydantic import HttpUrl, ValidationError, parse_obj_as
+from pydantic.v1 import HttpUrl, ValidationError, parse_obj_as
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
diff --git a/airflow/plugins/operators/dbt_manifest_to_dictionary_operator.py b/airflow/plugins/operators/dbt_manifest_to_dictionary_operator.py
@@ -1,7 +1,7 @@
 import json
 from typing import Literal, Optional, Sequence
 
-from pydantic import BaseModel, validator
+from pydantic.v1 import BaseModel, validator
 
 from airflow.models import BaseOperator
 from airflow.models.taskinstance import Context
diff --git a/airflow/plugins/operators/dbt_manifest_to_metadata_operator.py b/airflow/plugins/operators/dbt_manifest_to_metadata_operator.py
@@ -2,7 +2,7 @@
 from datetime import datetime, timedelta
 from typing import Literal, Optional, Sequence
 
-from pydantic import BaseModel, constr
+from pydantic.v1 import BaseModel, constr
 
 from airflow.models import BaseOperator, DagRun
 from airflow.models.taskinstance import Context
diff --git a/airflow/plugins/operators/littlepay_raw_sync_feed_v3.py b/airflow/plugins/operators/littlepay_raw_sync_feed_v3.py
@@ -14,9 +14,9 @@
     get_fs,
     get_latest_file,
 )
-from pydantic.class_validators import validator
-from pydantic.error_wrappers import ValidationError
-from pydantic.main import BaseModel
+from pydantic.v1.class_validators import validator
+from pydantic.v1.error_wrappers import ValidationError
+from pydantic.v1.main import BaseModel
 
 from airflow.models import BaseOperator
 
diff --git a/airflow/plugins/operators/scrape_ntd_xlsx.py b/airflow/plugins/operators/scrape_ntd_xlsx.py
@@ -13,7 +13,7 @@
     get_fs,
     make_name_bq_safe,
 )
-from pydantic import HttpUrl, parse_obj_as
+from pydantic.v1 import HttpUrl, parse_obj_as
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
diff --git a/airflow/plugins/operators/scrape_state_geoportal.py b/airflow/plugins/operators/scrape_state_geoportal.py
@@ -7,7 +7,7 @@
 import pendulum
 import requests
 from calitp_data_infra.storage import PartitionedGCSArtifact, get_fs  # type: ignore
-from pydantic import HttpUrl, parse_obj_as
+from pydantic.v1 import HttpUrl, parse_obj_as
 
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator  # type: ignore
diff --git a/airflow/plugins/scripts/gtfs_rt_parser.py b/airflow/plugins/scripts/gtfs_rt_parser.py
@@ -43,8 +43,8 @@
 from google.protobuf import json_format
 from google.protobuf.message import DecodeError
 from google.transit import gtfs_realtime_pb2  # type: ignore
-from pydantic import BaseModel, Field, validator
-from pydantic.tools import parse_obj_as
+from pydantic.v1 import BaseModel, Field, validator
+from pydantic.v1.tools import parse_obj_as
 
 JAR_DEFAULT = os.path.normpath(
     os.path.join(
diff --git a/airflow/plugins/utils.py b/airflow/plugins/utils.py
@@ -11,7 +11,7 @@
     PartitionedGCSArtifact,
     fetch_all_in_partition,
 )
-from pydantic import validator
+from pydantic.v1 import validator
 
 SCHEDULE_UNZIPPED_BUCKET = os.environ["CALITP_BUCKET__GTFS_SCHEDULE_UNZIPPED"]
 SCHEDULE_UNZIPPED_BUCKET_HOURLY = os.environ[
diff --git a/airflow/requirements.txt b/airflow/requirements.txt
@@ -1,7 +1,7 @@
-calitp-data-infra==2025.6.5
+calitp-data-infra==2025.11.11
 gusty==0.6.0
 pyairtable==2.2.1
-pydantic>=1.9,<2.0
+pydantic>=2.12.4
 typer==0.4.1
 sentry-sdk==1.17.0
 platformdirs>=2.5
diff --git a/packages/calitp-data-infra/calitp_data_infra/storage.py b/packages/calitp-data-infra/calitp_data_infra/storage.py
@@ -33,7 +33,7 @@
 import humanize
 import pendulum
 from google.cloud import storage  # type: ignore
-from pydantic import (
+from pydantic.v1 import (
     BaseModel,
     ConstrainedStr,
     Extra,
@@ -42,8 +42,8 @@
     ValidationError,
     validator,
 )
-from pydantic.class_validators import root_validator
-from pydantic.tools import parse_obj_as
+from pydantic.v1.class_validators import root_validator
+from pydantic.v1.tools import parse_obj_as
 from requests import Request, Session
 from typing_extensions import Annotated, Literal
 
diff --git a/packages/calitp-data-infra/poetry.lock b/packages/calitp-data-infra/poetry.lock
diff --git a/packages/calitp-data-infra/pyproject.toml b/packages/calitp-data-infra/pyproject.toml
@@ -1,35 +1,29 @@
-[tool.poetry]
+[project]
 name = "calitp-data-infra"
-version = "2025.6.5"
+version = "2025.11.11"
 description = "Shared code for developing data pipelines that process Cal-ITP data."
 package-mode = true
-authors = ["Andrew Vaccaro"]
-
-[tool.poetry.dependencies]
-python = ">=3.8,<3.12"
-# Some of these are pinned oddly to play nicely with Composer
-pydantic = ">=1.9,<2.0"
-tqdm = "^4.64.1"
-pendulum = "^2.1.2"
-humanize = "^4.6.0"
-backoff = "^2.2.1"
-typing-extensions = ">=3.10.0.2"
-gcsfs = "!=2022.7.1"
-google-api-core = "^2.24.1"
-google-cloud-secret-manager = "^2.23.1"
-
-[tool.poetry.group.dev.dependencies]
-pytest = "^7.2.1"
-mypy = "^1.0.0"
-hypothesis = "^6.68.0"
-types-requests = "^2.31.0.1"
-
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
+authors = [{name="Caltrans Data and Digital Services"}]
+readme = "README.md"
+requires-python = "==3.11.*"
+dependencies = [
+    "backoff>=2.2.1",
+    "gcsfs>=2025.10.0",
+    "google-api-core>=2.28.1",
+    "google-cloud-secret-manager>=2.25.0",
+    "humanize>=4.14.0",
+    "pendulum>=3.1.0",
+    "pydantic>=2.12.4",
+    "typing-extensions>=4.15.0",
+]
 
-[tool.black]
-line-length = 120
+[dependency-groups]
+dev = [
+    "hypothesis>=6.147.0",
+    "mypy>=1.18.2",
+    "pytest>=9.0.1",
+    "types-requests>=2.32.4.20250913",
+]
 
 [tool.mypy]
 exclude = [
diff --git a/packages/calitp-data-infra/tests/test_storage.py b/packages/calitp-data-infra/tests/test_storage.py
@@ -1,7 +1,7 @@
 import pendulum
 import pytest
 from calitp_data_infra.storage import GTFSDownloadConfig, make_name_bq_safe
-from pydantic import ValidationError
+from pydantic.v1 import ValidationError
 
 
 def test_gtfs_download_config() -> None:
diff --git a/services/gtfs-rt-archiver-v3/gtfs_rt_archiver_v3/models.py b/services/gtfs-rt-archiver-v3/gtfs_rt_archiver_v3/models.py
@@ -1,5 +1,5 @@
 import pendulum
-from pydantic import AnyHttpUrl, BaseModel
+from pydantic.v1 import AnyHttpUrl, BaseModel
 
 
 class FetchTask(BaseModel):
diff --git a/warehouse/scripts/dbt_artifacts/__init__.py b/warehouse/scripts/dbt_artifacts/__init__.py
@@ -8,7 +8,7 @@
 
 import humanize
 import pendulum
-from pydantic import BaseModel, Field, constr, root_validator
+from pydantic.v1 import BaseModel, Field, constr, root_validator
 from slugify import slugify
 from sqlalchemy import MetaData, Table, create_engine, select
 from sqlalchemy.sql import Select
diff --git a/warehouse/scripts/dbt_artifacts/catalog.py b/warehouse/scripts/dbt_artifacts/catalog.py
@@ -7,7 +7,7 @@
 from datetime import datetime
 from typing import Dict, List, Optional, Union
 
-from pydantic import BaseModel, Extra, Field
+from pydantic.v1 import BaseModel, Extra, Field
 
 
 class CatalogMetadata(BaseModel):
diff --git a/warehouse/scripts/dbt_artifacts/manifest.py b/warehouse/scripts/dbt_artifacts/manifest.py
@@ -8,7 +8,7 @@
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import BaseModel, Extra, Field
+from pydantic.v1 import BaseModel, Extra, Field
 
 
 class UserIdItem(BaseModel):
diff --git a/warehouse/scripts/dbt_artifacts/run_results.py b/warehouse/scripts/dbt_artifacts/run_results.py
@@ -8,7 +8,7 @@
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import BaseModel, Extra, Field
+from pydantic.v1 import BaseModel, Extra, Field
 
 
 class BaseArtifactMetadata(BaseModel):
diff --git a/warehouse/scripts/dbt_artifacts/sources.py b/warehouse/scripts/dbt_artifacts/sources.py
@@ -8,7 +8,7 @@
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import BaseModel, Extra, Field
+from pydantic.v1 import BaseModel, Extra, Field
 
 
 class FreshnessMetadata(BaseModel):
diff --git a/warehouse/scripts/publish.py b/warehouse/scripts/publish.py
@@ -33,7 +33,7 @@
     TilesDestination,
 )
 from google.cloud import bigquery  # type: ignore
-from pydantic import BaseModel, constr, validator
+from pydantic.v1 import BaseModel, constr, validator
 from requests import Response
 from requests_toolbelt import MultipartEncoder  # type: ignore
 from tqdm import tqdm

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@`
`19`	`19`	`GTFSDownloadConfigExtract,`
`20`	`20`	`get_fs,`
`21`	`21`	`)`
`22`		`-from pydantic import ValidationError`
	`22`	`+from pydantic.v1 import ValidationError`
`23`	`23`
`24`	`24`
`25`	`25`	`def gtfs_datasets_to_extract_configs(`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@`
`24`	`24`	`get_fs,`
`25`	`25`	`get_latest,`
`26`	`26`	`)`
`27`		`-from pydantic import validator`
	`27`	`+from pydantic.v1 import validator`
`28`	`28`	`from requests.exceptions import HTTPError`
`29`	`29`
`30`	`30`	`GTFS_FEED_LIST_ERROR_THRESHOLD = 0.95`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@`
`13`	`13`	`get_fs,`
`14`	`14`	`make_name_bq_safe,`
`15`	`15`	`)`
`16`		`-from pydantic import HttpUrl, parse_obj_as`
	`16`	`+from pydantic.v1 import HttpUrl, parse_obj_as`
`17`	`17`	`from requests.adapters import HTTPAdapter`
`18`	`18`	`from urllib3.util.retry import Retry`
`19`	`19`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@`
`11`	`11`	`PartitionedGCSArtifact,`
`12`	`12`	`fetch_all_in_partition,`
`13`	`13`	`)`
`14`		`-from pydantic import validator`
	`14`	`+from pydantic.v1 import validator`
`15`	`15`
`16`	`16`	`SCHEDULE_UNZIPPED_BUCKET = os.environ["CALITP_BUCKET__GTFS_SCHEDULE_UNZIPPED"]`
`17`	`17`	`SCHEDULE_UNZIPPED_BUCKET_HOURLY = os.environ[`