apache · autophagy · May 12, 2025
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -110,7 +110,7 @@ jobs:
           # Use manylinux2014 on Linux
           CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
           CIBW_BEFORE_ALL_LINUX: pip install patchelf
-          CIBW_BEFORE_BUILD_LINUX: pip install -r flink-python/dev/dev-requirements.txt
+          CIBW_BEFORE_BUILD_LINUX: pip install --group flink-python/pyproject.toml:dev
           CIBW_ENVIRONMENT_LINUX: CFLAGS="-I. -include ./dev/glibc_version_fix.h"
           # Run auditwheel repair on Linux
           CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair -w {dest_dir} {wheel}"

diff --git a/docs/content.zh/docs/flinkDev/building.md b/docs/content.zh/docs/flinkDev/building.md
@@ -92,7 +92,7 @@ mvn clean install -DskipTests -Dfast -Pskip-webui-build -T 1C
     还需要使用以下命令安装依赖项：
 
     ```shell
-    $ python -m pip install -r flink-python/dev/dev-requirements.txt
+    $ python -m pip install --group flink-python/pyproject.toml:dev
     ```
 
 #### 安装

diff --git a/docs/content/docs/flinkDev/building.md b/docs/content/docs/flinkDev/building.md
@@ -90,7 +90,7 @@ The `fast` and `skip-webui-build` profiles have a significant impact on the buil
     You also need to install the dependencies with following command:
 
     ```shell
-    $ python -m pip install -r flink-python/dev/dev-requirements.txt
+    $ python -m pip install --group flink-python/pyproject.toml:dev
     ```
 
 #### Installation

diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh
@@ -76,17 +76,17 @@ mkdir -p "$PYFLINK_DOCKER_DIR"
 cp "${FLINK_PYTHON_DIR}/dist/${PYFLINK_PACKAGE_FILE}" $PYFLINK_DOCKER_DIR/
 cp "${FLINK_PYTHON_DIR}/apache-flink-libraries/dist/${PYFLINK_LIBRARIES_PACKAGE_FILE}" $PYFLINK_DOCKER_DIR/
 cp "${FLINK_PYTHON_DIR}/dev/lint-python.sh" $PYFLINK_DOCKER_DIR/
-cp "${FLINK_PYTHON_DIR}/dev/dev-requirements.txt" $PYFLINK_DOCKER_DIR/
+cp "${FLINK_PYTHON_DIR}/pyproject.toml" $PYFLINK_DOCKER_DIR/
 if [[ -d "dist" ]]; then rm -Rf dist; fi
 cd ${PYFLINK_DOCKER_DIR}
 echo "FROM ${PURE_FLINK_IMAGE_NAME}" >> Dockerfile
 echo "RUN apt-get update && apt-get install build-essential -y" >> Dockerfile
 echo "COPY lint-python.sh /tmp/lint-python.sh" >> Dockerfile
-echo "COPY dev-requirements.txt /tmp/dev-requirements.txt" >> Dockerfile
+echo "COPY pyproject.toml /tmp/pyproject.toml" >> Dockerfile
 echo "RUN bash /tmp/lint-python.sh -s basic" >> Dockerfile
 echo "COPY ${PYFLINK_PACKAGE_FILE} ${PYFLINK_PACKAGE_FILE}" >> Dockerfile
 echo "COPY ${PYFLINK_LIBRARIES_PACKAGE_FILE} ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile
-echo "RUN /tmp/.uv/bin/python -m pip install -r /tmp/dev-requirements.txt"  >> Dockerfile
+echo "RUN /tmp/.uv/bin/python -m pip install --group /tmp/pyproject.toml:dev"  >> Dockerfile
 echo "RUN /tmp/.uv/bin/python -m pip install ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile
 echo "RUN /tmp/.uv/bin/python -m pip install ${PYFLINK_PACKAGE_FILE}" >> Dockerfile
 echo "RUN rm ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile

diff --git a/flink-end-to-end-tests/test-scripts/test_pyflink.sh b/flink-end-to-end-tests/test-scripts/test_pyflink.sh
@@ -97,7 +97,7 @@ cd "${FLINK_PYTHON_DIR}"
 
 rm -rf dist
 
-uv pip install -r dev/dev-requirements.txt
+uv pip install --group dev
 
 python setup.py sdist
 

diff --git a/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh b/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh
@@ -34,7 +34,7 @@ start_hadoop_cluster_and_prepare_flink
 
 # copy test files
 docker cp "${FLINK_PYTHON_DIR}/dev/lint-python.sh" master:/tmp/
-docker cp "${FLINK_PYTHON_DIR}/dev/dev-requirements.txt" master:/tmp/
+docker cp "${FLINK_PYTHON_DIR}/pyproject.toml" master:/tmp/
 docker cp "${FLINK_PYTHON_TEST_DIR}/target/PythonUdfSqlJobExample.jar" master:/tmp/
 docker cp "${FLINK_PYTHON_TEST_DIR}/python/add_one.py" master:/tmp/
 docker cp "${REQUIREMENTS_PATH}" master:/tmp/
@@ -48,7 +48,7 @@ docker cp "${FLINK_PYTHON_DIR}/apache-flink-libraries/dist/${PYFLINK_LIBRARIES_P
 docker exec master bash -c "
 /tmp/lint-python.sh -s uv
 source /tmp/.uv/bin/activate
-pip install -r /tmp/dev-requirements.txt
+pip install --group /tmp/pyproject.toml:dev
 pip install /tmp/${PYFLINK_LIBRARIES_PACKAGE_FILE}
 pip install /tmp/${PYFLINK_PACKAGE_FILE}
 apt-get install -y zip

diff --git a/flink-python/README.md b/flink-python/README.md
@@ -67,6 +67,6 @@ Then you can activate your environment and run tests, for example:
 
 ```shell
 source pyflink_39/bin/activate
-uv pip install -r ./dev/dev-requirements.txt
+uv pip install --group dev
 ./dev/lint-python.sh
 ```
diff --git a/flink-python/dev/build-wheels.sh b/flink-python/dev/build-wheels.sh
@@ -24,7 +24,7 @@ py_env=("3.9" "3.10" "3.11" "3.12")
 for ((i=0;i<${#py_env[@]};i++)) do
     source `pwd`/dev/.uv/envs/${py_env[i]}/bin/activate
     echo "Installing dependencies for environment: ${py_env[i]}"
-    uv pip install -r dev/dev-requirements.txt
+    uv pip install --group dev
     deactivate
 done
 

diff --git a/flink-python/dev/dev-requirements.txt b/flink-python/dev/dev-requirements.txt
diff --git a/flink-python/dev/lint-python.sh b/flink-python/dev/lint-python.sh
@@ -267,7 +267,7 @@ function install_tox() {
         fi
     fi
 
-    $CURRENT_DIR/install_command.sh -q tox==3.14.0 2>&1 >/dev/null
+    $CURRENT_DIR/install_command.sh -q --group "${PYPROJECT_PATH}:tox" 2>&1 >/dev/null
     if [ $? -ne 0 ]; then
         echo "uv pip install tox failed \
         please try to exec the script again.\
@@ -291,7 +291,7 @@ function install_flake8() {
         fi
     fi
 
-    $CURRENT_DIR/install_command.sh -q flake8==7.2.0 2>&1 >/dev/null
+    $CURRENT_DIR/install_command.sh -q --group "${PYPROJECT_PATH}:flake8" 2>&1 >/dev/null
     if [ $? -ne 0 ]; then
         echo "uv pip install flake8 failed \
         please try to exec the script again.\
@@ -315,7 +315,7 @@ function install_sphinx() {
         fi
     fi
 
-    $CURRENT_DIR/install_command.sh -q Sphinx==4.5.0 importlib-metadata==4.4.0 Docutils==0.17.1 pydata_sphinx_theme==0.11.0 sphinx_mdinclude==0.5.3 "Jinja2<3.1.0" "sphinxcontrib-applehelp<1.0.8" "sphinxcontrib.devhelp<1.0.6" "sphinxcontrib.htmlhelp<2.0.5" "sphinxcontrib-serializinghtml<1.1.10" "sphinxcontrib-qthelp<1.0.7" 2>&1 >/dev/null
+    $CURRENT_DIR/install_command.sh -q --group "${PYPROJECT_PATH}:sphinx" 2>&1 >/dev/null
     if [ $? -ne 0 ]; then
         echo "uv pip install sphinx failed \
         please try to exec the script again.\
@@ -339,7 +339,7 @@ function install_mypy() {
             exit 1
         fi
     fi
-    ${CURRENT_DIR}/install_command.sh -q mypy==1.5.1 types-pytz types-python-dateutil 2>&1 >/dev/null
+    ${CURRENT_DIR}/install_command.sh -q --group "${PYPROJECT_PATH}:mypy" 2>&1 >/dev/null
     if [[ $? -ne 0 ]]; then
         echo "uv pip install mypy failed \
         please try to exec the script again.\
@@ -677,6 +677,8 @@ CURRENT_DIR="$(cd "$( dirname "$0" )" && pwd)"
 # FLINK_PYTHON_DIR is "flink/flink-python"
 FLINK_PYTHON_DIR=$(dirname "$CURRENT_DIR")
 
+PYPROJECT_PATH="${FLINK_PYTHON_DIR}/pyproject.toml"
+
 # uv home path
 if [ -z "${FLINK_UV_HOME+x}" ]; then
     UV_HOME="$CURRENT_DIR/.uv"
@@ -731,7 +733,7 @@ create_dir $LOG_DIR
 echo >$LOG_FILE
 
 # static version of uv that we use across all envs
-UV_VERSION=0.5.23
+UV_VERSION=0.7.20
 
 # location of uv installation script
 UV_INSTALL_SH=$CURRENT_DIR/download/uv.sh

diff --git a/flink-python/pyproject.toml b/flink-python/pyproject.toml
@@ -27,6 +27,56 @@ requires = [
     "cython>=0.29.24"
 ]
 
+[dependency-groups]
+dev = [
+  "pip>=20.3",
+  "setuptools>=75.3",
+  "wheel",
+  "apache-beam>=2.54.0,<=2.61.0",
+  "cython>=0.29.24",
+  "py4j==0.10.9.7",
+  "python-dateutil>=2.8.0,<3",
+  "cloudpickle~=2.2.0",
+  "avro>=1.12.0",
+  "pandas>=1.3.0",
+  "pyarrow>=5.0.0",
+  "pytz>=2018.3",
+  "numpy>=1.22.4,<2.3.0",
+  "fastavro>=1.1.0,!=1.8.0",
+  "grpcio>=1.29.0,<=1.71.0",
+  "grpcio-tools>=1.29.0,<=1.71.0",
+  "pemja>=0.5.0,<0.5.4; platform_system != 'Windows'",
+  "httplib2>=0.19.0",
+  "protobuf~=4.25",
+  "pytest~=8.0",
+  "ruamel.yaml>=0.18.4",
+]
+tox = [
+    "tox==3.14.0"
+]
+flake8 = [
+    "flake8==7.2.0"
+]
+sphinx = [
+    "Sphinx==4.5.0",
+    "importlib-metadata==4.4.0",
+    "Docutils==0.17.1",
+    "pydata_sphinx_theme==0.11.0",
+    "sphinx_mdinclude==0.5.3",
+    "Jinja2<3.1.0",
+    "sphinxcontrib-applehelp<1.0.8",
+    "sphinxcontrib.devhelp<1.0.6",
+    "sphinxcontrib.htmlhelp<2.0.5",
+    "sphinxcontrib-serializinghtml<1.1.10",
+    "sphinxcontrib-qthelp<1.0.7"
+]
+mypy = [
+    "mypy==1.5.1",
+    "types-pytz",
+    "types-python-dateutil"
+]
+
+
 [tool.cibuildwheel]
 build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*"]
 

diff --git a/flink-python/tox.ini b/flink-python/tox.ini
@@ -27,7 +27,7 @@ envlist = {py39, py310, py311, py312}-cython
 whitelist_externals=
     /bin/bash
 deps =
-    -r dev/dev-requirements.txt
+    --group dev
     apache-flink-libraries
 passenv = *
 commands =

diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh
@@ -88,7 +88,7 @@ make_python_release() {
   # use lint-python.sh script to create a python environment.
   dev/lint-python.sh -s basic
   source dev/.uv/bin/activate
-  uv pip install -r dev/dev-requirements.txt
+  uv pip install --group dev
 
   # build apache-flink-libraries sdist
   pushd apache-flink-libraries