diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f2cc179e2dfca..25f938e03c723 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -110,7 +110,7 @@ jobs: # Use manylinux2014 on Linux CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_BEFORE_ALL_LINUX: pip install patchelf - CIBW_BEFORE_BUILD_LINUX: pip install -r flink-python/dev/dev-requirements.txt + CIBW_BEFORE_BUILD_LINUX: pip install --group flink-python/pyproject.toml:dev CIBW_ENVIRONMENT_LINUX: CFLAGS="-I. -include ./dev/glibc_version_fix.h" # Run auditwheel repair on Linux CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair -w {dest_dir} {wheel}" diff --git a/docs/content.zh/docs/flinkDev/building.md b/docs/content.zh/docs/flinkDev/building.md index 7b8bfcfa0d2e1..99a6a66627987 100644 --- a/docs/content.zh/docs/flinkDev/building.md +++ b/docs/content.zh/docs/flinkDev/building.md @@ -92,7 +92,7 @@ mvn clean install -DskipTests -Dfast -Pskip-webui-build -T 1C 还需要使用以下命令安装依赖项: ```shell - $ python -m pip install -r flink-python/dev/dev-requirements.txt + $ python -m pip install --group flink-python/pyproject.toml:dev ``` #### 安装 diff --git a/docs/content/docs/flinkDev/building.md b/docs/content/docs/flinkDev/building.md index b83c48ce682ad..e2c7059eb824b 100644 --- a/docs/content/docs/flinkDev/building.md +++ b/docs/content/docs/flinkDev/building.md @@ -90,7 +90,7 @@ The `fast` and `skip-webui-build` profiles have a significant impact on the buil You also need to install the dependencies with following command: ```shell - $ python -m pip install -r flink-python/dev/dev-requirements.txt + $ python -m pip install --group flink-python/pyproject.toml:dev ``` #### Installation diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh index 4b830f4fa83a4..389ee33b79277 100755 --- a/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh +++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh @@ -76,17 +76,17 @@ mkdir -p "$PYFLINK_DOCKER_DIR" cp "${FLINK_PYTHON_DIR}/dist/${PYFLINK_PACKAGE_FILE}" $PYFLINK_DOCKER_DIR/ cp "${FLINK_PYTHON_DIR}/apache-flink-libraries/dist/${PYFLINK_LIBRARIES_PACKAGE_FILE}" $PYFLINK_DOCKER_DIR/ cp "${FLINK_PYTHON_DIR}/dev/lint-python.sh" $PYFLINK_DOCKER_DIR/ -cp "${FLINK_PYTHON_DIR}/dev/dev-requirements.txt" $PYFLINK_DOCKER_DIR/ +cp "${FLINK_PYTHON_DIR}/pyproject.toml" $PYFLINK_DOCKER_DIR/ if [[ -d "dist" ]]; then rm -Rf dist; fi cd ${PYFLINK_DOCKER_DIR} echo "FROM ${PURE_FLINK_IMAGE_NAME}" >> Dockerfile echo "RUN apt-get update && apt-get install build-essential -y" >> Dockerfile echo "COPY lint-python.sh /tmp/lint-python.sh" >> Dockerfile -echo "COPY dev-requirements.txt /tmp/dev-requirements.txt" >> Dockerfile +echo "COPY pyproject.toml /tmp/pyproject.toml" >> Dockerfile echo "RUN bash /tmp/lint-python.sh -s basic" >> Dockerfile echo "COPY ${PYFLINK_PACKAGE_FILE} ${PYFLINK_PACKAGE_FILE}" >> Dockerfile echo "COPY ${PYFLINK_LIBRARIES_PACKAGE_FILE} ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile -echo "RUN /tmp/.uv/bin/python -m pip install -r /tmp/dev-requirements.txt" >> Dockerfile +echo "RUN /tmp/.uv/bin/python -m pip install --group /tmp/pyproject.toml:dev" >> Dockerfile echo "RUN /tmp/.uv/bin/python -m pip install ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile echo "RUN /tmp/.uv/bin/python -m pip install ${PYFLINK_PACKAGE_FILE}" >> Dockerfile echo "RUN rm ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile diff --git a/flink-end-to-end-tests/test-scripts/test_pyflink.sh b/flink-end-to-end-tests/test-scripts/test_pyflink.sh index 215b1f4f48289..20fd2640553c0 100755 --- a/flink-end-to-end-tests/test-scripts/test_pyflink.sh +++ b/flink-end-to-end-tests/test-scripts/test_pyflink.sh @@ -97,7 +97,7 @@ cd "${FLINK_PYTHON_DIR}" rm -rf dist -uv pip install -r dev/dev-requirements.txt +uv pip install --group dev python setup.py sdist diff --git a/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh b/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh index a604ab5c58e0b..f7c5e1ec48215 100755 --- a/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh +++ b/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh @@ -34,7 +34,7 @@ start_hadoop_cluster_and_prepare_flink # copy test files docker cp "${FLINK_PYTHON_DIR}/dev/lint-python.sh" master:/tmp/ -docker cp "${FLINK_PYTHON_DIR}/dev/dev-requirements.txt" master:/tmp/ +docker cp "${FLINK_PYTHON_DIR}/pyproject.toml" master:/tmp/ docker cp "${FLINK_PYTHON_TEST_DIR}/target/PythonUdfSqlJobExample.jar" master:/tmp/ docker cp "${FLINK_PYTHON_TEST_DIR}/python/add_one.py" master:/tmp/ docker cp "${REQUIREMENTS_PATH}" master:/tmp/ @@ -48,7 +48,7 @@ docker cp "${FLINK_PYTHON_DIR}/apache-flink-libraries/dist/${PYFLINK_LIBRARIES_P docker exec master bash -c " /tmp/lint-python.sh -s uv source /tmp/.uv/bin/activate -pip install -r /tmp/dev-requirements.txt +pip install --group /tmp/pyproject.toml:dev pip install /tmp/${PYFLINK_LIBRARIES_PACKAGE_FILE} pip install /tmp/${PYFLINK_PACKAGE_FILE} apt-get install -y zip diff --git a/flink-python/README.md b/flink-python/README.md index 6cb7ecfe6b170..32f5a95ce59a7 100644 --- a/flink-python/README.md +++ b/flink-python/README.md @@ -67,6 +67,6 @@ Then you can activate your environment and run tests, for example: ```shell source pyflink_39/bin/activate -uv pip install -r ./dev/dev-requirements.txt +uv pip install --group dev ./dev/lint-python.sh ``` diff --git a/flink-python/dev/build-wheels.sh b/flink-python/dev/build-wheels.sh index 2a2eef0336f75..39180452450cd 100755 --- a/flink-python/dev/build-wheels.sh +++ b/flink-python/dev/build-wheels.sh @@ -24,7 +24,7 @@ py_env=("3.9" "3.10" "3.11" "3.12") for ((i=0;i<${#py_env[@]};i++)) do source `pwd`/dev/.uv/envs/${py_env[i]}/bin/activate echo "Installing dependencies for environment: ${py_env[i]}" - uv pip install -r dev/dev-requirements.txt + uv pip install --group dev deactivate done diff --git a/flink-python/dev/dev-requirements.txt b/flink-python/dev/dev-requirements.txt deleted file mode 100755 index 4da2ba7f21014..0000000000000 --- a/flink-python/dev/dev-requirements.txt +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -pip>=20.3 -setuptools>=75.3 -wheel -apache-beam>=2.54.0,<=2.61.0 -cython>=0.29.24 -py4j==0.10.9.7 -python-dateutil>=2.8.0,<3 -cloudpickle~=2.2.0 -avro>=1.12.0 -pandas>=1.3.0 -pyarrow>=5.0.0 -pytz>=2018.3 -numpy>=1.22.4,<2.3.0 -fastavro>=1.1.0,!=1.8.0 -grpcio>=1.29.0,<=1.71.0 -grpcio-tools>=1.29.0,<=1.71.0 -pemja>=0.5.0,<0.5.4; platform_system != 'Windows' -httplib2>=0.19.0 -protobuf~=4.25 -pytest~=8.0 -ruamel.yaml>=0.18.4 diff --git a/flink-python/dev/lint-python.sh b/flink-python/dev/lint-python.sh index 7cf290eb8bba3..90fceed46ecea 100755 --- a/flink-python/dev/lint-python.sh +++ b/flink-python/dev/lint-python.sh @@ -267,7 +267,7 @@ function install_tox() { fi fi - $CURRENT_DIR/install_command.sh -q tox==3.14.0 2>&1 >/dev/null + $CURRENT_DIR/install_command.sh -q --group "${PYPROJECT_PATH}:tox" 2>&1 >/dev/null if [ $? -ne 0 ]; then echo "uv pip install tox failed \ please try to exec the script again.\ @@ -291,7 +291,7 @@ function install_flake8() { fi fi - $CURRENT_DIR/install_command.sh -q flake8==7.2.0 2>&1 >/dev/null + $CURRENT_DIR/install_command.sh -q --group "${PYPROJECT_PATH}:flake8" 2>&1 >/dev/null if [ $? -ne 0 ]; then echo "uv pip install flake8 failed \ please try to exec the script again.\ @@ -315,7 +315,7 @@ function install_sphinx() { fi fi - $CURRENT_DIR/install_command.sh -q Sphinx==4.5.0 importlib-metadata==4.4.0 Docutils==0.17.1 pydata_sphinx_theme==0.11.0 sphinx_mdinclude==0.5.3 "Jinja2<3.1.0" "sphinxcontrib-applehelp<1.0.8" "sphinxcontrib.devhelp<1.0.6" "sphinxcontrib.htmlhelp<2.0.5" "sphinxcontrib-serializinghtml<1.1.10" "sphinxcontrib-qthelp<1.0.7" 2>&1 >/dev/null + $CURRENT_DIR/install_command.sh -q --group "${PYPROJECT_PATH}:sphinx" 2>&1 >/dev/null if [ $? -ne 0 ]; then echo "uv pip install sphinx failed \ please try to exec the script again.\ @@ -339,7 +339,7 @@ function install_mypy() { exit 1 fi fi - ${CURRENT_DIR}/install_command.sh -q mypy==1.5.1 types-pytz types-python-dateutil 2>&1 >/dev/null + ${CURRENT_DIR}/install_command.sh -q --group "${PYPROJECT_PATH}:mypy" 2>&1 >/dev/null if [[ $? -ne 0 ]]; then echo "uv pip install mypy failed \ please try to exec the script again.\ @@ -677,6 +677,8 @@ CURRENT_DIR="$(cd "$( dirname "$0" )" && pwd)" # FLINK_PYTHON_DIR is "flink/flink-python" FLINK_PYTHON_DIR=$(dirname "$CURRENT_DIR") +PYPROJECT_PATH="${FLINK_PYTHON_DIR}/pyproject.toml" + # uv home path if [ -z "${FLINK_UV_HOME+x}" ]; then UV_HOME="$CURRENT_DIR/.uv" @@ -731,7 +733,7 @@ create_dir $LOG_DIR echo >$LOG_FILE # static version of uv that we use across all envs -UV_VERSION=0.5.23 +UV_VERSION=0.7.20 # location of uv installation script UV_INSTALL_SH=$CURRENT_DIR/download/uv.sh diff --git a/flink-python/pyproject.toml b/flink-python/pyproject.toml index c77177574d8e0..5c83edc7ef9dd 100644 --- a/flink-python/pyproject.toml +++ b/flink-python/pyproject.toml @@ -27,6 +27,56 @@ requires = [ "cython>=0.29.24" ] +[dependency-groups] +dev = [ + "pip>=20.3", + "setuptools>=75.3", + "wheel", + "apache-beam>=2.54.0,<=2.61.0", + "cython>=0.29.24", + "py4j==0.10.9.7", + "python-dateutil>=2.8.0,<3", + "cloudpickle~=2.2.0", + "avro>=1.12.0", + "pandas>=1.3.0", + "pyarrow>=5.0.0", + "pytz>=2018.3", + "numpy>=1.22.4,<2.3.0", + "fastavro>=1.1.0,!=1.8.0", + "grpcio>=1.29.0,<=1.71.0", + "grpcio-tools>=1.29.0,<=1.71.0", + "pemja>=0.5.0,<0.5.4; platform_system != 'Windows'", + "httplib2>=0.19.0", + "protobuf~=4.25", + "pytest~=8.0", + "ruamel.yaml>=0.18.4", +] +tox = [ + "tox==3.14.0" +] +flake8 = [ + "flake8==7.2.0" +] +sphinx = [ + "Sphinx==4.5.0", + "importlib-metadata==4.4.0", + "Docutils==0.17.1", + "pydata_sphinx_theme==0.11.0", + "sphinx_mdinclude==0.5.3", + "Jinja2<3.1.0", + "sphinxcontrib-applehelp<1.0.8", + "sphinxcontrib.devhelp<1.0.6", + "sphinxcontrib.htmlhelp<2.0.5", + "sphinxcontrib-serializinghtml<1.1.10", + "sphinxcontrib-qthelp<1.0.7" +] +mypy = [ + "mypy==1.5.1", + "types-pytz", + "types-python-dateutil" +] + + [tool.cibuildwheel] build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] diff --git a/flink-python/tox.ini b/flink-python/tox.ini index 0e36a7684b7e0..6f57a22dc26c0 100644 --- a/flink-python/tox.ini +++ b/flink-python/tox.ini @@ -27,7 +27,7 @@ envlist = {py39, py310, py311, py312}-cython whitelist_externals= /bin/bash deps = - -r dev/dev-requirements.txt + --group dev apache-flink-libraries passenv = * commands = diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index 2832d749011b2..f4bc5a1d23b59 100755 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -88,7 +88,7 @@ make_python_release() { # use lint-python.sh script to create a python environment. dev/lint-python.sh -s basic source dev/.uv/bin/activate - uv pip install -r dev/dev-requirements.txt + uv pip install --group dev # build apache-flink-libraries sdist pushd apache-flink-libraries