diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0b3c8cd98..52d79c11f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2f90537dd7df70f6b663cd654b1fa5dee483cf6a4edcfd46072b2775be8a23ec + digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 diff --git a/.github/release-please.yml b/.github/release-please.yml index 4507ad059..466597e5b 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1 +1,2 @@ releaseType: python +handleGHRelease: true diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml new file mode 100644 index 000000000..d4ca94189 --- /dev/null +++ b/.github/release-trigger.yml @@ -0,0 +1 @@ +enabled: true diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 3abba6e06..b030caeef 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-bigquery python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 922d7fe50..6ae81b743 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-bigquery/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } diff --git a/.repo-metadata.json b/.repo-metadata.json index 124b40eb9..670aba793 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,9 +2,9 @@ "name": "bigquery", "name_pretty": "Google Cloud BigQuery", "product_documentation": "https://cloud.google.com/bigquery", - "client_documentation": "https://googleapis.dev/python/bigquery/latest", + "client_documentation": "https://cloud.google.com/python/docs/reference/bigquery/latest", "issue_tracker": "https://issuetracker.google.com/savedsearches/559654", - "release_level": "ga", + "release_level": "stable", "language": "python", "library_type": "GAPIC_COMBO", "repo": "googleapis/python-bigquery", @@ -12,5 +12,6 @@ "api_id": "bigquery.googleapis.com", "requires_billing": false, "default_version": "v2", - "codeowner_team": "@googleapis/api-bigquery" + "codeowner_team": "@googleapis/api-bigquery", + "api_shortname": "bigquery" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba219d20..6e69fa621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.32.0](https://github.com/googleapis/python-bigquery/compare/v2.31.0...v2.32.0) (2022-01-12) + + +### Features + +* support authorized dataset entity ([#1075](https://github.com/googleapis/python-bigquery/issues/1075)) ([c098cd0](https://github.com/googleapis/python-bigquery/commit/c098cd01c755633bfaba7193dd5c044a489a5b61)) + + +### Bug Fixes + +* remove query text from exception message, use `exception.debug_message` instead ([#1105](https://github.com/googleapis/python-bigquery/issues/1105)) ([e23114c](https://github.com/googleapis/python-bigquery/commit/e23114ce362e09ac72f733a640e53a561cc9ce69)) + ## [2.31.0](https://www.github.com/googleapis/python-bigquery/compare/v2.30.1...v2.31.0) (2021-11-24) diff --git a/README.rst b/README.rst index d0ad059a2..e8578916a 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Python Client for Google BigQuery ================================= -|GA| |pypi| |versions| +|GA| |pypi| |versions| Querying massive datasets can be time consuming and expensive without the right hardware and infrastructure. Google `BigQuery`_ solves this problem by @@ -52,7 +52,7 @@ dependencies. Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ -Python >= 3.6, < 3.10 +Python >= 3.6, < 3.11 Unsupported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -140,6 +140,3 @@ In this example all tracing data will be published to the Google .. _OpenTelemetry documentation: https://opentelemetry-python.readthedocs.io .. _Cloud Trace: https://cloud.google.com/trace - - - diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 61e860de5..4e31f7dba 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -66,6 +66,7 @@ _CONTAINS_ORDER_BY = re.compile(r"ORDER\s+BY", re.IGNORECASE) +_EXCEPTION_FOOTER_TEMPLATE = "{message}\n\nLocation: {location}\nJob ID: {job_id}\n" _TIMEOUT_BUFFER_SECS = 0.1 @@ -1196,17 +1197,17 @@ def _blocking_poll(self, timeout=None, **kwargs): super(QueryJob, self)._blocking_poll(timeout=timeout, **kwargs) @staticmethod - def _format_for_exception(query, job_id): + def _format_for_exception(message: str, query: str): """Format a query for the output in exception message. Args: + message (str): The original exception message. query (str): The SQL query to format. - job_id (str): The ID of the job that ran the query. Returns: str: A formatted query text. """ - template = "\n\n(job ID: {job_id})\n\n{header}\n\n{ruler}\n{body}\n{ruler}" + template = "{message}\n\n{header}\n\n{ruler}\n{body}\n{ruler}" lines = query.splitlines() max_line_len = max(len(line) for line in lines) @@ -1223,7 +1224,7 @@ def _format_for_exception(query, job_id): "{:4}:{}".format(n, line) for n, line in enumerate(lines, start=1) ) - return template.format(job_id=job_id, header=header, ruler=ruler, body=body) + return template.format(message=message, header=header, ruler=ruler, body=body) def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): """API call: begin the job via a POST request @@ -1248,7 +1249,10 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): try: super(QueryJob, self)._begin(client=client, retry=retry, timeout=timeout) except exceptions.GoogleAPICallError as exc: - exc.message += self._format_for_exception(self.query, self.job_id) + exc.message = _EXCEPTION_FOOTER_TEMPLATE.format( + message=exc.message, location=self.location, job_id=self.job_id + ) + exc.debug_message = self._format_for_exception(exc.message, self.query) exc.query_job = self raise @@ -1447,7 +1451,10 @@ def do_get_result(): do_get_result() except exceptions.GoogleAPICallError as exc: - exc.message += self._format_for_exception(self.query, self.job_id) + exc.message = _EXCEPTION_FOOTER_TEMPLATE.format( + message=exc.message, location=self.location, job_id=self.job_id + ) + exc.debug_message = self._format_for_exception(exc.message, self.query) # type: ignore exc.query_job = self # type: ignore raise except requests.exceptions.Timeout as exc: diff --git a/samples/geography/noxfile.py b/samples/geography/noxfile.py index 93a9122cc..20cdfc620 100644 --- a/samples/geography/noxfile.py +++ b/samples/geography/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -184,37 +185,45 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index faa0ed174..f788e6562 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,18 +1,17 @@ -attrs==21.2.0 -cachetools==4.2.4 +attrs==21.4.0 certifi==2021.10.8 cffi==1.15.0 -charset-normalizer==2.0.9 +charset-normalizer==2.0.10 click==8.0.3 click-plugins==1.1.1 cligj==0.7.2 -dataclasses==0.6; python_version < '3.7' -db-dtypes==0.3.0 +dataclasses==0.8; python_version < '3.7' +db-dtypes==0.3.1 Fiona==1.8.20 geojson==2.5.0 geopandas==0.9.0; python_version < '3.7' geopandas==0.10.2; python_version >= '3.7' -google-api-core==2.3.0 +google-api-core==2.4.0 google-auth==2.3.3 google-cloud-bigquery==2.31.0 google-cloud-bigquery-storage==2.10.1 @@ -20,32 +19,28 @@ google-cloud-core==2.2.1 google-crc32c==1.3.0 google-resumable-media==2.1.0 googleapis-common-protos==1.54.0 -grpcio==1.42.0 +grpcio==1.43.0 idna==3.3 -importlib-metadata==4.8.2 -libcst==0.3.23 +libcst==0.4.0 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' proto-plus==1.19.8 -protobuf==3.19.1 +protobuf==3.19.3 pyarrow==6.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.6 -pyproj==3.0.1; python_version < "3.7" -pyproj==3.1.0; python_version > "3.6" python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 -requests==2.26.0 +requests==2.27.1 rsa==4.8 Shapely==1.8.0 six==1.16.0 typing-extensions==4.0.1 typing-inspect==0.7.1 -urllib3==1.26.7 -zipp==3.6.0 +urllib3==1.26.8 diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py index 93a9122cc..20cdfc620 100644 --- a/samples/magics/noxfile.py +++ b/samples/magics/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -184,37 +185,45 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 91ef59887..7fef28615 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,13 +1,14 @@ db-dtypes==0.3.1 google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 -grpcio==1.42.0 +grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.5.0rc1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 typing-extensions==3.10.0.2 diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 93a9122cc..20cdfc620 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -184,37 +185,45 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 5d655797f..7fef28615 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,13 +1,14 @@ db-dtypes==0.3.1 google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 -grpcio==1.42.0 +grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.4.1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 typing-extensions==3.10.0.2 diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index 4da035b78..5fb76b9e9 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -1360,13 +1360,19 @@ def test_result_error(self): exc_job_instance = getattr(exc_info.exception, "query_job", None) self.assertIs(exc_job_instance, job) + # Query text could contain sensitive information, so it must not be + # included in logs / exception representation. full_text = str(exc_info.exception) assert job.job_id in full_text - assert "Query Job SQL Follows" in full_text + assert "Query Job SQL Follows" not in full_text + # It is useful to have query text available, so it is provided in a + # debug_message property. + debug_message = exc_info.exception.debug_message + assert "Query Job SQL Follows" in debug_message for i, line in enumerate(query.splitlines(), start=1): expected_line = "{}:{}".format(i, line) - assert expected_line in full_text + assert expected_line in debug_message def test_result_transport_timeout_error(self): query = textwrap.dedent( @@ -1452,13 +1458,19 @@ def test__begin_error(self): exc_job_instance = getattr(exc_info.exception, "query_job", None) self.assertIs(exc_job_instance, job) + # Query text could contain sensitive information, so it must not be + # included in logs / exception representation. full_text = str(exc_info.exception) assert job.job_id in full_text - assert "Query Job SQL Follows" in full_text + assert "Query Job SQL Follows" not in full_text + # It is useful to have query text available, so it is provided in a + # debug_message property. + debug_message = exc_info.exception.debug_message + assert "Query Job SQL Follows" in debug_message for i, line in enumerate(query.splitlines(), start=1): expected_line = "{}:{}".format(i, line) - assert expected_line in full_text + assert expected_line in debug_message def test__begin_w_timeout(self): PATH = "/projects/%s/jobs" % (self.PROJECT,) diff --git a/tests/unit/test_dbapi_connection.py b/tests/unit/test_dbapi_connection.py index 6b3a99439..770154377 100644 --- a/tests/unit/test_dbapi_connection.py +++ b/tests/unit/test_dbapi_connection.py @@ -199,8 +199,14 @@ def test_does_not_keep_cursor_instances_alive(self): # Connections should not hold strong references to the Cursor instances # they created, unnecessarily keeping them alive. gc.collect() - cursors = [obj for obj in gc.get_objects() if isinstance(obj, Cursor)] - self.assertEqual(len(cursors), 2) + cursor_count = 0 + for obj in gc.get_objects(): + try: + if isinstance(obj, Cursor): + cursor_count += 1 + except ReferenceError: # pragma: NO COVER + pass + self.assertEqual(cursor_count, 2) def test_commit(self): connection = self._make_one(client=self._mock_client())