From 6d9c926cd6ed236f2c30137a80c533f5fdd7fb0a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 9 Aug 2017 15:28:35 -0700 Subject: [PATCH 1/3] BQ: wait for load jobs to complete in system tests. Specify job ID in BQ DB-API DML tests to allow for further test flake debugging. --- bigquery/tests/system.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index 9d3bb7794256..65a6522166af 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -381,6 +381,7 @@ def test_load_table_from_local_file_then_dump_table(self): ) # Retry until done. + job.result() retry = RetryInstanceState(_job_done, max_tries=8) retry(job.reload)() @@ -419,6 +420,7 @@ def test_load_table_from_local_avro_file_then_dump_table(self): ) # Retry until done. + job.result() retry = RetryInstanceState(_job_done, max_tries=8) retry(job.reload)() @@ -770,6 +772,7 @@ def _load_table_for_dml(self, rows, dataset_name, table_name): ) # Retry until done. + job.result() retry = RetryInstanceState(_job_done, max_tries=8) retry(job.reload)() self._fetch_single_page(table) @@ -799,7 +802,9 @@ def test_dbapi_w_dml(self): WHERE greeting = 'Hello World' """ - Config.CURSOR.execute(query_template.format(dataset_name, table_name)) + Config.CURSOR.execute( + query_template.format(dataset_name, table_name), + job_id='test_dbapi_w_dml_{}'.format(str(uuid.uuid4()))) self.assertEqual(Config.CURSOR.rowcount, 1) self.assertIsNone(Config.CURSOR.fetchone()) From 32ca223846a1a7b9d6f7efc07c839c1c897c5816 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 10 Aug 2017 16:36:31 -0700 Subject: [PATCH 2/3] Use unique_resource_id for job IDs --- bigquery/tests/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index 65a6522166af..ac16f8816797 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -804,7 +804,7 @@ def test_dbapi_w_dml(self): Config.CURSOR.execute( query_template.format(dataset_name, table_name), - job_id='test_dbapi_w_dml_{}'.format(str(uuid.uuid4()))) + job_id='test_dbapi_w_dml_{}'.format(unique_resource_id())) self.assertEqual(Config.CURSOR.rowcount, 1) self.assertIsNone(Config.CURSOR.fetchone()) From 562a26137cfb2567b5298733977df56800eaef66 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 10 Aug 2017 16:40:42 -0700 Subject: [PATCH 3/3] Remove redundant RetryInstanceState and add timeout to futures API .result calls --- bigquery/tests/system.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index ac16f8816797..5d0b38ffac41 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -35,6 +35,7 @@ from test_utils.system import unique_resource_id +JOB_TIMEOUT = 120 # 2 minutes WHERE = os.path.abspath(os.path.dirname(__file__)) @@ -381,9 +382,7 @@ def test_load_table_from_local_file_then_dump_table(self): ) # Retry until done. - job.result() - retry = RetryInstanceState(_job_done, max_tries=8) - retry(job.reload)() + job.result(timeout=JOB_TIMEOUT) self.assertEqual(job.output_rows, len(ROWS)) @@ -420,9 +419,7 @@ def test_load_table_from_local_avro_file_then_dump_table(self): ) # Retry until done. - job.result() - retry = RetryInstanceState(_job_done, max_tries=8) - retry(job.reload)() + job.result(timeout=JOB_TIMEOUT) self.assertEqual(job.output_rows, len(ROWS)) @@ -772,9 +769,7 @@ def _load_table_for_dml(self, rows, dataset_name, table_name): ) # Retry until done. - job.result() - retry = RetryInstanceState(_job_done, max_tries=8) - retry(job.reload)() + job.result(timeout=JOB_TIMEOUT) self._fetch_single_page(table) def test_sync_query_w_dml(self): @@ -1091,7 +1086,7 @@ def test_async_query_future(self): str(uuid.uuid4()), 'SELECT 1') query_job.use_legacy_sql = False - iterator = query_job.result().fetch_data() + iterator = query_job.result(timeout=JOB_TIMEOUT).fetch_data() rows = list(iterator) self.assertEqual(rows, [(1,)])