From 27da289dda68b4221a1a39f70b5eac60aab9da68 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 10 Jul 2019 10:59:19 -0700 Subject: [PATCH 1/4] bigquery: implement missing DatasetListOption, JobListOption filters. --- .../com/google/cloud/bigquery/BigQuery.java | 16 ++++++++ .../cloud/bigquery/spi/v2/BigQueryRpc.java | 3 ++ .../bigquery/spi/v2/HttpBigQueryRpc.java | 3 ++ .../cloud/bigquery/it/ITBigQueryTest.java | 40 +++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index a566fbeb8421..79cb0f4933c7 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -191,6 +191,14 @@ private DatasetListOption(BigQueryRpc.Option option, Object value) { super(option, value); } + /** + * Returns an option to specify a label filter. See + * https://cloud.google.com/bigquery/docs/adding-using-labels#filtering_datasets_using_labels + */ + public static DatasetListOption labelFilter(String labelFilter) { + return new DatasetListOption(BigQueryRpc.Option.LABEL_FILTER, labelFilter); + } + /** Returns an option to specify the maximum number of datasets returned per page. */ public static DatasetListOption pageSize(long pageSize) { return new DatasetListOption(BigQueryRpc.Option.MAX_RESULTS, pageSize); @@ -387,6 +395,14 @@ public String apply(JobStatus.State state) { return new JobListOption(BigQueryRpc.Option.STATE_FILTER, stringFilters); } + public static JobListOption minCreationTime(long minCreationTime) { + return new JobListOption(BigQueryRpc.Option.MIN_CREATION_TIME, minCreationTime); + } + + public static JobListOption maxCreationTime(long maxCreationTime) { + return new JobListOption(BigQueryRpc.Option.MAX_CREATION_TIME, maxCreationTime); + } + /** Returns an option to specify the maximum number of jobs returned per page. */ public static JobListOption pageSize(long pageSize) { checkArgument(pageSize >= 0); diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index 8255bcfd5a56..7e103408dc6f 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -39,6 +39,9 @@ enum Option { DELETE_CONTENTS("deleteContents"), ALL_DATASETS("all"), ALL_USERS("allUsers"), + LABEL_FILTER("filter"), + MIN_CREATION_TIME("minCreationTime"), + MAX_CREATION_TIME("maxCreationTime"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), START_INDEX("startIndex"), diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 4a83eb32fdbc..39d0fa0d2966 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -128,6 +128,7 @@ public Tuple> listDatasets(String projectId, Map> listJobs(String projectId, Map op .list(projectId) .setAllUsers(Option.ALL_USERS.getBoolean(options)) .setFields(Option.FIELDS.getString(options)) + .setMinCreationTime(BigInteger.valueOf(Option.MIN_CREATION_TIME.getLong(options))) + .setMaxCreationTime(BigInteger.valueOf(Option.MAX_CREATION_TIME.getLong(options))) .setStateFilter(Option.STATE_FILTER.>get(options)) .setMaxResults(Option.MAX_RESULTS.getLong(options)) .setPageToken(Option.PAGE_TOKEN.getString(options)) diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 7ce38134d5a5..e9aebf85c446 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -18,6 +18,7 @@ import static com.google.cloud.bigquery.JobStatus.State.DONE; import static com.google.common.truth.Truth.assertThat; +import static java.lang.System.currentTimeMillis; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -32,6 +33,7 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; import com.google.cloud.bigquery.BigQuery.DatasetField; +import com.google.cloud.bigquery.BigQuery.DatasetListOption; import com.google.cloud.bigquery.BigQuery.DatasetOption; import com.google.cloud.bigquery.BigQuery.JobField; import com.google.cloud.bigquery.BigQuery.JobListOption; @@ -324,6 +326,24 @@ public void testListDatasets() { } } + @Test + public void testListDatasetsWithFilter() { + + String labelFilter = "labels.example-label1:example-value1"; + Page datasets = bigquery.listDatasets(DatasetListOption.labelFilter(labelFilter)); + int count = 0; + for (Dataset dataset : datasets.getValues()) { + assertTrue( + "seeking " + + dataset.getDatasetId().getDataset() + + " label in labels" + + dataset.getLabels(), + dataset.getLabels().containsKey("example-label1")); + count++; + } + assertTrue(count > 0); + } + @Test public void testGetDataset() { Dataset dataset = bigquery.getDataset(DATASET); @@ -1243,6 +1263,26 @@ public void testListJobsWithSelectedFields() { } } + @Test + public void testListJobsWithCreationBounding() { + long currentMillis = currentTimeMillis(); + long lowerBound = currentMillis - 3600 * 1000; + long upperBound = currentMillis; + Page jobs = + bigquery.listJobs( + JobListOption.minCreationTime(lowerBound), JobListOption.maxCreationTime(upperBound)); + long foundMin = upperBound; + long foundMax = lowerBound; + for (Job job : jobs.getValues()) { + foundMin = Math.min(job.getStatistics().getCreationTime(), foundMin); + foundMax = Math.max(job.getStatistics().getCreationTime(), foundMax); + } + assertTrue( + "Found min job time " + foundMin + " earlier than " + lowerBound, foundMin >= lowerBound); + assertTrue( + "Found max job time " + foundMax + " later than " + upperBound, foundMax <= upperBound); + } + @Test public void testCreateAndGetJob() throws InterruptedException, TimeoutException { String sourceTableName = "test_create_and_get_job_source_table"; From 85f1b034024407815feb140e1ce0023bb92960fa Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 10 Jul 2019 11:04:03 -0700 Subject: [PATCH 2/4] update assertion, cleanup message. --- .../com/google/cloud/bigquery/it/ITBigQueryTest.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index e9aebf85c446..eb2d68642cb1 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -334,11 +334,10 @@ public void testListDatasetsWithFilter() { int count = 0; for (Dataset dataset : datasets.getValues()) { assertTrue( - "seeking " - + dataset.getDatasetId().getDataset() - + " label in labels" - + dataset.getLabels(), - dataset.getLabels().containsKey("example-label1")); + "failed to find label key in dataset", dataset.getLabels().containsKey("example-label1")); + assertTrue( + "failed to find label value in dataset", + dataset.getLabels().get("example-label1").equals("example-value1")); count++; } assertTrue(count > 0); From 1eb562e59e23fb48a41c8197d367ca87e60114a7 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 10 Jul 2019 11:27:11 -0700 Subject: [PATCH 3/4] deal with null conversions for min/max creation time. --- .../cloud/bigquery/spi/v2/HttpBigQueryRpc.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 39d0fa0d2966..7d8583fe101a 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -419,19 +419,24 @@ public Job getJob(String projectId, String jobId, String location, Map> listJobs(String projectId, Map options) { try { - JobList jobsList = + Bigquery.Jobs.List request = bigquery .jobs() .list(projectId) .setAllUsers(Option.ALL_USERS.getBoolean(options)) .setFields(Option.FIELDS.getString(options)) - .setMinCreationTime(BigInteger.valueOf(Option.MIN_CREATION_TIME.getLong(options))) - .setMaxCreationTime(BigInteger.valueOf(Option.MAX_CREATION_TIME.getLong(options))) .setStateFilter(Option.STATE_FILTER.>get(options)) .setMaxResults(Option.MAX_RESULTS.getLong(options)) .setPageToken(Option.PAGE_TOKEN.getString(options)) - .setProjection(DEFAULT_PROJECTION) - .execute(); + .setProjection(DEFAULT_PROJECTION); + if (Option.MIN_CREATION_TIME.getLong(options) != null) { + request.setMinCreationTime(BigInteger.valueOf(Option.MIN_CREATION_TIME.getLong(options))); + } + if (Option.MAX_CREATION_TIME.getLong(options) != null) { + request.setMaxCreationTime(BigInteger.valueOf(Option.MAX_CREATION_TIME.getLong(options))); + } + JobList jobsList = request.execute(); + Iterable jobs = jobsList.getJobs(); return Tuple.of( jobsList.getNextPageToken(), From 54750f54c8ea996e4fd7078396c45389dc32b9e4 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 11 Jul 2019 10:30:05 -0700 Subject: [PATCH 4/4] address reviewer comments --- .../src/main/java/com/google/cloud/bigquery/BigQuery.java | 2 ++ .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 79cb0f4933c7..60ce6f971486 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -395,10 +395,12 @@ public String apply(JobStatus.State state) { return new JobListOption(BigQueryRpc.Option.STATE_FILTER, stringFilters); } + /** Returns an option to filter out jobs before the given minimum creation time. */ public static JobListOption minCreationTime(long minCreationTime) { return new JobListOption(BigQueryRpc.Option.MIN_CREATION_TIME, minCreationTime); } + /** Returns an option to filter out jobs after the given maximum creation time. */ public static JobListOption maxCreationTime(long maxCreationTime) { return new JobListOption(BigQueryRpc.Option.MAX_CREATION_TIME, maxCreationTime); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index eb2d68642cb1..12eedb6f7e50 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -328,7 +328,6 @@ public void testListDatasets() { @Test public void testListDatasetsWithFilter() { - String labelFilter = "labels.example-label1:example-value1"; Page datasets = bigquery.listDatasets(DatasetListOption.labelFilter(labelFilter)); int count = 0; @@ -1272,7 +1271,9 @@ public void testListJobsWithCreationBounding() { JobListOption.minCreationTime(lowerBound), JobListOption.maxCreationTime(upperBound)); long foundMin = upperBound; long foundMax = lowerBound; + long jobCount = 0; for (Job job : jobs.getValues()) { + jobCount++; foundMin = Math.min(job.getStatistics().getCreationTime(), foundMin); foundMax = Math.max(job.getStatistics().getCreationTime(), foundMax); } @@ -1280,6 +1281,7 @@ public void testListJobsWithCreationBounding() { "Found min job time " + foundMin + " earlier than " + lowerBound, foundMin >= lowerBound); assertTrue( "Found max job time " + foundMax + " later than " + upperBound, foundMax <= upperBound); + assertTrue("no jobs listed", jobCount > 0); } @Test