From 10f515e6fdc676115d1057761dafb8bf649e7e90 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Fri, 10 May 2019 19:41:06 -0300 Subject: [PATCH 1/9] Add Data Catalog lookupEntry samples and tests --- datacatalog/cloud-client/pom.xml | 72 +++++++++++ .../datacatalog/LookupEntryExample.java | 116 ++++++++++++++++++ .../datacatalog/LookupEntryExampleTest.java | 112 +++++++++++++++++ 3 files changed, 300 insertions(+) create mode 100644 datacatalog/cloud-client/pom.xml create mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java create mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java diff --git a/datacatalog/cloud-client/pom.xml b/datacatalog/cloud-client/pom.xml new file mode 100644 index 00000000000..fa7b6dccb1f --- /dev/null +++ b/datacatalog/cloud-client/pom.xml @@ -0,0 +1,72 @@ + + + + 4.0.0 + com.example.datacatalog + datacatalog-google-cloud-samples + jar + + + + com.google.cloud.samples + shared-configuration + 1.0.11 + + + + 1.8 + 1.8 + + + + + com.google.cloud + google-cloud-datacatalog + 0.4.0-alpha + + + + + junit + junit + 4.13-beta-2 + test + + + com.google.truth + truth + 0.42 + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + false + + + + + diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java new file mode 100644 index 00000000000..42133a0d371 --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import com.google.api.gax.rpc.ApiException; +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.LookupEntryRequest; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; + +public class LookupEntryExample { + + /** + * Lookup a catalog entry. + * + * @param args projectId + * resourceType + * { bigquery-dataset | bigquery-table | pubsub-topic }, + * datasetId, + * tableId, + * topicId, + * useSqlResource (optional) + * @throws Exception exception thrown if operation is unsuccessful + */ + public static void main(String... args) throws Exception { + + LookupEntryRequest request = buildRequest(args); + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + Entry entry = dataCatalogClient.lookupEntry(request); + System.out.printf("Entry name: %s:\n", entry.getName()); + } catch (ApiException e) { + System.out.print(e.getStatusCode().getCode()); + System.out.print(e.isRetryable()); + } + } + + private static LookupEntryRequest buildRequest(String... args) { + + String projectId = args[0]; + String resourceType = args[1]; + + boolean useSqlResource = "--sql-resource".equals(args[args.length - 1]); + + return useSqlResource + ? buildSqlResourceRequest(projectId, resourceType, args) + : buildLinkedResourceRequest(projectId, resourceType, args); + } + + private static LookupEntryRequest buildLinkedResourceRequest( + String projectId, String resourceType, String... args) { + + String linkedResource = null; + + switch (resourceType) { + case "bigquery-dataset": + linkedResource = String.format( + "//bigquery.googleapis.com/projects/%s/datasets/%s", + projectId, args[2]); + break; + case "bigquery-table": + linkedResource = String.format( + "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", + projectId, args[2], args[3]); + break; + case "pubsub-topic": + linkedResource = String.format( + "//pubsub.googleapis.com/projects/%s/topics/%s", + projectId, args[2]); + break; + default: + break; + } + + return LookupEntryRequest.newBuilder() + .setLinkedResource(linkedResource).build(); + } + + private static LookupEntryRequest buildSqlResourceRequest( + String projectId, String resourceType, String... args) { + + String sqlResource = null; + + switch (resourceType) { + case "bigquery-dataset": + sqlResource = String.format("bigquery.dataset.`%s`.`%s`", + projectId, args[2]); + break; + case "bigquery-table": + sqlResource = String.format("bigquery.table.`%s`.`%s`.`%s`", + projectId, args[2], args[3]); + break; + case "pubsub-topic": + sqlResource = String.format("pubsub.topic.`%s`.`%s`", + projectId, args[2]); + break; + default: + break; + } + + return LookupEntryRequest.newBuilder() + .setSqlResource(sqlResource).build(); + } +} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java new file mode 100644 index 00000000000..1f759fe3bcc --- /dev/null +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java @@ -0,0 +1,112 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LookupEntryExampleTest { + + private static final String BIGQUERY_PROJECT = "bigquery-public-data"; + private static final String BIGQUERY_DATASET = "new_york_taxi_trips"; + private static final String BIGQUERY_TABLE = "taxi_zone_geom"; + + private static final String PUBSUB_PROJECT = "pubsub-public-data"; + private static final String PUBSUB_TOPIC = "taxirides-realtime"; + + private ByteArrayOutputStream bout; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() throws IOException { + bout.close(); + System.setOut(null); + } + + @Test + public void testLookupBigQueryDataset() throws Exception { + LookupEntryExample + .main(BIGQUERY_PROJECT, "bigquery-dataset", BIGQUERY_DATASET); + String got = bout.toString(); + assertThat(got).contains("projects/" + BIGQUERY_PROJECT); + assertThat(got).contains("entryGroups/@bigquery/entries"); + } + + @Test + public void testLookupBigQueryDatasetSqlResource() throws Exception { + LookupEntryExample + .main(BIGQUERY_PROJECT, "bigquery-dataset", BIGQUERY_DATASET, + "--sql-resource"); + String got = bout.toString(); + assertThat(got).contains("projects/" + BIGQUERY_PROJECT); + assertThat(got).contains("entryGroups/@bigquery/entries"); + } + + @Test + public void testLookupBigQueryTable() throws Exception { + LookupEntryExample + .main(BIGQUERY_PROJECT, "bigquery-table", BIGQUERY_DATASET, + BIGQUERY_TABLE); + String got = bout.toString(); + assertThat(got).contains("projects/" + BIGQUERY_PROJECT); + assertThat(got).contains("entryGroups/@bigquery/entries"); + } + + @Test + public void testLookupBigQueryTableSqlResource() throws Exception { + LookupEntryExample + .main(BIGQUERY_PROJECT, "bigquery-table", BIGQUERY_DATASET, + BIGQUERY_TABLE, "--sql-resource"); + String got = bout.toString(); + assertThat(got).contains("projects/" + BIGQUERY_PROJECT); + assertThat(got).contains("entryGroups/@bigquery/entries"); + } + + @Test + public void testLookupPubSubTopic() throws Exception { + LookupEntryExample + .main(PUBSUB_PROJECT, "pubsub-topic", PUBSUB_TOPIC); + String got = bout.toString(); + assertThat(got).contains("projects/" + PUBSUB_PROJECT); + assertThat(got).contains("entryGroups/@pubsub/entries"); + } + + @Test + public void testLookupPubSubTopicSqlResource() throws Exception { + LookupEntryExample + .main(PUBSUB_PROJECT, "pubsub-topic", PUBSUB_TOPIC, + "--sql-resource"); + String got = bout.toString(); + assertThat(got).contains("projects/" + PUBSUB_PROJECT); + assertThat(got).contains("entryGroups/@pubsub/entries"); + } +} From d4c04bd59e711dd2f6e5d01c0bdbf049a763a26a Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Mon, 13 May 2019 14:14:15 -0300 Subject: [PATCH 2/9] Improve code readability --- .../datacatalog/LookupEntryExample.java | 162 +++++++++++------- 1 file changed, 103 insertions(+), 59 deletions(-) diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java index 42133a0d371..35a136aa4cd 100644 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java @@ -23,12 +23,98 @@ public class LookupEntryExample { + private static Entry lookupBigQueryDataset(String projectId, + String datasetId) throws Exception { + + String linkedResource = String.format( + "//bigquery.googleapis.com/projects/%s/datasets/%s", + projectId, datasetId); + + LookupEntryRequest request = LookupEntryRequest.newBuilder() + .setLinkedResource(linkedResource).build(); + + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + return dataCatalogClient.lookupEntry(request); + } + } + + private static Entry lookupBigQueryDatasetSqlResource(String projectId, + String datasetId) throws Exception { + + String sqlResource = String.format( + "bigquery.dataset.`%s`.`%s`", projectId, datasetId); + + LookupEntryRequest request = LookupEntryRequest.newBuilder() + .setSqlResource(sqlResource).build(); + + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + return dataCatalogClient.lookupEntry(request); + } + } + + private static Entry lookupBigQueryTable(String projectId, String datasetId, + String tableId) throws Exception { + + String linkedResource = String.format( + "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", + projectId, datasetId, tableId); + + LookupEntryRequest request = LookupEntryRequest.newBuilder() + .setLinkedResource(linkedResource).build(); + + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + return dataCatalogClient.lookupEntry(request); + } + } + + private static Entry lookupBigQueryTableSqlResource(String projectId, + String datasetId, String tableId) throws Exception { + + String sqlResource = String.format( + "bigquery.table.`%s`.`%s`.`%s`", projectId, datasetId, tableId); + + LookupEntryRequest request = LookupEntryRequest.newBuilder() + .setSqlResource(sqlResource).build(); + + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + return dataCatalogClient.lookupEntry(request); + } + } + + private static Entry lookupPubSubTopic(String projectId, String topicId) + throws Exception { + + String linkedResource = String.format( + "//pubsub.googleapis.com/projects/%s/topics/%s", projectId, topicId); + + LookupEntryRequest request = LookupEntryRequest.newBuilder() + .setLinkedResource(linkedResource).build(); + + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + return dataCatalogClient.lookupEntry(request); + } + } + + private static Entry lookupPubSubTopicSqlResource(String projectId, + String topicId) throws Exception { + + String sqlResource = String.format( + "pubsub.topic.`%s`.`%s`", projectId, topicId); + + LookupEntryRequest request = LookupEntryRequest.newBuilder() + .setSqlResource(sqlResource).build(); + + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + return dataCatalogClient.lookupEntry(request); + } + } + /** * Lookup a catalog entry. * * @param args projectId * resourceType - * { bigquery-dataset | bigquery-table | pubsub-topic }, + * { bigquery-dataset | bigquery-table | pubsub-topic }, * datasetId, * tableId, * topicId, @@ -37,80 +123,38 @@ public class LookupEntryExample { */ public static void main(String... args) throws Exception { - LookupEntryRequest request = buildRequest(args); - try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - Entry entry = dataCatalogClient.lookupEntry(request); - System.out.printf("Entry name: %s:\n", entry.getName()); + try { + Entry entry = lookupEntry(args); + if (entry != null) { + System.out.printf("Entry name: %s:\n", entry.getName()); + } } catch (ApiException e) { System.out.print(e.getStatusCode().getCode()); - System.out.print(e.isRetryable()); } } - private static LookupEntryRequest buildRequest(String... args) { + private static Entry lookupEntry(String... args) throws Exception { String projectId = args[0]; String resourceType = args[1]; boolean useSqlResource = "--sql-resource".equals(args[args.length - 1]); - return useSqlResource - ? buildSqlResourceRequest(projectId, resourceType, args) - : buildLinkedResourceRequest(projectId, resourceType, args); - } - - private static LookupEntryRequest buildLinkedResourceRequest( - String projectId, String resourceType, String... args) { - - String linkedResource = null; - - switch (resourceType) { - case "bigquery-dataset": - linkedResource = String.format( - "//bigquery.googleapis.com/projects/%s/datasets/%s", - projectId, args[2]); - break; - case "bigquery-table": - linkedResource = String.format( - "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", - projectId, args[2], args[3]); - break; - case "pubsub-topic": - linkedResource = String.format( - "//pubsub.googleapis.com/projects/%s/topics/%s", - projectId, args[2]); - break; - default: - break; - } - - return LookupEntryRequest.newBuilder() - .setLinkedResource(linkedResource).build(); - } - - private static LookupEntryRequest buildSqlResourceRequest( - String projectId, String resourceType, String... args) { - - String sqlResource = null; - switch (resourceType) { case "bigquery-dataset": - sqlResource = String.format("bigquery.dataset.`%s`.`%s`", - projectId, args[2]); - break; + return useSqlResource ? + lookupBigQueryDatasetSqlResource(projectId, args[2]) : + lookupBigQueryDataset(projectId, args[2]); case "bigquery-table": - sqlResource = String.format("bigquery.table.`%s`.`%s`.`%s`", - projectId, args[2], args[3]); - break; + return useSqlResource ? + lookupBigQueryTableSqlResource(projectId, args[2], args[3]) : + lookupBigQueryTable(projectId, args[2], args[3]); case "pubsub-topic": - sqlResource = String.format("pubsub.topic.`%s`.`%s`", - projectId, args[2]); - break; + return useSqlResource ? + lookupPubSubTopicSqlResource(projectId, args[2]) : + lookupPubSubTopic(projectId, args[2]); default: - break; + return null; } - - return LookupEntryRequest.newBuilder() - .setSqlResource(sqlResource).build(); } } From a61479d5e7c215b5fc19c871f7c117fcf27b3eee Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Mon, 13 May 2019 16:04:19 -0300 Subject: [PATCH 3/9] Assert integration tests using regex --- datacatalog/cloud-client/README.md | 0 .../datacatalog/LookupEntryExample.java | 42 +++++++++++++----- .../datacatalog/LookupEntryExampleTest.java | 44 +++++++++++-------- 3 files changed, 58 insertions(+), 28 deletions(-) create mode 100644 datacatalog/cloud-client/README.md diff --git a/datacatalog/cloud-client/README.md b/datacatalog/cloud-client/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java index 35a136aa4cd..e01fa131b92 100644 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java @@ -35,7 +35,11 @@ private static Entry lookupBigQueryDataset(String projectId, try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { return dataCatalogClient.lookupEntry(request); + } catch (ApiException e) { + System.out.print(e.getStatusCode().getCode()); } + + return null; } private static Entry lookupBigQueryDatasetSqlResource(String projectId, @@ -49,7 +53,11 @@ private static Entry lookupBigQueryDatasetSqlResource(String projectId, try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { return dataCatalogClient.lookupEntry(request); + } catch (ApiException e) { + System.out.print(e.getStatusCode().getCode()); } + + return null; } private static Entry lookupBigQueryTable(String projectId, String datasetId, @@ -64,7 +72,11 @@ private static Entry lookupBigQueryTable(String projectId, String datasetId, try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { return dataCatalogClient.lookupEntry(request); + } catch (ApiException e) { + System.out.print(e.getStatusCode().getCode()); } + + return null; } private static Entry lookupBigQueryTableSqlResource(String projectId, @@ -78,7 +90,11 @@ private static Entry lookupBigQueryTableSqlResource(String projectId, try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { return dataCatalogClient.lookupEntry(request); + } catch (ApiException e) { + System.out.print(e.getStatusCode().getCode()); } + + return null; } private static Entry lookupPubSubTopic(String projectId, String topicId) @@ -92,7 +108,11 @@ private static Entry lookupPubSubTopic(String projectId, String topicId) try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { return dataCatalogClient.lookupEntry(request); + } catch (ApiException e) { + System.out.print(e.getStatusCode().getCode()); } + + return null; } private static Entry lookupPubSubTopicSqlResource(String projectId, @@ -106,30 +126,31 @@ private static Entry lookupPubSubTopicSqlResource(String projectId, try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { return dataCatalogClient.lookupEntry(request); + } catch (ApiException e) { + System.out.print(e.getStatusCode().getCode()); } + + return null; } /** * Lookup a catalog entry. * * @param args projectId - * resourceType * { bigquery-dataset | bigquery-table | pubsub-topic }, - * datasetId, - * tableId, - * topicId, - * useSqlResource (optional) - * @throws Exception exception thrown if operation is unsuccessful + * datasetId | topicId, + * tableId (required when looking for tables), + * --sql-resource (optional) */ - public static void main(String... args) throws Exception { + public static void main(String... args) { try { Entry entry = lookupEntry(args); if (entry != null) { - System.out.printf("Entry name: %s:\n", entry.getName()); + System.out.printf("Entry name: %s\n", entry.getName()); } - } catch (ApiException e) { - System.out.print(e.getStatusCode().getCode()); + } catch (Exception e) { + System.out.print("Error during lookupEntry:\n" + e.toString()); } } @@ -154,6 +175,7 @@ private static Entry lookupEntry(String... args) throws Exception { lookupPubSubTopicSqlResource(projectId, args[2]) : lookupPubSubTopic(projectId, args[2]); default: + System.out.printf("Invalid resource type: %s\n", resourceType); return null; } } diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java index 1f759fe3bcc..f58d1a9a1f7 100644 --- a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java @@ -53,60 +53,68 @@ public void tearDown() throws IOException { } @Test - public void testLookupBigQueryDataset() throws Exception { + public void testLookupBigQueryDataset() { LookupEntryExample .main(BIGQUERY_PROJECT, "bigquery-dataset", BIGQUERY_DATASET); String got = bout.toString(); - assertThat(got).contains("projects/" + BIGQUERY_PROJECT); - assertThat(got).contains("entryGroups/@bigquery/entries"); + assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT + + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); } @Test - public void testLookupBigQueryDatasetSqlResource() throws Exception { + public void testLookupBigQueryDatasetSqlResource() { LookupEntryExample .main(BIGQUERY_PROJECT, "bigquery-dataset", BIGQUERY_DATASET, "--sql-resource"); String got = bout.toString(); - assertThat(got).contains("projects/" + BIGQUERY_PROJECT); - assertThat(got).contains("entryGroups/@bigquery/entries"); + assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT + + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); } @Test - public void testLookupBigQueryTable() throws Exception { + public void testLookupBigQueryTable() { LookupEntryExample .main(BIGQUERY_PROJECT, "bigquery-table", BIGQUERY_DATASET, BIGQUERY_TABLE); String got = bout.toString(); - assertThat(got).contains("projects/" + BIGQUERY_PROJECT); - assertThat(got).contains("entryGroups/@bigquery/entries"); + assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT + + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); } @Test - public void testLookupBigQueryTableSqlResource() throws Exception { + public void testLookupBigQueryTableSqlResource() { LookupEntryExample .main(BIGQUERY_PROJECT, "bigquery-table", BIGQUERY_DATASET, BIGQUERY_TABLE, "--sql-resource"); String got = bout.toString(); - assertThat(got).contains("projects/" + BIGQUERY_PROJECT); - assertThat(got).contains("entryGroups/@bigquery/entries"); + assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT + + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); } @Test - public void testLookupPubSubTopic() throws Exception { + public void testLookupPubSubTopic() { LookupEntryExample .main(PUBSUB_PROJECT, "pubsub-topic", PUBSUB_TOPIC); String got = bout.toString(); - assertThat(got).contains("projects/" + PUBSUB_PROJECT); - assertThat(got).contains("entryGroups/@pubsub/entries"); + assertThat(got).containsMatch("projects/" + PUBSUB_PROJECT + + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); } @Test - public void testLookupPubSubTopicSqlResource() throws Exception { + public void testLookupPubSubTopicSqlResource() { LookupEntryExample .main(PUBSUB_PROJECT, "pubsub-topic", PUBSUB_TOPIC, "--sql-resource"); String got = bout.toString(); - assertThat(got).contains("projects/" + PUBSUB_PROJECT); - assertThat(got).contains("entryGroups/@pubsub/entries"); + assertThat(got).containsMatch("projects/" + PUBSUB_PROJECT + + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); + } + + @Test + public void testInvalidResourceType() { + LookupEntryExample.main(BIGQUERY_PROJECT, "dataset", + BIGQUERY_DATASET); + String got = bout.toString(); + assertThat(got).startsWith("Invalid resource type: dataset"); } } From 939eacecab519b1159cb82376ae6c39d251f407a Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Mon, 13 May 2019 16:04:36 -0300 Subject: [PATCH 4/9] Add readme file --- datacatalog/cloud-client/README.md | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/datacatalog/cloud-client/README.md b/datacatalog/cloud-client/README.md index e69de29bb2d..74983c01b2e 100644 --- a/datacatalog/cloud-client/README.md +++ b/datacatalog/cloud-client/README.md @@ -0,0 +1,49 @@ +# Getting Started with Data Catalog and the Google Cloud Client libraries + + +Open in Cloud Shell + +[Data Catalog][datacatalog] is a fully managed and scalable metadata management service that empowers organizations +to quickly discover, manage, and understand all their data in Google Cloud. +This sample Java application demonstrates how to access the Data Catalog API using +the [Google Cloud Client Library for Java][google-cloud-java]. + +[datacatalog]: https://cloud.google.com/data-catalog/ +[google-cloud-java]: https://github.com/GoogleCloudPlatform/google-cloud-java + +## Quickstart + +#### Setup +- Install [Maven](http://maven.apache.org/). +- [Enable](https://console.cloud.google.com/apis/api/datacatalog.googleapis.com/overview) Data Catalog API. +- Set up [authentication](https://cloud.google.com/docs/authentication/getting-started). + +#### Build +- Build your project with: +``` + mvn clean package -DskipTests +``` + +#### Lookup a BigQuery Dataset +``` + mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryExample -Dexec.args="my-project bigquery-dataset my_dataset" +``` + +#### Lookup a BigQuery Table +``` + mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryExample -Dexec.args="my-project bigquery-table my_dataset my_table" +``` + +#### Lookup a Pub/Sub Topic +``` + mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryExample -Dexec.args="my-project pubsub-topic my-topic" +``` + +#### Lookup by Sql Resource +Append `--sql-resource` to command args to lookup by Sql Resource instead of Linked Resource. + +#### Testing +Run the test with Maven. +``` + mvn verify +``` From 21f624f13e442f47f6349be483c3aa6a2dc555f0 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Mon, 13 May 2019 16:06:32 -0300 Subject: [PATCH 5/9] Update root readme file --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3f7fbcc9419..6d08e3e677f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ This is a repository that contains java code snippets on [Cloud Platform Documen Technology Samples: * [Bigquery](bigquery) +* [Data Catalog](datacatalog) * [Datastore](datastore) * [Endpoints](endpoints) * [Identity-Aware Proxy](iap) From d82c61e95f1c6b6c915edf93764e593dd5ea52c9 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Mon, 13 May 2019 20:39:34 -0300 Subject: [PATCH 6/9] Update code to keep consistecy with repo standards --- datacatalog/cloud-client/README.md | 8 +- datacatalog/cloud-client/build.gradle | 41 ++++ .../LookupEntryBigQueryDataset.java | 72 +++++++ .../datacatalog/LookupEntryBigQueryTable.java | 77 ++++++++ .../datacatalog/LookupEntryExample.java | 182 ------------------ .../datacatalog/LookupEntryPubSubTopic.java | 72 +++++++ .../LookupEntryBigQueryDatasetTest.java | 68 +++++++ .../LookupEntryBigQueryTableTest.java | 70 +++++++ .../datacatalog/LookupEntryExampleTest.java | 120 ------------ .../LookupEntryPubSubTopicTest.java | 68 +++++++ 10 files changed, 472 insertions(+), 306 deletions(-) create mode 100644 datacatalog/cloud-client/build.gradle create mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java create mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java delete mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java create mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java create mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java create mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryTableTest.java delete mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java create mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java diff --git a/datacatalog/cloud-client/README.md b/datacatalog/cloud-client/README.md index 74983c01b2e..2ac4dd85c9d 100644 --- a/datacatalog/cloud-client/README.md +++ b/datacatalog/cloud-client/README.md @@ -26,21 +26,21 @@ the [Google Cloud Client Library for Java][google-cloud-java]. #### Lookup a BigQuery Dataset ``` - mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryExample -Dexec.args="my-project bigquery-dataset my_dataset" + mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryBigQueryDataset -Dexec.args="my-project my_dataset" ``` #### Lookup a BigQuery Table ``` - mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryExample -Dexec.args="my-project bigquery-table my_dataset my_table" + mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryBigQueryTable -Dexec.args="my-project my_dataset my_table" ``` #### Lookup a Pub/Sub Topic ``` - mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryExample -Dexec.args="my-project pubsub-topic my-topic" + mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryPubSubTopic -Dexec.args="my-project my-topic" ``` #### Lookup by Sql Resource -Append `--sql-resource` to command args to lookup by Sql Resource instead of Linked Resource. +Append `-lookupBySqlResource` to command args to lookup by Sql Resource instead of Linked Resource. #### Testing Run the test with Maven. diff --git a/datacatalog/cloud-client/build.gradle b/datacatalog/cloud-client/build.gradle new file mode 100644 index 00000000000..cb6066b14b0 --- /dev/null +++ b/datacatalog/cloud-client/build.gradle @@ -0,0 +1,41 @@ +// Copyright 2019 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +apply plugin: 'java' + +repositories { + mavenCentral() +} + +dependencies { + compile group: 'com.google.cloud', name: 'google-cloud-datacatalog', version:'0.4.0-alpha' + + testCompile group: 'com.google.truth', name: 'truth', version:'0.42' + testCompile group: 'junit', name: 'junit', version:'4.13-beta-2' +} + +test { + useJUnit() + testLogging.showStandardStreams = true + beforeTest { descriptor -> + logger.lifecycle("test: " + descriptor + " Running") + } + + onOutput { descriptor, event -> + logger.lifecycle("test: " + descriptor + ": " + event.message ) + } + afterTest { descriptor, result -> + logger.lifecycle("test: " + descriptor + ": " + result ) + } +} diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java new file mode 100644 index 00000000000..7dc361ee181 --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.LookupEntryRequest; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; + +public class LookupEntryBigQueryDataset { + + /** + * Lookup the Data Catalog entry referring to a BigQuery Dataset + * + * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' + * @param datasetId The dataset ID to which the Catalog Entry refers, e.g. 'my_dataset' + * @param lookupBySqlResource Indicates whether the lookup should be performed by Sql Resource + * instead of Linked Resource, e.g. 'false' + */ + private static void lookupEntryBigQueryDataset( + String projectId, String datasetId, boolean lookupBySqlResource) { + + LookupEntryRequest request; + + // Construct the Lookup request to be sent by the client. + if (lookupBySqlResource) { + String sqlResource = String.format("bigquery.dataset.`%s`.`%s`", projectId, datasetId); + request = LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); + } else { + String linkedResource = + String.format("//bigquery.googleapis.com/projects/%s/datasets/%s", projectId, datasetId); + request = LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); + } + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + Entry entry = dataCatalogClient.lookupEntry(request); + System.out.printf("Entry name: %s\n", entry.getName()); + } catch (Exception e) { + System.out.print("Error during lookupEntryBigQueryDataset:\n" + e.toString()); + } + } + + /** + * Command line application to lookup the Data Catalog entry referring to a BigQuery Dataset. + * Requires 2 positional args: projectId and datasetId. A third arg is optional: + * lookupBySqlResource (when set, the lookup is done by Sql Resource instead of Linked Resource). + */ + public static void main(String... args) { + + String projectId = args[0]; + String datasetId = args[1]; + boolean lookupBySqlResource = "-lookupBySqlResource".equals(args[args.length - 1]); + + lookupEntryBigQueryDataset(projectId, datasetId, lookupBySqlResource); + } +} diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java new file mode 100644 index 00000000000..e3cffdbd65a --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java @@ -0,0 +1,77 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.LookupEntryRequest; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; + +public class LookupEntryBigQueryTable { + + /** + * Lookup the Data Catalog entry referring to a BigQuery Table + * + * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' + * @param datasetId The dataset ID to which the Table belongs, e.g. 'my_dataset' + * @param tableId The table ID to which the Catalog Entry refers, e.g. 'my_table' + * @param lookupBySqlResource Indicates whether the lookup should be performed by Sql Resource + * instead of Linked Resource, e.g. 'false' + */ + private static void lookupEntryBigQueryTable( + String projectId, String datasetId, String tableId, boolean lookupBySqlResource) { + + LookupEntryRequest request; + + // Construct the Lookup request to be sent by the client. + if (lookupBySqlResource) { + String sqlResource = + String.format("bigquery.table.`%s`.`%s`.`%s`", projectId, datasetId, tableId); + request = LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); + } else { + String linkedResource = + String.format( + "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", + projectId, datasetId, tableId); + request = LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); + } + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + Entry entry = dataCatalogClient.lookupEntry(request); + System.out.printf("Entry name: %s\n", entry.getName()); + } catch (Exception e) { + System.out.print("Error during lookupEntryBigQueryTable:\n" + e.toString()); + } + } + + /** + * Command line application to lookup the Data Catalog entry referring to a BigQuery Table. + * Requires 3 positional args: projectId, datasetId, and tableId. A fourth arg is optional: + * lookupBySqlResource (when set, the lookup is done by Sql Resource instead of Linked Resource). + */ + public static void main(String... args) { + + String projectId = args[0]; + String datasetId = args[1]; + String tableId = args[2]; + boolean lookupBySqlResource = "-lookupBySqlResource".equals(args[args.length - 1]); + + lookupEntryBigQueryTable(projectId, datasetId, tableId, lookupBySqlResource); + } +} diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java deleted file mode 100644 index e01fa131b92..00000000000 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryExample.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2019 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.datacatalog; - -import com.google.api.gax.rpc.ApiException; -import com.google.cloud.datacatalog.Entry; -import com.google.cloud.datacatalog.LookupEntryRequest; -import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; - -public class LookupEntryExample { - - private static Entry lookupBigQueryDataset(String projectId, - String datasetId) throws Exception { - - String linkedResource = String.format( - "//bigquery.googleapis.com/projects/%s/datasets/%s", - projectId, datasetId); - - LookupEntryRequest request = LookupEntryRequest.newBuilder() - .setLinkedResource(linkedResource).build(); - - try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - return dataCatalogClient.lookupEntry(request); - } catch (ApiException e) { - System.out.print(e.getStatusCode().getCode()); - } - - return null; - } - - private static Entry lookupBigQueryDatasetSqlResource(String projectId, - String datasetId) throws Exception { - - String sqlResource = String.format( - "bigquery.dataset.`%s`.`%s`", projectId, datasetId); - - LookupEntryRequest request = LookupEntryRequest.newBuilder() - .setSqlResource(sqlResource).build(); - - try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - return dataCatalogClient.lookupEntry(request); - } catch (ApiException e) { - System.out.print(e.getStatusCode().getCode()); - } - - return null; - } - - private static Entry lookupBigQueryTable(String projectId, String datasetId, - String tableId) throws Exception { - - String linkedResource = String.format( - "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", - projectId, datasetId, tableId); - - LookupEntryRequest request = LookupEntryRequest.newBuilder() - .setLinkedResource(linkedResource).build(); - - try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - return dataCatalogClient.lookupEntry(request); - } catch (ApiException e) { - System.out.print(e.getStatusCode().getCode()); - } - - return null; - } - - private static Entry lookupBigQueryTableSqlResource(String projectId, - String datasetId, String tableId) throws Exception { - - String sqlResource = String.format( - "bigquery.table.`%s`.`%s`.`%s`", projectId, datasetId, tableId); - - LookupEntryRequest request = LookupEntryRequest.newBuilder() - .setSqlResource(sqlResource).build(); - - try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - return dataCatalogClient.lookupEntry(request); - } catch (ApiException e) { - System.out.print(e.getStatusCode().getCode()); - } - - return null; - } - - private static Entry lookupPubSubTopic(String projectId, String topicId) - throws Exception { - - String linkedResource = String.format( - "//pubsub.googleapis.com/projects/%s/topics/%s", projectId, topicId); - - LookupEntryRequest request = LookupEntryRequest.newBuilder() - .setLinkedResource(linkedResource).build(); - - try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - return dataCatalogClient.lookupEntry(request); - } catch (ApiException e) { - System.out.print(e.getStatusCode().getCode()); - } - - return null; - } - - private static Entry lookupPubSubTopicSqlResource(String projectId, - String topicId) throws Exception { - - String sqlResource = String.format( - "pubsub.topic.`%s`.`%s`", projectId, topicId); - - LookupEntryRequest request = LookupEntryRequest.newBuilder() - .setSqlResource(sqlResource).build(); - - try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - return dataCatalogClient.lookupEntry(request); - } catch (ApiException e) { - System.out.print(e.getStatusCode().getCode()); - } - - return null; - } - - /** - * Lookup a catalog entry. - * - * @param args projectId - * { bigquery-dataset | bigquery-table | pubsub-topic }, - * datasetId | topicId, - * tableId (required when looking for tables), - * --sql-resource (optional) - */ - public static void main(String... args) { - - try { - Entry entry = lookupEntry(args); - if (entry != null) { - System.out.printf("Entry name: %s\n", entry.getName()); - } - } catch (Exception e) { - System.out.print("Error during lookupEntry:\n" + e.toString()); - } - } - - private static Entry lookupEntry(String... args) throws Exception { - - String projectId = args[0]; - String resourceType = args[1]; - - boolean useSqlResource = "--sql-resource".equals(args[args.length - 1]); - - switch (resourceType) { - case "bigquery-dataset": - return useSqlResource ? - lookupBigQueryDatasetSqlResource(projectId, args[2]) : - lookupBigQueryDataset(projectId, args[2]); - case "bigquery-table": - return useSqlResource ? - lookupBigQueryTableSqlResource(projectId, args[2], args[3]) : - lookupBigQueryTable(projectId, args[2], args[3]); - case "pubsub-topic": - return useSqlResource ? - lookupPubSubTopicSqlResource(projectId, args[2]) : - lookupPubSubTopic(projectId, args[2]); - default: - System.out.printf("Invalid resource type: %s\n", resourceType); - return null; - } - } -} diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java new file mode 100644 index 00000000000..1da99ffb961 --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.LookupEntryRequest; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; + +public class LookupEntryPubSubTopic { + + /** + * Lookup the Data Catalog entry referring to a BigQuery Dataset + * + * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' + * @param topicId The topic ID to which the Catalog Entry refers, e.g. 'my-topic' + * @param lookupBySqlResource Indicates whether the lookup should be performed by Sql Resource + * instead of Linked Resource, e.g. 'false' + */ + private static void lookupEntryPubSubTopic( + String projectId, String topicId, boolean lookupBySqlResource) { + + LookupEntryRequest request; + + // Construct the Lookup request to be sent by the client. + if (lookupBySqlResource) { + String sqlResource = String.format("pubsub.topic.`%s`.`%s`", projectId, topicId); + request = LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); + } else { + String linkedResource = + String.format("//pubsub.googleapis.com/projects/%s/topics/%s", projectId, topicId); + request = LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); + } + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + Entry entry = dataCatalogClient.lookupEntry(request); + System.out.printf("Entry name: %s\n", entry.getName()); + } catch (Exception e) { + System.out.print("Error during lookupEntryPubSubTopic:\n" + e.toString()); + } + } + + /** + * Command line application to lookup the Data Catalog entry referring to a Pub/Sub Topic. + * Requires 2 positional args: projectId and topicId. A third arg is optional: lookupBySqlResource + * (when set, the lookup is done by Sql Resource instead of Linked Resource). + */ + public static void main(String... args) { + + String projectId = args[0]; + String topicId = args[1]; + boolean lookupBySqlResource = "-lookupBySqlResource".equals(args[args.length - 1]); + + lookupEntryPubSubTopic(projectId, topicId, lookupBySqlResource); + } +} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java new file mode 100644 index 00000000000..ab774094c3c --- /dev/null +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LookupEntryBigQueryDatasetTest { + + private static final String BIGQUERY_PROJECT = "bigquery-public-data"; + private static final String BIGQUERY_DATASET = "new_york_taxi_trips"; + + private ByteArrayOutputStream bout; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() throws IOException { + bout.close(); + System.setOut(null); + } + + @Test + public void testLookupEntryBigQueryDataset() { + LookupEntryBigQueryDataset.main(BIGQUERY_PROJECT, BIGQUERY_DATASET); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); + } + + @Test + public void testLookupEntryBigQueryDatasetSqlResource() { + LookupEntryBigQueryDataset.main(BIGQUERY_PROJECT, BIGQUERY_DATASET, "-lookupBySqlResource"); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); + } +} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryTableTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryTableTest.java new file mode 100644 index 00000000000..ebff5e95be0 --- /dev/null +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryTableTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LookupEntryBigQueryTableTest { + + private static final String BIGQUERY_PROJECT = "bigquery-public-data"; + private static final String BIGQUERY_DATASET = "new_york_taxi_trips"; + private static final String BIGQUERY_TABLE = "taxi_zone_geom"; + + private ByteArrayOutputStream bout; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() throws IOException { + bout.close(); + System.setOut(null); + } + + @Test + public void testLookupEntryBigQueryTable() { + LookupEntryBigQueryTable.main(BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); + } + + @Test + public void testLookupEntryBigQueryTableSqlResource() { + LookupEntryBigQueryTable.main( + BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE, "-lookupBySqlResource"); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); + } +} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java deleted file mode 100644 index f58d1a9a1f7..00000000000 --- a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryExampleTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2019 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.datacatalog; - -import static com.google.common.truth.Truth.assertThat; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class LookupEntryExampleTest { - - private static final String BIGQUERY_PROJECT = "bigquery-public-data"; - private static final String BIGQUERY_DATASET = "new_york_taxi_trips"; - private static final String BIGQUERY_TABLE = "taxi_zone_geom"; - - private static final String PUBSUB_PROJECT = "pubsub-public-data"; - private static final String PUBSUB_TOPIC = "taxirides-realtime"; - - private ByteArrayOutputStream bout; - - @Before - public void setUp() { - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - System.setOut(out); - } - - @After - public void tearDown() throws IOException { - bout.close(); - System.setOut(null); - } - - @Test - public void testLookupBigQueryDataset() { - LookupEntryExample - .main(BIGQUERY_PROJECT, "bigquery-dataset", BIGQUERY_DATASET); - String got = bout.toString(); - assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT - + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); - } - - @Test - public void testLookupBigQueryDatasetSqlResource() { - LookupEntryExample - .main(BIGQUERY_PROJECT, "bigquery-dataset", BIGQUERY_DATASET, - "--sql-resource"); - String got = bout.toString(); - assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT - + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); - } - - @Test - public void testLookupBigQueryTable() { - LookupEntryExample - .main(BIGQUERY_PROJECT, "bigquery-table", BIGQUERY_DATASET, - BIGQUERY_TABLE); - String got = bout.toString(); - assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT - + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); - } - - @Test - public void testLookupBigQueryTableSqlResource() { - LookupEntryExample - .main(BIGQUERY_PROJECT, "bigquery-table", BIGQUERY_DATASET, - BIGQUERY_TABLE, "--sql-resource"); - String got = bout.toString(); - assertThat(got).containsMatch("projects/" + BIGQUERY_PROJECT - + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); - } - - @Test - public void testLookupPubSubTopic() { - LookupEntryExample - .main(PUBSUB_PROJECT, "pubsub-topic", PUBSUB_TOPIC); - String got = bout.toString(); - assertThat(got).containsMatch("projects/" + PUBSUB_PROJECT - + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); - } - - @Test - public void testLookupPubSubTopicSqlResource() { - LookupEntryExample - .main(PUBSUB_PROJECT, "pubsub-topic", PUBSUB_TOPIC, - "--sql-resource"); - String got = bout.toString(); - assertThat(got).containsMatch("projects/" + PUBSUB_PROJECT - + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); - } - - @Test - public void testInvalidResourceType() { - LookupEntryExample.main(BIGQUERY_PROJECT, "dataset", - BIGQUERY_DATASET); - String got = bout.toString(); - assertThat(got).startsWith("Invalid resource type: dataset"); - } -} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java new file mode 100644 index 00000000000..eed2077fb0e --- /dev/null +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LookupEntryPubSubTopicTest { + + private static final String PUBSUB_PROJECT = "pubsub-public-data"; + private static final String PUBSUB_TOPIC = "taxirides-realtime"; + + private ByteArrayOutputStream bout; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() throws IOException { + bout.close(); + System.setOut(null); + } + + @Test + public void testLookupPubSubTopic() { + LookupEntryPubSubTopic.main(PUBSUB_PROJECT, PUBSUB_TOPIC); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + PUBSUB_PROJECT + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); + } + + @Test + public void testLookupPubSubTopicSqlResource() { + LookupEntryPubSubTopic.main(PUBSUB_PROJECT, PUBSUB_TOPIC, "-lookupBySqlResource"); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + PUBSUB_PROJECT + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); + } +} From 95e7693ebc9ef11050bb1860388f88f78e4363d5 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes <50331050+ricardosm-cit@users.noreply.github.com> Date: Tue, 14 May 2019 13:21:56 -0300 Subject: [PATCH 7/9] Update LookupEntryBigQueryDataset.java Co-Authored-By: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> --- .../com/example/datacatalog/LookupEntryBigQueryDataset.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java index 7dc361ee181..566fd3a13b2 100644 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java @@ -36,6 +36,8 @@ private static void lookupEntryBigQueryDataset( LookupEntryRequest request; // Construct the Lookup request to be sent by the client. + LookupEntryRequest request; + if (lookupBySqlResource) { if (lookupBySqlResource) { String sqlResource = String.format("bigquery.dataset.`%s`.`%s`", projectId, datasetId); request = LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); From aef7843febddf1fcaccf26266f9b38f45601d096 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Tue, 14 May 2019 14:07:56 -0300 Subject: [PATCH 8/9] Update code to keep consistecyt with repo standards --- datacatalog/cloud-client/README.md | 18 ----- .../LookupEntryBigQueryDataset.java | 42 ++++-------- .../datacatalog/LookupEntryBigQueryTable.java | 52 +++++--------- .../datacatalog/LookupEntryPubSubTopic.java | 40 ++++------- .../com/example/datacatalog/QuickStart.java | 4 ++ .../LookupEntryBigQueryDatasetTest.java | 68 ------------------- .../LookupEntryPubSubTopicTest.java | 68 ------------------- ...ryTableTest.java => LookupEntryTests.java} | 23 +++++-- 8 files changed, 63 insertions(+), 252 deletions(-) create mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java delete mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java delete mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java rename datacatalog/cloud-client/src/test/java/com/example/datacatalog/{LookupEntryBigQueryTableTest.java => LookupEntryTests.java} (73%) diff --git a/datacatalog/cloud-client/README.md b/datacatalog/cloud-client/README.md index 2ac4dd85c9d..f4128e2c4b0 100644 --- a/datacatalog/cloud-client/README.md +++ b/datacatalog/cloud-client/README.md @@ -24,24 +24,6 @@ the [Google Cloud Client Library for Java][google-cloud-java]. mvn clean package -DskipTests ``` -#### Lookup a BigQuery Dataset -``` - mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryBigQueryDataset -Dexec.args="my-project my_dataset" -``` - -#### Lookup a BigQuery Table -``` - mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryBigQueryTable -Dexec.args="my-project my_dataset my_table" -``` - -#### Lookup a Pub/Sub Topic -``` - mvn exec:java -Dexec.mainClass=com.example.datacatalog.LookupEntryPubSubTopic -Dexec.args="my-project my-topic" -``` - -#### Lookup by Sql Resource -Append `-lookupBySqlResource` to command args to lookup by Sql Resource instead of Linked Resource. - #### Testing Run the test with Maven. ``` diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java index 566fd3a13b2..b81737645fc 100644 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java @@ -27,25 +27,21 @@ public class LookupEntryBigQueryDataset { * * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' * @param datasetId The dataset ID to which the Catalog Entry refers, e.g. 'my_dataset' - * @param lookupBySqlResource Indicates whether the lookup should be performed by Sql Resource - * instead of Linked Resource, e.g. 'false' */ - private static void lookupEntryBigQueryDataset( - String projectId, String datasetId, boolean lookupBySqlResource) { + public static void lookupEntry(String projectId, String datasetId) { + // String projectId = "my-project" + // String datasetId = "my_dataset" - LookupEntryRequest request; + // Get an entry by the resource name from the source Google Cloud Platform service. + String linkedResource = + String.format("//bigquery.googleapis.com/projects/%s/datasets/%s", projectId, datasetId); + LookupEntryRequest request = + LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); - // Construct the Lookup request to be sent by the client. - LookupEntryRequest request; - if (lookupBySqlResource) { - if (lookupBySqlResource) { - String sqlResource = String.format("bigquery.dataset.`%s`.`%s`", projectId, datasetId); - request = LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); - } else { - String linkedResource = - String.format("//bigquery.googleapis.com/projects/%s/datasets/%s", projectId, datasetId); - request = LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); - } + // Alternatively, lookup by the SQL name of the entry would have the same result: + // String sqlResource = String.format("bigquery.dataset.`%s`.`%s`", projectId, datasetId); + // LookupEntryRequest request = + // LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. After completing all of your requests, call @@ -57,18 +53,4 @@ private static void lookupEntryBigQueryDataset( System.out.print("Error during lookupEntryBigQueryDataset:\n" + e.toString()); } } - - /** - * Command line application to lookup the Data Catalog entry referring to a BigQuery Dataset. - * Requires 2 positional args: projectId and datasetId. A third arg is optional: - * lookupBySqlResource (when set, the lookup is done by Sql Resource instead of Linked Resource). - */ - public static void main(String... args) { - - String projectId = args[0]; - String datasetId = args[1]; - boolean lookupBySqlResource = "-lookupBySqlResource".equals(args[args.length - 1]); - - lookupEntryBigQueryDataset(projectId, datasetId, lookupBySqlResource); - } } diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java index e3cffdbd65a..bae8e35c361 100644 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java @@ -28,26 +28,25 @@ public class LookupEntryBigQueryTable { * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' * @param datasetId The dataset ID to which the Table belongs, e.g. 'my_dataset' * @param tableId The table ID to which the Catalog Entry refers, e.g. 'my_table' - * @param lookupBySqlResource Indicates whether the lookup should be performed by Sql Resource - * instead of Linked Resource, e.g. 'false' */ - private static void lookupEntryBigQueryTable( - String projectId, String datasetId, String tableId, boolean lookupBySqlResource) { - - LookupEntryRequest request; - - // Construct the Lookup request to be sent by the client. - if (lookupBySqlResource) { - String sqlResource = - String.format("bigquery.table.`%s`.`%s`.`%s`", projectId, datasetId, tableId); - request = LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); - } else { - String linkedResource = - String.format( - "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", - projectId, datasetId, tableId); - request = LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); - } + public static void lookupEntry(String projectId, String datasetId, String tableId) { + // String projectId = "my-project" + // String datasetId = "my_dataset" + // String tableId = "my_table" + + // Get an entry by the resource name from the source Google Cloud Platform service. + String linkedResource = + String.format( + "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", + projectId, datasetId, tableId); + LookupEntryRequest request = + LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); + + // Alternatively, lookup by the SQL name of the entry would have the same result: + // String sqlResource = String.format("bigquery.table.`%s`.`%s`.`%s`", projectId, datasetId, + // tableId); + // LookupEntryRequest request = + // LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. After completing all of your requests, call @@ -59,19 +58,4 @@ private static void lookupEntryBigQueryTable( System.out.print("Error during lookupEntryBigQueryTable:\n" + e.toString()); } } - - /** - * Command line application to lookup the Data Catalog entry referring to a BigQuery Table. - * Requires 3 positional args: projectId, datasetId, and tableId. A fourth arg is optional: - * lookupBySqlResource (when set, the lookup is done by Sql Resource instead of Linked Resource). - */ - public static void main(String... args) { - - String projectId = args[0]; - String datasetId = args[1]; - String tableId = args[2]; - boolean lookupBySqlResource = "-lookupBySqlResource".equals(args[args.length - 1]); - - lookupEntryBigQueryTable(projectId, datasetId, tableId, lookupBySqlResource); - } } diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java index 1da99ffb961..53195fffa9d 100644 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java @@ -27,23 +27,21 @@ public class LookupEntryPubSubTopic { * * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' * @param topicId The topic ID to which the Catalog Entry refers, e.g. 'my-topic' - * @param lookupBySqlResource Indicates whether the lookup should be performed by Sql Resource - * instead of Linked Resource, e.g. 'false' */ - private static void lookupEntryPubSubTopic( - String projectId, String topicId, boolean lookupBySqlResource) { + public static void lookupEntry(String projectId, String topicId) { + // String projectId = "my-project" + // String topicId = "my-topic" - LookupEntryRequest request; + // Get an entry by the resource name from the source Google Cloud Platform service. + String linkedResource = + String.format("//pubsub.googleapis.com/projects/%s/topics/%s", projectId, topicId); + LookupEntryRequest request = + LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); - // Construct the Lookup request to be sent by the client. - if (lookupBySqlResource) { - String sqlResource = String.format("pubsub.topic.`%s`.`%s`", projectId, topicId); - request = LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); - } else { - String linkedResource = - String.format("//pubsub.googleapis.com/projects/%s/topics/%s", projectId, topicId); - request = LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); - } + // Alternatively, lookup by the SQL name of the entry would have the same result: + // String sqlResource = String.format("pubsub.topic.`%s`.`%s`", projectId, topicId); + // LookupEntryRequest request = + // LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. After completing all of your requests, call @@ -55,18 +53,4 @@ private static void lookupEntryPubSubTopic( System.out.print("Error during lookupEntryPubSubTopic:\n" + e.toString()); } } - - /** - * Command line application to lookup the Data Catalog entry referring to a Pub/Sub Topic. - * Requires 2 positional args: projectId and topicId. A third arg is optional: lookupBySqlResource - * (when set, the lookup is done by Sql Resource instead of Linked Resource). - */ - public static void main(String... args) { - - String projectId = args[0]; - String topicId = args[1]; - boolean lookupBySqlResource = "-lookupBySqlResource".equals(args[args.length - 1]); - - lookupEntryPubSubTopic(projectId, topicId, lookupBySqlResource); - } } diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java new file mode 100644 index 00000000000..7e2697b117a --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java @@ -0,0 +1,4 @@ +package com.example.datacatalog; + +public class QuickStart { +} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java deleted file mode 100644 index ab774094c3c..00000000000 --- a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryDatasetTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2019 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.datacatalog; - -import static com.google.common.truth.Truth.assertThat; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class LookupEntryBigQueryDatasetTest { - - private static final String BIGQUERY_PROJECT = "bigquery-public-data"; - private static final String BIGQUERY_DATASET = "new_york_taxi_trips"; - - private ByteArrayOutputStream bout; - - @Before - public void setUp() { - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - System.setOut(out); - } - - @After - public void tearDown() throws IOException { - bout.close(); - System.setOut(null); - } - - @Test - public void testLookupEntryBigQueryDataset() { - LookupEntryBigQueryDataset.main(BIGQUERY_PROJECT, BIGQUERY_DATASET); - String got = bout.toString(); - assertThat(got) - .containsMatch( - "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); - } - - @Test - public void testLookupEntryBigQueryDatasetSqlResource() { - LookupEntryBigQueryDataset.main(BIGQUERY_PROJECT, BIGQUERY_DATASET, "-lookupBySqlResource"); - String got = bout.toString(); - assertThat(got) - .containsMatch( - "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); - } -} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java deleted file mode 100644 index eed2077fb0e..00000000000 --- a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryPubSubTopicTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2019 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.datacatalog; - -import static com.google.common.truth.Truth.assertThat; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class LookupEntryPubSubTopicTest { - - private static final String PUBSUB_PROJECT = "pubsub-public-data"; - private static final String PUBSUB_TOPIC = "taxirides-realtime"; - - private ByteArrayOutputStream bout; - - @Before - public void setUp() { - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - System.setOut(out); - } - - @After - public void tearDown() throws IOException { - bout.close(); - System.setOut(null); - } - - @Test - public void testLookupPubSubTopic() { - LookupEntryPubSubTopic.main(PUBSUB_PROJECT, PUBSUB_TOPIC); - String got = bout.toString(); - assertThat(got) - .containsMatch( - "projects/" + PUBSUB_PROJECT + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); - } - - @Test - public void testLookupPubSubTopicSqlResource() { - LookupEntryPubSubTopic.main(PUBSUB_PROJECT, PUBSUB_TOPIC, "-lookupBySqlResource"); - String got = bout.toString(); - assertThat(got) - .containsMatch( - "projects/" + PUBSUB_PROJECT + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); - } -} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryTableTest.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryTests.java similarity index 73% rename from datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryTableTest.java rename to datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryTests.java index ebff5e95be0..bf254fec42e 100644 --- a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryBigQueryTableTest.java +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryTests.java @@ -28,12 +28,15 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class LookupEntryBigQueryTableTest { +public class LookupEntryTests { private static final String BIGQUERY_PROJECT = "bigquery-public-data"; private static final String BIGQUERY_DATASET = "new_york_taxi_trips"; private static final String BIGQUERY_TABLE = "taxi_zone_geom"; + private static final String PUBSUB_PROJECT = "pubsub-public-data"; + private static final String PUBSUB_TOPIC = "taxirides-realtime"; + private ByteArrayOutputStream bout; @Before @@ -50,8 +53,8 @@ public void tearDown() throws IOException { } @Test - public void testLookupEntryBigQueryTable() { - LookupEntryBigQueryTable.main(BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE); + public void testLookupEntryBigQueryDataset() { + LookupEntryBigQueryDataset.lookupEntry(BIGQUERY_PROJECT, BIGQUERY_DATASET); String got = bout.toString(); assertThat(got) .containsMatch( @@ -59,12 +62,20 @@ public void testLookupEntryBigQueryTable() { } @Test - public void testLookupEntryBigQueryTableSqlResource() { - LookupEntryBigQueryTable.main( - BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE, "-lookupBySqlResource"); + public void testLookupEntryBigQueryTable() { + LookupEntryBigQueryTable.lookupEntry(BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE); String got = bout.toString(); assertThat(got) .containsMatch( "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); } + + @Test + public void testLookupPubSubTopic() { + LookupEntryPubSubTopic.lookupEntry(PUBSUB_PROJECT, PUBSUB_TOPIC); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + PUBSUB_PROJECT + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); + } } From 96dc9c90c8f0b92e6665fee4ee38f6b4039a2565 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Tue, 14 May 2019 14:26:42 -0300 Subject: [PATCH 9/9] Update code to keep consistecy with repo standards --- datacatalog/cloud-client/pom.xml | 13 ------------- .../java/com/example/datacatalog/QuickStart.java | 4 ---- 2 files changed, 17 deletions(-) delete mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java diff --git a/datacatalog/cloud-client/pom.xml b/datacatalog/cloud-client/pom.xml index fa7b6dccb1f..9ccd5c45756 100644 --- a/datacatalog/cloud-client/pom.xml +++ b/datacatalog/cloud-client/pom.xml @@ -56,17 +56,4 @@ test - - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - false - - - - diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java deleted file mode 100644 index 7e2697b117a..00000000000 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/QuickStart.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.example.datacatalog; - -public class QuickStart { -}