From 5fbfed262401a90a993f96779d0e778b4b4e7a97 Mon Sep 17 00:00:00 2001 From: elisheva-qlogic Date: Mon, 7 Jan 2019 11:29:46 -0500 Subject: [PATCH 1/6] tableAdmin sample and tests --- .../cloud/examples/bigtable/TableAdmin.java | 338 ++++++++++++++++++ .../cloud/examples/bigtable/ITTableAdmin.java | 210 +++++++++++ 2 files changed, 548 insertions(+) create mode 100644 google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java create mode 100644 google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java new file mode 100644 index 000000000000..64edadca99c0 --- /dev/null +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java @@ -0,0 +1,338 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules; +import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class TableAdmin { + + private static final String COLUMN_FAMILY_1 = "cf1"; + private static final String COLUMN_FAMILY_2 = "cf2"; + private static final String COLUMN_FAMILY_3 = "cf3"; + private static final String COLUMN_FAMILY_4 = "cf4"; + private static final String COLUMN_FAMILY_5 = "cf5"; + private final String tableId; + private final BigtableTableAdminClient adminClient; + + public static void main(String[] args) throws IOException { + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + return; + } + String projectId = args[0]; + String instanceId = args[1]; + + TableAdmin tableAdmin = new TableAdmin(projectId, instanceId, "test-table"); + tableAdmin.run(); + } + + public TableAdmin(String projectId, String instanceId, String tableId) throws IOException { + this.tableId = tableId; + + // [START connecting_to_bigtable] + // Create the settings to configure a bigtable table admin client + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceName(com.google.bigtable.admin.v2.InstanceName.of(projectId, instanceId)) + .build(); + + // Create bigtable table admin client + adminClient = BigtableTableAdminClient.create(adminSettings); + // [END connecting_to_bigtable] + } + + public void run() { + createTable(); + listAllTables(); + getTableMeta(); + maxAgeRule(); + maxVersionsRule(); + unionRule(); + intersectionRule(); + nestedRule(); + listColumnFamilies(); + modifyColumnFamilyRule(); + printModifiedColumnFamily(); + deleteColumnFamily(); + deleteTable(); + adminClient.close(); + } + + public void createTable() { + // [START bigtable_create_table] + // Check if table exists, create table if does not exist + if (!adminClient.exists(tableId)) { + System.out.println("Table does not exist, creating table: " + tableId); + CreateTableRequest createTableRequest = CreateTableRequest.of(tableId).addFamily("cf"); + Table table = adminClient.createTable(createTableRequest); + System.out.printf("Table: %s created successfully%n", table.getId()); + } + // [END bigtable_create_table] + } + + public void listAllTables() { + System.out.println("\nListing tables in current instance"); + // [START bigtable_list_tables] + // List tables in current instance + try { + List listTables = adminClient.listTables(); + for (String tableName : listTables) { + System.out.println(tableName); + } + } catch (NotFoundException e) { + System.err.println("Failed to list tables from a non-existent instance: " + e.getMessage()); + } + // [END bigtable_list_tables] + } + + public void getTableMeta() { + System.out.println("\nPrinting table metadata"); + // [START bigtable_get_table_metadata] + // Get table metadata, and apply a view to the table fields + try { + Table table = adminClient.getTable(tableId); + System.out.println("Table: " + table.getId()); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + printColumnFamily(columnFamily); + } + } catch (NotFoundException e) { + System.err.println( + "Failed to retrieve table metadata for a non-existent table: " + e.getMessage()); + } + // [END bigtable_get_table_metadata] + } + + public void maxAgeRule() { + System.out.printf("%nCreating column family %s with max age GC rule%n", COLUMN_FAMILY_1); + // [START bigtable_create_family_gc_max_age] + // Create a column family with GC policy : maximum age + // where age = current time minus cell timestamp + + // Define the GC rule to retain data with max age of 5 days + GCRules.DurationRule maxAgeRule1 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + + // Create column family with given GC rule + try { + ModifyColumnFamiliesRequest columnFamiliesRequest1 = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_1, maxAgeRule1); + adminClient.modifyFamilies(columnFamiliesRequest1); + System.out.println("Created column family: " + COLUMN_FAMILY_1); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_max_age] + } + + public void maxVersionsRule() { + System.out.printf("%nCreating column family %s with max versions GC rule%n", COLUMN_FAMILY_2); + // [START bigtable_create_family_gc_max_versions] + // Create a column family with GC policy : most recent N versions + // where 1 = most recent version + + // Define the GC policy to retain only the most recent 2 versions + GCRules.VersionRule versionRule1 = GCRules.GCRULES.maxVersions(2); + + // Create column family with given GC rule + try { + ModifyColumnFamiliesRequest columnFamiliesRequest2 = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_2, versionRule1); + adminClient.modifyFamilies(columnFamiliesRequest2); + System.out.println("Created column family: " + COLUMN_FAMILY_2); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_max_versions] + } + + public void unionRule() { + System.out.printf("%nCreating column family %s with union GC rule%n", COLUMN_FAMILY_3); + // [START bigtable_create_family_gc_union] + // Create a column family with GC policy to drop data that matches at least one condition. + + // Define a GC rule to drop cells older than 5 days OR not the most recent version + GCRules.DurationRule maxAgeRule2 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + GCRules.VersionRule versionRule2 = GCRules.GCRULES.maxVersions(1); + // Add rules to union rule list + GCRules.UnionRule unionRule1 = GCRules.GCRULES.union().rule(maxAgeRule2).rule(versionRule2); + + // Create column family with given GC rule + try { + ModifyColumnFamiliesRequest columnFamiliesRequest3 = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_3, unionRule1); + adminClient.modifyFamilies(columnFamiliesRequest3); + System.out.println("Created column family: " + COLUMN_FAMILY_3); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_union] + } + + public void intersectionRule() { + System.out.printf("%nCreating column family %s with intersection GC rule%n", COLUMN_FAMILY_4); + // [START bigtable_create_family_gc_intersection] + // Create a column family with GC policy to drop data that matches all conditions + + // GC rule: Drop cells older than 5 days AND older than the most recent 2 versions + GCRules.DurationRule maxAgeRule3 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + GCRules.VersionRule versionRule3 = GCRules.GCRULES.maxVersions(2); + GCRules.IntersectionRule intersectionRule1 = + GCRules.GCRULES.intersection().rule(maxAgeRule3).rule(versionRule3); + + // Create column family with given GC rule + try { + ModifyColumnFamiliesRequest columnFamiliesRequest4 = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_4, intersectionRule1); + adminClient.modifyFamilies(columnFamiliesRequest4); + System.out.println("Created column family: " + COLUMN_FAMILY_4); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_intersection] + } + + public void nestedRule() { + System.out.printf("%nCreating column family %s with a nested GC rule%n", COLUMN_FAMILY_5); + // [START bigtable_create_family_gc_nested] + // Create a nested GC rule: + // Drop cells that are either older than the 10 recent versions + // OR + // Drop cells that are older than a month AND older than the 2 recent versions + GCRules.VersionRule versionRule4 = GCRules.GCRULES.maxVersions(10); + GCRules.DurationRule maxAgeRule4 = GCRules.GCRULES.maxAge(30, TimeUnit.DAYS); + GCRules.VersionRule versionRule5 = GCRules.GCRULES.maxVersions(2); + GCRules.IntersectionRule intersectionRule2 = + GCRules.GCRULES.intersection().rule(maxAgeRule4).rule(versionRule5); + GCRules.UnionRule unionRule2 = + GCRules.GCRULES.union().rule(intersectionRule2).rule(versionRule4); + + // Create column family with given GC rule + try { + ModifyColumnFamiliesRequest columnFamiliesRequest5 = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_5, unionRule2); + adminClient.modifyFamilies(columnFamiliesRequest5); + System.out.println("Created column family: " + COLUMN_FAMILY_5); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_nested] + } + + public void listColumnFamilies() { + System.out.println("\nPrinting ID and GC Rule for all column families"); + // [START bigtable_list_column_families] + // List all families in the table with GC rules + try { + Table table = adminClient.getTable(tableId); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + printColumnFamily(columnFamily); + } + } catch (NotFoundException e) { + System.err.println( + "Failed to list column families from a non-existent table: " + e.getMessage()); + } + // [END bigtable_list_column_families] + } + + public void modifyColumnFamilyRule() { + System.out.printf("%nUpdating column family %s GC rule%n", COLUMN_FAMILY_1); + // [START bigtable_update_gc_rule] + // Update the column family metadata to update the GC rule + // Update a column family GC rule + GCRules.VersionRule versionRule6 = GCRules.GCRULES.maxVersions(1); + try { + // Update column family with given GC rule + ModifyColumnFamiliesRequest updateRequest = + ModifyColumnFamiliesRequest.of(tableId).updateFamily(COLUMN_FAMILY_1, versionRule6); + adminClient.modifyFamilies(updateRequest); + System.out.printf("Column family %s GC rule updated%n", COLUMN_FAMILY_1); + } catch (NotFoundException e) { + System.err.println("Failed to modify a non-existent column family: " + e.getMessage()); + } + // [END bigtable_update_gc_rule] + } + + public void printModifiedColumnFamily() { + System.out.printf("%nPrint updated GC rule for column family %s%n", COLUMN_FAMILY_1); + // [START bigtable_family_get_gc_rule] + try { + Table table = adminClient.getTable(tableId); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.getId().equals(COLUMN_FAMILY_1)) { + printColumnFamily(columnFamily); + } + } + } catch (NotFoundException e) { + System.err.println("Failed to print a non-existent column family: " + e.getMessage()); + } + // [END bigtable_family_get_gc_rule] + } + + public void deleteColumnFamily() { + System.out.println("\nDelete column family: " + COLUMN_FAMILY_2); + // [START bigtable_delete_family] + // Delete a column family + try { + ModifyColumnFamiliesRequest deleted = + ModifyColumnFamiliesRequest.of(tableId).dropFamily(COLUMN_FAMILY_2); + adminClient.modifyFamilies(deleted); + System.out.printf("Column family %s deleted successfully%n", COLUMN_FAMILY_2); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent column family: " + e.getMessage()); + } + // [END bigtable_delete_family] + } + + public void deleteTable() { + // [START bigtable_delete_table] + // Delete the entire table + System.out.println("\nDelete table: " + tableId); + try { + adminClient.deleteTable(tableId); + System.out.printf("Table: %s deleted successfully%n", tableId); + + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent table: " + e.getMessage()); + } + // [END bigtable_delete_table] + } + + private static void printColumnFamily(ColumnFamily columnFamily) { + System.out.printf( + "Column family: %s%nMetadata: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } +} diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java new file mode 100644 index 000000000000..951b60e9b91d --- /dev/null +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java @@ -0,0 +1,210 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import static org.junit.Assert.assertTrue; + +import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules; +import java.io.IOException; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ITTableAdmin { + + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PREFIX = "table"; + private static String tableId; + private static BigtableTableAdminClient adminClient; + private static InstanceName instanceName; + private TableAdmin tableAdmin; + + @BeforeClass + public static void beforeClass() throws IOException { + String targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME); + if (targetInstance == null) { + adminClient = null; + return; + } + instanceName = InstanceName.parse(targetInstance); + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder().setInstanceName(instanceName).build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() { + garbageCollect(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null) { + throw new AssumptionViolatedException( + INSTANCE_PROPERTY_NAME + " property is not set, skipping integration tests."); + } + tableId = generateTableId(); + tableAdmin = new TableAdmin(instanceName.getProject(), instanceName.getInstance(), tableId); + adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf")); + } + + @After + public void after() { + if (adminClient.exists(tableId)) { + adminClient.deleteTable(tableId); + } + } + + @Test + public void testCreateAndDeleteTable() throws IOException { + // Create table + String fakeTable = generateTableId(); + TableAdmin testTableAdmin = + new TableAdmin(instanceName.getProject(), instanceName.getInstance(), fakeTable); + testTableAdmin.createTable(); + assertTrue(adminClient.exists(fakeTable)); + + // Delete table + testTableAdmin.deleteTable(); + assertTrue(!adminClient.exists(fakeTable)); + } + + @Test + public void testCreateMaxAgeRuleAndModifyAndPrintColumnFamily() { + // Max age rule + tableAdmin.maxAgeRule(); + GCRules.DurationRule maxAgeCondition = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + boolean maxAgeRule = ruleCheck(maxAgeCondition); + assertTrue(maxAgeRule); + + // Modify cf1 + tableAdmin.modifyColumnFamilyRule(); + Object modifiedRule = GCRules.GCRULES.maxVersions(1); + boolean maxVersionRule = ruleCheck(modifiedRule); + assertTrue(maxVersionRule); + } + + @Test + public void testCreateMaxVersionsRuleAndDeleteColumnFamily() { + // Max versions rule + tableAdmin.maxVersionsRule(); + GCRules.VersionRule maxVersionCondition = GCRules.GCRULES.maxVersions(2); + boolean maxVersionRule = ruleCheck(maxVersionCondition); + assertTrue(maxVersionRule); + + // Delete cf2 + tableAdmin.deleteColumnFamily(); + boolean found = true; + List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.equals("cf2")) { + found = false; + break; + } + } + assertTrue(found); + } + + @Test + public void testCreateUnionRule() { + // Union rule + tableAdmin.unionRule(); + GCRules.DurationRule maxAgeRule = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + GCRules.VersionRule versionRule = GCRules.GCRULES.maxVersions(1); + GCRules.UnionRule unionCondition = GCRules.GCRULES.union().rule(maxAgeRule).rule(versionRule); + boolean unionRule = ruleCheck(unionCondition); + assertTrue(unionRule); + } + + @Test + public void testCreateIntersectionRule() { + // Intersection rule + tableAdmin.intersectionRule(); + GCRules.DurationRule maxAgeRule = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + GCRules.VersionRule versionRule = GCRules.GCRULES.maxVersions(2); + GCRules.IntersectionRule intersectionCondition = + GCRules.GCRULES.intersection().rule(maxAgeRule).rule(versionRule); + boolean intersectionRule = ruleCheck(intersectionCondition); + assertTrue(intersectionRule); + } + + @Test + public void testCreateNestedRule() { + // Nested rule + tableAdmin.nestedRule(); + GCRules.VersionRule versionRule = GCRules.GCRULES.maxVersions(10); + GCRules.DurationRule maxAgeRule = GCRules.GCRULES.maxAge(30, TimeUnit.DAYS); + GCRules.VersionRule versionRule2 = GCRules.GCRULES.maxVersions(2); + GCRules.IntersectionRule intersectionRule = + GCRules.GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); + GCRules.UnionRule nestedCondition = + GCRules.GCRULES.union().rule(intersectionRule).rule(versionRule); + boolean nestedRule = ruleCheck(nestedCondition); + assertTrue(nestedRule); + } + + @Test + public void testRunDoesNotFail() { + tableAdmin.run(); + } + + // TODO: add test for tableAdmin.listAllTables() + // TODO: add test for tableAdmin.getTableMeta() + // TODO: add test for tableAdmin.listColumnFamilies + + private boolean ruleCheck(Object condition) { + boolean found = false; + List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.getGCRule().equals(condition)) { + found = true; + break; + } + } + return found; + } + + private String generateTableId() { + return String.format( + "%s-%016x-%x", TABLE_PREFIX, System.currentTimeMillis(), new Random().nextLong()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(TABLE_PREFIX + "-([0-9a-f]+)-([0-9a-f]+)"); + for (String tableId : adminClient.listTables()) { + Matcher matcher = timestampPattern.matcher(tableId); + if (!matcher.matches()) { + continue; + } + System.out.println("\nGarbage collecting orphaned table: " + tableId); + adminClient.deleteTable(tableId); + } + } +} From 18368ee3a48a0e005890220d08b61ae915019d92 Mon Sep 17 00:00:00 2001 From: elisheva-qlogic Date: Mon, 7 Jan 2019 15:17:32 -0500 Subject: [PATCH 2/6] comments added --- .../cloud/examples/bigtable/TableAdmin.java | 85 +++++++++++++------ .../cloud/examples/bigtable/ITTableAdmin.java | 9 +- 2 files changed, 65 insertions(+), 29 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java index 64edadca99c0..1a1d06f541ab 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java @@ -29,6 +29,28 @@ import java.util.List; import java.util.concurrent.TimeUnit; +/** + * An example of using Google Cloud Bigtable. + * + *

This example demonstrates the usage of BigtableTableAdminClient to create, configure and + * delete a Cloud Bigtable table. + * + * 

+ *   creates table
+ *   lists all tables
+ *   gets table metadata
+ *   creates DurationRule
+ *   creates VersionRule
+ *   creates UnionRule
+ *   creates IntersectionRule
+ *   creates nested rule
+ *   lists column families
+ *   modifies column family rule
+ *   prints modified column family
+ *   deletes column family
+ *   deletes table
+ *
+ */ public class TableAdmin { private static final String COLUMN_FAMILY_1 = "cf1"; @@ -56,13 +78,13 @@ public TableAdmin(String projectId, String instanceId, String tableId) throws IO this.tableId = tableId; // [START connecting_to_bigtable] - // Create the settings to configure a bigtable table admin client + // Creates the settings to configure a bigtable table admin client. BigtableTableAdminSettings adminSettings = BigtableTableAdminSettings.newBuilder() .setInstanceName(com.google.bigtable.admin.v2.InstanceName.of(projectId, instanceId)) .build(); - // Create bigtable table admin client + // Creates a bigtable table admin client. adminClient = BigtableTableAdminClient.create(adminSettings); // [END connecting_to_bigtable] } @@ -84,9 +106,10 @@ public void run() { adminClient.close(); } + /** Demonstrates how to create a table with the specified configuration. */ public void createTable() { // [START bigtable_create_table] - // Check if table exists, create table if does not exist + // Checks if table exists, creates table if does not exist. if (!adminClient.exists(tableId)) { System.out.println("Table does not exist, creating table: " + tableId); CreateTableRequest createTableRequest = CreateTableRequest.of(tableId).addFamily("cf"); @@ -96,10 +119,11 @@ public void createTable() { // [END bigtable_create_table] } + /** Demonstrates how to list all tables within an instance. */ public void listAllTables() { System.out.println("\nListing tables in current instance"); // [START bigtable_list_tables] - // List tables in current instance + // Lists tables in the current instance. try { List listTables = adminClient.listTables(); for (String tableName : listTables) { @@ -111,10 +135,11 @@ public void listAllTables() { // [END bigtable_list_tables] } + /** Demonstrates how to get a table's metadata. */ public void getTableMeta() { System.out.println("\nPrinting table metadata"); // [START bigtable_get_table_metadata] - // Get table metadata, and apply a view to the table fields + // Gets table metadata, and applies a view to the table fields. try { Table table = adminClient.getTable(tableId); System.out.println("Table: " + table.getId()); @@ -129,16 +154,17 @@ public void getTableMeta() { // [END bigtable_get_table_metadata] } + /** Demonstrates how to create a new instance of the DurationRule. */ public void maxAgeRule() { System.out.printf("%nCreating column family %s with max age GC rule%n", COLUMN_FAMILY_1); // [START bigtable_create_family_gc_max_age] - // Create a column family with GC policy : maximum age + // Creates a column family with GC policy : maximum age // where age = current time minus cell timestamp - // Define the GC rule to retain data with max age of 5 days + // Defines the GC rule to retain data with max age of 5 days. GCRules.DurationRule maxAgeRule1 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); - // Create column family with given GC rule + // Creates column family with given GC rule. try { ModifyColumnFamiliesRequest columnFamiliesRequest1 = ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_1, maxAgeRule1); @@ -151,16 +177,17 @@ public void maxAgeRule() { // [END bigtable_create_family_gc_max_age] } + /** Demonstrates how to create a new instance of the VersionRule. */ public void maxVersionsRule() { System.out.printf("%nCreating column family %s with max versions GC rule%n", COLUMN_FAMILY_2); // [START bigtable_create_family_gc_max_versions] - // Create a column family with GC policy : most recent N versions + // Creates a column family with GC policy : most recent N versions // where 1 = most recent version - // Define the GC policy to retain only the most recent 2 versions + // Defines the GC policy to retain only the most recent 2 versions. GCRules.VersionRule versionRule1 = GCRules.GCRULES.maxVersions(2); - // Create column family with given GC rule + // Creates column family with given GC rule. try { ModifyColumnFamiliesRequest columnFamiliesRequest2 = ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_2, versionRule1); @@ -173,18 +200,19 @@ public void maxVersionsRule() { // [END bigtable_create_family_gc_max_versions] } + /** Demonstrates how to create a new instance of the UnionRule. */ public void unionRule() { System.out.printf("%nCreating column family %s with union GC rule%n", COLUMN_FAMILY_3); // [START bigtable_create_family_gc_union] - // Create a column family with GC policy to drop data that matches at least one condition. + // Creates a column family with GC policy to drop data that matches at least one condition. - // Define a GC rule to drop cells older than 5 days OR not the most recent version + // Defines a GC rule to drop cells older than 5 days OR not the most recent version. GCRules.DurationRule maxAgeRule2 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); GCRules.VersionRule versionRule2 = GCRules.GCRULES.maxVersions(1); // Add rules to union rule list GCRules.UnionRule unionRule1 = GCRules.GCRULES.union().rule(maxAgeRule2).rule(versionRule2); - // Create column family with given GC rule + // Creates column family with given GC rule. try { ModifyColumnFamiliesRequest columnFamiliesRequest3 = ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_3, unionRule1); @@ -197,18 +225,19 @@ public void unionRule() { // [END bigtable_create_family_gc_union] } + /** Demonstrates how to create a new instance of the IntersectionRule. */ public void intersectionRule() { System.out.printf("%nCreating column family %s with intersection GC rule%n", COLUMN_FAMILY_4); // [START bigtable_create_family_gc_intersection] - // Create a column family with GC policy to drop data that matches all conditions + // Creates a column family with GC policy to drop data that matches all conditions. - // GC rule: Drop cells older than 5 days AND older than the most recent 2 versions + // Defines a GC rule to drop cells older than 5 days AND older than the most recent 2 versions. GCRules.DurationRule maxAgeRule3 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); GCRules.VersionRule versionRule3 = GCRules.GCRULES.maxVersions(2); GCRules.IntersectionRule intersectionRule1 = GCRules.GCRULES.intersection().rule(maxAgeRule3).rule(versionRule3); - // Create column family with given GC rule + // Creates column family with given GC rule. try { ModifyColumnFamiliesRequest columnFamiliesRequest4 = ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_4, intersectionRule1); @@ -221,10 +250,11 @@ public void intersectionRule() { // [END bigtable_create_family_gc_intersection] } + /** Demonstrates how to create a nested rule using the IntersectionRule and UnionRule. */ public void nestedRule() { System.out.printf("%nCreating column family %s with a nested GC rule%n", COLUMN_FAMILY_5); // [START bigtable_create_family_gc_nested] - // Create a nested GC rule: + // Creates a nested GC rule: // Drop cells that are either older than the 10 recent versions // OR // Drop cells that are older than a month AND older than the 2 recent versions @@ -236,7 +266,7 @@ public void nestedRule() { GCRules.UnionRule unionRule2 = GCRules.GCRULES.union().rule(intersectionRule2).rule(versionRule4); - // Create column family with given GC rule + // Creates column family with given GC rule. try { ModifyColumnFamiliesRequest columnFamiliesRequest5 = ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_5, unionRule2); @@ -249,10 +279,11 @@ public void nestedRule() { // [END bigtable_create_family_gc_nested] } + /** Demonstrates how to list a table's column families. */ public void listColumnFamilies() { System.out.println("\nPrinting ID and GC Rule for all column families"); // [START bigtable_list_column_families] - // List all families in the table with GC rules + // Lists all families in the table with GC rules. try { Table table = adminClient.getTable(tableId); Collection columnFamilies = table.getColumnFamilies(); @@ -266,14 +297,15 @@ public void listColumnFamilies() { // [END bigtable_list_column_families] } + /** Demonstrates how to modify a column family's rule. */ public void modifyColumnFamilyRule() { System.out.printf("%nUpdating column family %s GC rule%n", COLUMN_FAMILY_1); // [START bigtable_update_gc_rule] - // Update the column family metadata to update the GC rule - // Update a column family GC rule + // Updates the column family metadata to update the GC rule. + // Updates a column family GC rule. GCRules.VersionRule versionRule6 = GCRules.GCRULES.maxVersions(1); try { - // Update column family with given GC rule + // Updates column family with given GC rule. ModifyColumnFamiliesRequest updateRequest = ModifyColumnFamiliesRequest.of(tableId).updateFamily(COLUMN_FAMILY_1, versionRule6); adminClient.modifyFamilies(updateRequest); @@ -284,6 +316,7 @@ public void modifyColumnFamilyRule() { // [END bigtable_update_gc_rule] } + /** Demonstrates how to print the modified column family. */ public void printModifiedColumnFamily() { System.out.printf("%nPrint updated GC rule for column family %s%n", COLUMN_FAMILY_1); // [START bigtable_family_get_gc_rule] @@ -301,10 +334,11 @@ public void printModifiedColumnFamily() { // [END bigtable_family_get_gc_rule] } + /** Demonstrates how to delete a column family. */ public void deleteColumnFamily() { System.out.println("\nDelete column family: " + COLUMN_FAMILY_2); // [START bigtable_delete_family] - // Delete a column family + // Deletes a column family. try { ModifyColumnFamiliesRequest deleted = ModifyColumnFamiliesRequest.of(tableId).dropFamily(COLUMN_FAMILY_2); @@ -316,9 +350,10 @@ public void deleteColumnFamily() { // [END bigtable_delete_family] } + /** Demonstrates how to delete a table. */ public void deleteTable() { // [START bigtable_delete_table] - // Delete the entire table + // Deletes the entire table. System.out.println("\nDelete table: " + tableId); try { adminClient.deleteTable(tableId); diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java index 951b60e9b91d..4081d4effb7c 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java @@ -36,6 +36,7 @@ import org.junit.BeforeClass; import org.junit.Test; +/** Integration tests for {@link TableAdmin} */ public class ITTableAdmin { private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; @@ -84,14 +85,14 @@ public void after() { @Test public void testCreateAndDeleteTable() throws IOException { - // Create table + // Creates a table. String fakeTable = generateTableId(); TableAdmin testTableAdmin = new TableAdmin(instanceName.getProject(), instanceName.getInstance(), fakeTable); testTableAdmin.createTable(); assertTrue(adminClient.exists(fakeTable)); - // Delete table + // Deletes a table. testTableAdmin.deleteTable(); assertTrue(!adminClient.exists(fakeTable)); } @@ -104,7 +105,7 @@ public void testCreateMaxAgeRuleAndModifyAndPrintColumnFamily() { boolean maxAgeRule = ruleCheck(maxAgeCondition); assertTrue(maxAgeRule); - // Modify cf1 + // Modifies cf1. tableAdmin.modifyColumnFamilyRule(); Object modifiedRule = GCRules.GCRULES.maxVersions(1); boolean maxVersionRule = ruleCheck(modifiedRule); @@ -119,7 +120,7 @@ public void testCreateMaxVersionsRuleAndDeleteColumnFamily() { boolean maxVersionRule = ruleCheck(maxVersionCondition); assertTrue(maxVersionRule); - // Delete cf2 + // Deletes cf2. tableAdmin.deleteColumnFamily(); boolean found = true; List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); From b504c6635c81e1809b09087e2bb8aa4d105e49b0 Mon Sep 17 00:00:00 2001 From: elisheva-qlogic Date: Wed, 9 Jan 2019 09:26:49 -0500 Subject: [PATCH 3/6] files renamed and other edits --- ...TableAdmin.java => TableAdminExample.java} | 42 +++++++++---------- ...bleAdmin.java => ITTableAdminExample.java} | 33 ++++++++------- 2 files changed, 40 insertions(+), 35 deletions(-) rename google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/{TableAdmin.java => TableAdminExample.java} (95%) rename google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/{ITTableAdmin.java => ITTableAdminExample.java} (87%) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdminExample.java similarity index 95% rename from google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java rename to google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdminExample.java index 1a1d06f541ab..c1773e38b89a 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdmin.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdminExample.java @@ -35,23 +35,23 @@ *

This example demonstrates the usage of BigtableTableAdminClient to create, configure and * delete a Cloud Bigtable table. * - * 

- *   creates table
- *   lists all tables
- *   gets table metadata
- *   creates DurationRule
- *   creates VersionRule
- *   creates UnionRule
- *   creates IntersectionRule
- *   creates nested rule
- *   lists column families
- *   modifies column family rule
- *   prints modified column family
- *   deletes column family
- *   deletes table
- *
+ *
    + *
  • creates table + *
  • lists all tables + *
  • gets table metadata + *
  • creates DurationRule + *
  • creates VersionRule + *
  • creates UnionRule + *
  • creates IntersectionRule + *
  • creates nested rule + *
  • lists column families + *
  • modifies column family rule + *
  • prints modified column family + *
  • deletes column family + *
  • deletes table + *
*/ -public class TableAdmin { +public class TableAdminExample { private static final String COLUMN_FAMILY_1 = "cf1"; private static final String COLUMN_FAMILY_2 = "cf2"; @@ -70,18 +70,19 @@ public static void main(String[] args) throws IOException { String projectId = args[0]; String instanceId = args[1]; - TableAdmin tableAdmin = new TableAdmin(projectId, instanceId, "test-table"); + TableAdminExample tableAdmin = new TableAdminExample(projectId, instanceId, "test-table"); tableAdmin.run(); } - public TableAdmin(String projectId, String instanceId, String tableId) throws IOException { + public TableAdminExample(String projectId, String instanceId, String tableId) throws IOException { this.tableId = tableId; // [START connecting_to_bigtable] // Creates the settings to configure a bigtable table admin client. BigtableTableAdminSettings adminSettings = BigtableTableAdminSettings.newBuilder() - .setInstanceName(com.google.bigtable.admin.v2.InstanceName.of(projectId, instanceId)) + .setProjectId(projectId) + .setInstanceId(instanceId) .build(); // Creates a bigtable table admin client. @@ -358,7 +359,6 @@ public void deleteTable() { try { adminClient.deleteTable(tableId); System.out.printf("Table: %s deleted successfully%n", tableId); - } catch (NotFoundException e) { System.err.println("Failed to delete a non-existent table: " + e.getMessage()); } @@ -367,7 +367,7 @@ public void deleteTable() { private static void printColumnFamily(ColumnFamily columnFamily) { System.out.printf( - "Column family: %s%nMetadata: %s%n", + "Column family: %s%nGC Rule: %s%n", columnFamily.getId(), columnFamily.getGCRule().toString()); } } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java similarity index 87% rename from google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java rename to google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java index 4081d4effb7c..4c3a1c2831df 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdmin.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java @@ -15,6 +15,7 @@ */ package com.google.cloud.examples.bigtable; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import com.google.bigtable.admin.v2.InstanceName; @@ -36,15 +37,16 @@ import org.junit.BeforeClass; import org.junit.Test; -/** Integration tests for {@link TableAdmin} */ -public class ITTableAdmin { +/** Integration tests for {@link TableAdminExample} */ +public class ITTableAdminExample { private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; private static final String TABLE_PREFIX = "table"; - private static String tableId; private static BigtableTableAdminClient adminClient; - private static InstanceName instanceName; - private TableAdmin tableAdmin; + private static String instanceName; + private static String projectName; + private String tableId; + private TableAdminExample tableAdmin; @BeforeClass public static void beforeClass() throws IOException { @@ -53,9 +55,13 @@ public static void beforeClass() throws IOException { adminClient = null; return; } - instanceName = InstanceName.parse(targetInstance); + instanceName = InstanceName.parse(targetInstance).getInstance(); + projectName = InstanceName.parse(targetInstance).getProject(); BigtableTableAdminSettings adminSettings = - BigtableTableAdminSettings.newBuilder().setInstanceName(instanceName).build(); + BigtableTableAdminSettings.newBuilder() + .setInstanceId(instanceName) + .setProjectId(projectName) + .build(); adminClient = BigtableTableAdminClient.create(adminSettings); } @@ -72,7 +78,7 @@ public void setup() throws IOException { INSTANCE_PROPERTY_NAME + " property is not set, skipping integration tests."); } tableId = generateTableId(); - tableAdmin = new TableAdmin(instanceName.getProject(), instanceName.getInstance(), tableId); + tableAdmin = new TableAdminExample(projectName, instanceName, tableId); adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf")); } @@ -86,15 +92,14 @@ public void after() { @Test public void testCreateAndDeleteTable() throws IOException { // Creates a table. - String fakeTable = generateTableId(); - TableAdmin testTableAdmin = - new TableAdmin(instanceName.getProject(), instanceName.getInstance(), fakeTable); + String testTable = generateTableId(); + TableAdminExample testTableAdmin = new TableAdminExample(projectName, instanceName, testTable); testTableAdmin.createTable(); - assertTrue(adminClient.exists(fakeTable)); + assertTrue(adminClient.exists(testTable)); // Deletes a table. testTableAdmin.deleteTable(); - assertTrue(!adminClient.exists(fakeTable)); + assertFalse(adminClient.exists(testTable)); } @Test @@ -178,7 +183,7 @@ public void testRunDoesNotFail() { // TODO: add test for tableAdmin.listAllTables() // TODO: add test for tableAdmin.getTableMeta() - // TODO: add test for tableAdmin.listColumnFamilies + // TODO: add test for tableAdmin.listColumnFamilies() private boolean ruleCheck(Object condition) { boolean found = false; From f72f1e257ab55b9e2bb75dc171bcf2ecb18bf297 Mon Sep 17 00:00:00 2001 From: elisheva-qlogic Date: Thu, 10 Jan 2019 09:43:41 -0500 Subject: [PATCH 4/6] some changes in TableAdminExample and tests --- .../examples/bigtable/TableAdminExample.java | 129 ++++++++++-------- .../bigtable/ITTableAdminExample.java | 56 ++++---- 2 files changed, 103 insertions(+), 82 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdminExample.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdminExample.java index c1773e38b89a..23f7996ec745 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdminExample.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/TableAdminExample.java @@ -15,13 +15,18 @@ */ package com.google.cloud.examples.bigtable; +import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; + import com.google.api.gax.rpc.AlreadyExistsException; import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; -import com.google.cloud.bigtable.admin.v2.models.GCRules; +import com.google.cloud.bigtable.admin.v2.models.GCRules.DurationRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.IntersectionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.UnionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule; import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest; import com.google.cloud.bigtable.admin.v2.models.Table; import java.io.IOException; @@ -94,11 +99,11 @@ public void run() { createTable(); listAllTables(); getTableMeta(); - maxAgeRule(); - maxVersionsRule(); - unionRule(); - intersectionRule(); - nestedRule(); + addFamilyWithMaxAgeRule(); + addFamilyWithMaxVersionsRule(); + addFamilyWithUnionRule(); + addFamilyWithIntersectionRule(); + addFamilyWithNestedRule(); listColumnFamilies(); modifyColumnFamilyRule(); printModifiedColumnFamily(); @@ -126,9 +131,9 @@ public void listAllTables() { // [START bigtable_list_tables] // Lists tables in the current instance. try { - List listTables = adminClient.listTables(); - for (String tableName : listTables) { - System.out.println(tableName); + List tableIds = adminClient.listTables(); + for (String tableId : tableIds) { + System.out.println(tableId); } } catch (NotFoundException e) { System.err.println("Failed to list tables from a non-existent instance: " + e.getMessage()); @@ -146,7 +151,9 @@ public void getTableMeta() { System.out.println("Table: " + table.getId()); Collection columnFamilies = table.getColumnFamilies(); for (ColumnFamily columnFamily : columnFamilies) { - printColumnFamily(columnFamily); + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); } } catch (NotFoundException e) { System.err.println( @@ -156,20 +163,22 @@ public void getTableMeta() { } /** Demonstrates how to create a new instance of the DurationRule. */ - public void maxAgeRule() { + public void addFamilyWithMaxAgeRule() { System.out.printf("%nCreating column family %s with max age GC rule%n", COLUMN_FAMILY_1); // [START bigtable_create_family_gc_max_age] // Creates a column family with GC policy : maximum age // where age = current time minus cell timestamp // Defines the GC rule to retain data with max age of 5 days. - GCRules.DurationRule maxAgeRule1 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); // Creates column family with given GC rule. try { - ModifyColumnFamiliesRequest columnFamiliesRequest1 = - ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_1, maxAgeRule1); - adminClient.modifyFamilies(columnFamiliesRequest1); + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_1, maxAgeRule); + adminClient.modifyFamilies(columnFamiliesRequest); System.out.println("Created column family: " + COLUMN_FAMILY_1); } catch (AlreadyExistsException e) { System.err.println( @@ -179,20 +188,22 @@ public void maxAgeRule() { } /** Demonstrates how to create a new instance of the VersionRule. */ - public void maxVersionsRule() { + public void addFamilyWithMaxVersionsRule() { System.out.printf("%nCreating column family %s with max versions GC rule%n", COLUMN_FAMILY_2); // [START bigtable_create_family_gc_max_versions] // Creates a column family with GC policy : most recent N versions // where 1 = most recent version // Defines the GC policy to retain only the most recent 2 versions. - GCRules.VersionRule versionRule1 = GCRules.GCRULES.maxVersions(2); + VersionRule versionRule = GCRULES.maxVersions(2); // Creates column family with given GC rule. try { - ModifyColumnFamiliesRequest columnFamiliesRequest2 = - ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_2, versionRule1); - adminClient.modifyFamilies(columnFamiliesRequest2); + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_2, versionRule); + adminClient.modifyFamilies(columnFamiliesRequest); System.out.println("Created column family: " + COLUMN_FAMILY_2); } catch (AlreadyExistsException e) { System.err.println( @@ -202,22 +213,23 @@ public void maxVersionsRule() { } /** Demonstrates how to create a new instance of the UnionRule. */ - public void unionRule() { + public void addFamilyWithUnionRule() { System.out.printf("%nCreating column family %s with union GC rule%n", COLUMN_FAMILY_3); // [START bigtable_create_family_gc_union] // Creates a column family with GC policy to drop data that matches at least one condition. - // Defines a GC rule to drop cells older than 5 days OR not the most recent version. - GCRules.DurationRule maxAgeRule2 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); - GCRules.VersionRule versionRule2 = GCRules.GCRULES.maxVersions(1); - // Add rules to union rule list - GCRules.UnionRule unionRule1 = GCRules.GCRULES.union().rule(maxAgeRule2).rule(versionRule2); + // Defines a list of GC rules to drop cells older than 5 days OR not the most recent + // version. + UnionRule unionRule = + GCRULES.union().rule(GCRULES.maxAge(5, TimeUnit.DAYS)).rule(GCRULES.maxVersions(1)); // Creates column family with given GC rule. try { - ModifyColumnFamiliesRequest columnFamiliesRequest3 = - ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_3, unionRule1); - adminClient.modifyFamilies(columnFamiliesRequest3); + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_3, unionRule); + adminClient.modifyFamilies(columnFamiliesRequest); System.out.println("Created column family: " + COLUMN_FAMILY_3); } catch (AlreadyExistsException e) { System.err.println( @@ -227,22 +239,23 @@ public void unionRule() { } /** Demonstrates how to create a new instance of the IntersectionRule. */ - public void intersectionRule() { + public void addFamilyWithIntersectionRule() { System.out.printf("%nCreating column family %s with intersection GC rule%n", COLUMN_FAMILY_4); // [START bigtable_create_family_gc_intersection] // Creates a column family with GC policy to drop data that matches all conditions. // Defines a GC rule to drop cells older than 5 days AND older than the most recent 2 versions. - GCRules.DurationRule maxAgeRule3 = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); - GCRules.VersionRule versionRule3 = GCRules.GCRULES.maxVersions(2); - GCRules.IntersectionRule intersectionRule1 = - GCRules.GCRULES.intersection().rule(maxAgeRule3).rule(versionRule3); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(2); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule); // Creates column family with given GC rule. try { - ModifyColumnFamiliesRequest columnFamiliesRequest4 = - ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_4, intersectionRule1); - adminClient.modifyFamilies(columnFamiliesRequest4); + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_4, intersectionRule); + adminClient.modifyFamilies(columnFamiliesRequest); System.out.println("Created column family: " + COLUMN_FAMILY_4); } catch (AlreadyExistsException e) { System.err.println( @@ -252,26 +265,26 @@ public void intersectionRule() { } /** Demonstrates how to create a nested rule using the IntersectionRule and UnionRule. */ - public void nestedRule() { + public void addFamilyWithNestedRule() { System.out.printf("%nCreating column family %s with a nested GC rule%n", COLUMN_FAMILY_5); // [START bigtable_create_family_gc_nested] // Creates a nested GC rule: // Drop cells that are either older than the 10 recent versions // OR // Drop cells that are older than a month AND older than the 2 recent versions - GCRules.VersionRule versionRule4 = GCRules.GCRULES.maxVersions(10); - GCRules.DurationRule maxAgeRule4 = GCRules.GCRULES.maxAge(30, TimeUnit.DAYS); - GCRules.VersionRule versionRule5 = GCRules.GCRULES.maxVersions(2); - GCRules.IntersectionRule intersectionRule2 = - GCRules.GCRULES.intersection().rule(maxAgeRule4).rule(versionRule5); - GCRules.UnionRule unionRule2 = - GCRules.GCRULES.union().rule(intersectionRule2).rule(versionRule4); + VersionRule versionRule1 = GCRULES.maxVersions(10); + VersionRule versionRule2 = GCRULES.maxVersions(2); + DurationRule maxAgeRule = GCRULES.maxAge(30, TimeUnit.DAYS); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); + UnionRule unionRule = GCRULES.union().rule(intersectionRule).rule(versionRule1); // Creates column family with given GC rule. try { - ModifyColumnFamiliesRequest columnFamiliesRequest5 = - ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_5, unionRule2); - adminClient.modifyFamilies(columnFamiliesRequest5); + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_5, unionRule); + adminClient.modifyFamilies(columnFamiliesRequest); System.out.println("Created column family: " + COLUMN_FAMILY_5); } catch (AlreadyExistsException e) { System.err.println( @@ -289,7 +302,9 @@ public void listColumnFamilies() { Table table = adminClient.getTable(tableId); Collection columnFamilies = table.getColumnFamilies(); for (ColumnFamily columnFamily : columnFamilies) { - printColumnFamily(columnFamily); + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); } } catch (NotFoundException e) { System.err.println( @@ -304,11 +319,13 @@ public void modifyColumnFamilyRule() { // [START bigtable_update_gc_rule] // Updates the column family metadata to update the GC rule. // Updates a column family GC rule. - GCRules.VersionRule versionRule6 = GCRules.GCRULES.maxVersions(1); + VersionRule versionRule = GCRULES.maxVersions(1); try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to modify a family // Updates column family with given GC rule. ModifyColumnFamiliesRequest updateRequest = - ModifyColumnFamiliesRequest.of(tableId).updateFamily(COLUMN_FAMILY_1, versionRule6); + ModifyColumnFamiliesRequest.of(tableId).updateFamily(COLUMN_FAMILY_1, versionRule); adminClient.modifyFamilies(updateRequest); System.out.printf("Column family %s GC rule updated%n", COLUMN_FAMILY_1); } catch (NotFoundException e) { @@ -326,7 +343,9 @@ public void printModifiedColumnFamily() { Collection columnFamilies = table.getColumnFamilies(); for (ColumnFamily columnFamily : columnFamilies) { if (columnFamily.getId().equals(COLUMN_FAMILY_1)) { - printColumnFamily(columnFamily); + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); } } } catch (NotFoundException e) { @@ -364,10 +383,4 @@ public void deleteTable() { } // [END bigtable_delete_table] } - - private static void printColumnFamily(ColumnFamily columnFamily) { - System.out.printf( - "Column family: %s%nGC Rule: %s%n", - columnFamily.getId(), columnFamily.getGCRule().toString()); - } } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java index 4c3a1c2831df..d87175e90e15 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java @@ -15,6 +15,7 @@ */ package com.google.cloud.examples.bigtable; +import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -23,7 +24,11 @@ import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; -import com.google.cloud.bigtable.admin.v2.models.GCRules; +import com.google.cloud.bigtable.admin.v2.models.GCRules.DurationRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.GCRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.IntersectionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.UnionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule; import java.io.IOException; import java.util.List; import java.util.Random; @@ -105,14 +110,14 @@ public void testCreateAndDeleteTable() throws IOException { @Test public void testCreateMaxAgeRuleAndModifyAndPrintColumnFamily() { // Max age rule - tableAdmin.maxAgeRule(); - GCRules.DurationRule maxAgeCondition = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); + tableAdmin.addFamilyWithMaxAgeRule(); + DurationRule maxAgeCondition = GCRULES.maxAge(5, TimeUnit.DAYS); boolean maxAgeRule = ruleCheck(maxAgeCondition); assertTrue(maxAgeRule); // Modifies cf1. tableAdmin.modifyColumnFamilyRule(); - Object modifiedRule = GCRules.GCRULES.maxVersions(1); + GCRule modifiedRule = GCRULES.maxVersions(1); boolean maxVersionRule = ruleCheck(modifiedRule); assertTrue(maxVersionRule); } @@ -120,8 +125,8 @@ public void testCreateMaxAgeRuleAndModifyAndPrintColumnFamily() { @Test public void testCreateMaxVersionsRuleAndDeleteColumnFamily() { // Max versions rule - tableAdmin.maxVersionsRule(); - GCRules.VersionRule maxVersionCondition = GCRules.GCRULES.maxVersions(2); + tableAdmin.addFamilyWithMaxVersionsRule(); + VersionRule maxVersionCondition = GCRULES.maxVersions(2); boolean maxVersionRule = ruleCheck(maxVersionCondition); assertTrue(maxVersionRule); @@ -141,10 +146,10 @@ public void testCreateMaxVersionsRuleAndDeleteColumnFamily() { @Test public void testCreateUnionRule() { // Union rule - tableAdmin.unionRule(); - GCRules.DurationRule maxAgeRule = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); - GCRules.VersionRule versionRule = GCRules.GCRULES.maxVersions(1); - GCRules.UnionRule unionCondition = GCRules.GCRULES.union().rule(maxAgeRule).rule(versionRule); + tableAdmin.addFamilyWithUnionRule(); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(1); + UnionRule unionCondition = GCRULES.union().rule(maxAgeRule).rule(versionRule); boolean unionRule = ruleCheck(unionCondition); assertTrue(unionRule); } @@ -152,11 +157,11 @@ public void testCreateUnionRule() { @Test public void testCreateIntersectionRule() { // Intersection rule - tableAdmin.intersectionRule(); - GCRules.DurationRule maxAgeRule = GCRules.GCRULES.maxAge(5, TimeUnit.DAYS); - GCRules.VersionRule versionRule = GCRules.GCRULES.maxVersions(2); - GCRules.IntersectionRule intersectionCondition = - GCRules.GCRULES.intersection().rule(maxAgeRule).rule(versionRule); + tableAdmin.addFamilyWithIntersectionRule(); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(2); + IntersectionRule intersectionCondition = + GCRULES.intersection().rule(maxAgeRule).rule(versionRule); boolean intersectionRule = ruleCheck(intersectionCondition); assertTrue(intersectionRule); } @@ -164,14 +169,12 @@ public void testCreateIntersectionRule() { @Test public void testCreateNestedRule() { // Nested rule - tableAdmin.nestedRule(); - GCRules.VersionRule versionRule = GCRules.GCRULES.maxVersions(10); - GCRules.DurationRule maxAgeRule = GCRules.GCRULES.maxAge(30, TimeUnit.DAYS); - GCRules.VersionRule versionRule2 = GCRules.GCRULES.maxVersions(2); - GCRules.IntersectionRule intersectionRule = - GCRules.GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); - GCRules.UnionRule nestedCondition = - GCRules.GCRULES.union().rule(intersectionRule).rule(versionRule); + tableAdmin.addFamilyWithNestedRule(); + VersionRule versionRule = GCRULES.maxVersions(10); + DurationRule maxAgeRule = GCRULES.maxAge(30, TimeUnit.DAYS); + VersionRule versionRule2 = GCRULES.maxVersions(2); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); + UnionRule nestedCondition = GCRULES.union().rule(intersectionRule).rule(versionRule); boolean nestedRule = ruleCheck(nestedCondition); assertTrue(nestedRule); } @@ -185,7 +188,7 @@ public void testRunDoesNotFail() { // TODO: add test for tableAdmin.getTableMeta() // TODO: add test for tableAdmin.listColumnFamilies() - private boolean ruleCheck(Object condition) { + private boolean ruleCheck(GCRule condition) { boolean found = false; List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); for (ColumnFamily columnFamily : columnFamilies) { @@ -209,6 +212,11 @@ private static void garbageCollect() { if (!matcher.matches()) { continue; } + String timestampStr = matcher.group(1); + long timestamp = Long.parseLong(timestampStr); + if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(10)) { + continue; + } System.out.println("\nGarbage collecting orphaned table: " + tableId); adminClient.deleteTable(tableId); } From 8c4aa1d275040347101de8d9faf7b07bf1b11fc6 Mon Sep 17 00:00:00 2001 From: elisheva-qlogic Date: Fri, 11 Jan 2019 09:13:07 -0500 Subject: [PATCH 5/6] fixed timestamp to base 16 --- .../com/google/cloud/examples/bigtable/ITTableAdminExample.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java index d87175e90e15..e3a4698485a2 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java @@ -213,7 +213,7 @@ private static void garbageCollect() { continue; } String timestampStr = matcher.group(1); - long timestamp = Long.parseLong(timestampStr); + long timestamp = Long.parseLong(timestampStr, 16); if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(10)) { continue; } From f6fa52358c4451384fbb2bf8107bd5113ef79a48 Mon Sep 17 00:00:00 2001 From: elisheva-qlogic Date: Tue, 15 Jan 2019 15:30:08 -0500 Subject: [PATCH 6/6] separate properties for bigtable.project and bigtable.instance --- .../bigtable/ITTableAdminExample.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java index e3a4698485a2..1ad916822017 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITTableAdminExample.java @@ -19,7 +19,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.google.bigtable.admin.v2.InstanceName; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; @@ -45,27 +44,27 @@ /** Integration tests for {@link TableAdminExample} */ public class ITTableAdminExample { + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; private static final String TABLE_PREFIX = "table"; private static BigtableTableAdminClient adminClient; - private static String instanceName; - private static String projectName; + private static String instanceId; + private static String projectId; private String tableId; private TableAdminExample tableAdmin; @BeforeClass public static void beforeClass() throws IOException { - String targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME); - if (targetInstance == null) { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + if (projectId == null || instanceId == null) { adminClient = null; return; } - instanceName = InstanceName.parse(targetInstance).getInstance(); - projectName = InstanceName.parse(targetInstance).getProject(); BigtableTableAdminSettings adminSettings = BigtableTableAdminSettings.newBuilder() - .setInstanceId(instanceName) - .setProjectId(projectName) + .setInstanceId(instanceId) + .setProjectId(projectId) .build(); adminClient = BigtableTableAdminClient.create(adminSettings); } @@ -80,10 +79,13 @@ public static void afterClass() { public void setup() throws IOException { if (adminClient == null) { throw new AssumptionViolatedException( - INSTANCE_PROPERTY_NAME + " property is not set, skipping integration tests."); + INSTANCE_PROPERTY_NAME + + " or " + + PROJECT_PROPERTY_NAME + + " property is not set, skipping integration tests."); } tableId = generateTableId(); - tableAdmin = new TableAdminExample(projectName, instanceName, tableId); + tableAdmin = new TableAdminExample(projectId, instanceId, tableId); adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf")); } @@ -98,7 +100,7 @@ public void after() { public void testCreateAndDeleteTable() throws IOException { // Creates a table. String testTable = generateTableId(); - TableAdminExample testTableAdmin = new TableAdminExample(projectName, instanceName, testTable); + TableAdminExample testTableAdmin = new TableAdminExample(projectId, instanceId, testTable); testTableAdmin.createTable(); assertTrue(adminClient.exists(testTable));