implementing Google's Cloud BigTable as an storage engine

CMakeLists.txt

@@ -108,6 +108,8 @@ else ()
         elseif ("${BACKEND}" STREQUAL "bolt")
             set (DVID_BACKEND_DEPEND    "gobolt" ${DVID_BACKEND_DEPEND})
             message ("Installing pure Go LMDB-inspired Bolt key-value store.")
+        elseif ("${BACKEND}" STREQUAL "bigtable")
+            message ("Installing Google's Cloud BigTable key-value store.")
         elseif ("${BACKEND}" STREQUAL "couchbase" ${DVID_BACKEND_DEPEND})
             message (FATAL_ERROR "Couchbase is currently not supported as a DVID storage engine.")
         endif ()
@@ -149,7 +151,12 @@ else ()
         ${BUILDEM_ENV_STRING} go get ${GO_GET} gopkg.in/natefinch/lumberjack.v2
         COMMENT     "Adding lumberjack library...")
 
-    set (DVID_DEP_GO_PACKAGES gopackages gojsonschema goji msgp context lumberjack)
+    add_custom_target (gcloud
+        ${BUILDEM_ENV_STRING} go get ${GO_GET} golang.org/x/net/context
+        COMMAND ${BUILDEM_ENV_STRING} go get ${GO_GET} google.golang.org/cloud/bigtable
+        COMMENT     "Adding gcloud packages...")  
+
+    set (DVID_DEP_GO_PACKAGES gopackages gojsonschema goji msgp context lumberjack gcloud)
 
     add_custom_target (nrsc
         ${BUILDEM_ENV_STRING} ${GO_ENV} go build -o ${BUILDEM_BIN_DIR}/nrsc
@@ -199,8 +206,7 @@ else ()
         ${BUILDEM_ENV_STRING} nrsc ${BUILDEM_BIN_DIR}/dvid dvid-console
         WORKING_DIRECTORY   ${BUILDEM_DIR}/src
         DEPENDS     dvid-exe nrsc dvid-console dvid-backup
-        COMMENT     "Adding embedded console into dvid executable..."
-    )
+        COMMENT     "Adding embedded console into dvid executable...")
 
     set (DVID_PACKAGES ${DVID_GO}/dvid ${DVID_GO}/storage/... ${DVID_GO}/datastore ${DVID_GO}/server
                        ${DVID_GO}/datatype/... ${DVID_GO}/tests_integration)

datastore/bigtable.go

@@ -0,0 +1,5 @@
+// +build bigtable
+
+package datastore
+
+import _ "github.com/janelia-flyem/dvid/storage/bigtable"

datastore/test_support.go

@@ -7,6 +7,8 @@ import (
 	"log"
 	"sync"
 
+	"google.golang.org/cloud/bigtable/bttest"
+
 	"github.com/janelia-flyem/dvid/dvid"
 	"github.com/janelia-flyem/dvid/storage"
 
@@ -48,9 +50,30 @@ func GetTestStoreConfig() (*dvid.StoreConfig, error) {
 	if testableEng == nil {
 		return nil, fmt.Errorf("Could not find a storage engine that was testable")
 	}
-	dbname := fmt.Sprintf("dvid-test-%x", uuid.NewV4().Bytes())
-	engConfig := dvid.EngineConfig{Engine: testableEng.GetName(), Path: dbname, Testing: true}
-	return &dvid.StoreConfig{Mutable: engConfig}, nil
+
+	if testableEng.GetName() == "bigtable" {
+
+		testSrv, err := bttest.NewServer() //TODO close the testSrv if neccesary
+		if err != nil {
+			return nil, fmt.Errorf("Unable to create bigTable local test server. %v", err)
+		}
+
+		table := fmt.Sprintf("dvid-test-%x", uuid.NewV4().Bytes())
+		engConfig := dvid.EngineConfig{Engine: testableEng.GetName(),
+			Project: "project",
+			Zone:    "zone",
+			Cluster: "cluster",
+			Table:   table,
+			Testing: true,
+			TestSrv: testSrv,
+		}
+
+		return &dvid.StoreConfig{Mutable: engConfig}, nil
+	} else {
+		dbname := fmt.Sprintf("dvid-test-%x", uuid.NewV4().Bytes())
+		engConfig := dvid.EngineConfig{Engine: testableEng.GetName(), Path: dbname, Testing: true}
+		return &dvid.StoreConfig{Mutable: engConfig}, nil
+	}
 }
 
 func openStore(create bool) {
@@ -74,7 +97,6 @@ func openStore(create bool) {
 
 func OpenTest() {
 	testStore.Lock()
-
 	dvid.Infof("Opening test datastore...\n")
 	openStore(true)
 }

dvid/storage.go

@@ -1,5 +1,7 @@
 package dvid
 
+import "google.golang.org/cloud/bigtable/bttest"
+
 type StoreConfig struct {
 	MetaData  EngineConfig
 	Mutable   EngineConfig
@@ -17,4 +19,23 @@ type EngineConfig struct {
 
 	// Testing is true if this store is to be used for testing.
 	Testing bool
+
+	// --- Used for big table
+	//Id of the google cloud's Project where dvid and bigTable's cluster are running
+	Project string
+
+	//Zone where the cluster is allocated
+	Zone string
+
+	//Name of the cluster running bigTable servers
+	Cluster string
+
+	//Name of the table to be used by the engine
+	Table string
+
+	//Test server used for testing big table
+	TestSrv *bttest.Server
+
+	// --- Used for big table
+
 }