Add 'jupiterone' storage engine

CHANGELOG.md

@@ -2,8 +2,12 @@
 
 #### 🐛 Bug Fix
 
-- Adding in optional Neo4j database name configuration [#78](https://github.com/JupiterOne/starbase/pull/78) ([@adam-in-ict](https://github.com/adam-in-ict))
-- Initial Starbase documentation website [#75](https://github.com/JupiterOne/starbase/pull/75) ([@austinkelleher](https://github.com/austinkelleher))
+- Adding in optional Neo4j database name configuration
+  [#78](https://github.com/JupiterOne/starbase/pull/78)
+  ([@adam-in-ict](https://github.com/adam-in-ict))
+- Initial Starbase documentation website
+  [#75](https://github.com/JupiterOne/starbase/pull/75)
+  ([@austinkelleher](https://github.com/austinkelleher))
 
 #### Authors: 2
 

README.md

@@ -172,12 +172,12 @@ cp config.yaml.example config.yaml
 >       ORGANIZATION_ID: '...'
 >       CONFIGURE_ORGANIZATION_PROJECTS: false
 > storage:
->   engine: neo4j
->   config:
->     username: neo4j
->     password: devpass
->     uri: bolt://localhost:7687
->     database: neo4j
+>   - engine: neo4j
+>     config:
+>       username: neo4j
+>       password: devpass
+>       uri: bolt://localhost:7687
+>       database: neo4j
 > ```
 
 ### Running Starbase

config.yaml.example

@@ -17,9 +17,10 @@ integrations:
       AUTH_TOKEN: example-token
       ORGANIZATION_SLUG: example-organization
 storage:
-  engine: neo4j
-  config: 
-    username: neo4j
-    password: devpass
-    uri: bolt://localhost:7687
-    database: neo4j
+  - 
+    engine: neo4j
+    config: 
+      username: neo4j
+      password: devpass
+      uri: bolt://localhost:7687
+      database: neo4j

src/starbase/config.test.ts

@@ -39,15 +39,17 @@ const testConfig: StarbaseConfig = {
       },
     },
   ],
-  storage: {
-    engine: 'neo4j',
-    config: {
-      username: 'neo4j',
-      password: 'devpass',
-      uri: 'bolt://localhost:7687',
-      database: 'neo4j',
+  storage: [
+    {
+      engine: 'neo4j',
+      config: {
+        username: 'neo4j',
+        password: 'devpass',
+        uri: 'bolt://localhost:7687',
+        database: 'neo4j',
+      },
     },
-  },
+  ],
 };
 
 describe('#parseConfig', () => {

src/starbase/config.ts

@@ -36,7 +36,10 @@ const configSchema: Schema = {
       type: 'array',
       items: integrationSchema,
     },
-    storage: storageSchema,
+    storage: {
+      type: 'array',
+      items: storageSchema,
+    },
   },
 };
 
@@ -54,7 +57,7 @@ function integrationConfigToEnvFormat(config: StarbaseIntegration['config']) {
  * This is the `.env` file that is written to the root of the Starbase project.
  */
 async function writeNeo4jRootConfig(storage: StarbaseStorage) {
-  await fs.writeFile(
+  await fs.appendFile(
     '.env',
     `NEO4J_URI=${storage.config.uri}
 NEO4J_USER=${storage.config.username}
@@ -63,6 +66,18 @@ NEO4J_PASSWORD=${storage.config.password}
   );
 }
 
+async function writeJ1RootConfig(storage: StarbaseStorage) {
+  await fs.appendFile(
+    '.env',
+    `JUPITERONE_API_KEY=${storage.config.apiKey}
+JUPITERONE_ACCOUNT=${storage.config.accountId}
+JUPITERONE_API_BASEURL=${
+      storage.config.apiBaseUrl || 'https://api.us.jupiterone.io'
+    }
+`,
+  );
+}
+
 async function writeIntegrationConfig<TConfig = any>(
   integration: StarbaseIntegration<TConfig>,
 ) {
@@ -128,5 +143,6 @@ export {
   parseConfigYaml,
   writeIntegrationConfig,
   writeNeo4jRootConfig,
+  writeJ1RootConfig,
   integrationConfigToEnvFormat,
 };

src/starbase/execution.ts

@@ -1,4 +1,8 @@
-import { writeIntegrationConfig, writeNeo4jRootConfig } from './config';
+import {
+  writeIntegrationConfig,
+  writeNeo4jRootConfig,
+  writeJ1RootConfig,
+} from './config';
 import { executeIntegration } from './integration';
 import { isDirectoryPresent } from '@jupiterone/integration-sdk-runtime';
 import { StarbaseConfigurationError } from './error';
@@ -7,13 +11,22 @@ import { StarbaseConfig, StarbaseIntegration } from './types';
 async function setupStarbaseStorageEngine(starbaseConfig: StarbaseConfig) {
   if (!starbaseConfig.storage) return;
 
-  if (starbaseConfig.storage.engine !== 'neo4j') {
-    throw new StarbaseConfigurationError(
-      'Invalid storage engine supplied. Neo4j is the only storage engine supported at this time.',
-    );
-  }
+  // TODO: determine if we need to validate array here
 
-  await writeNeo4jRootConfig(starbaseConfig.storage);
+  for (const storageConfig of starbaseConfig.storage || []) {
+    switch (storageConfig.engine) {
+      case 'neo4j':
+        await writeNeo4jRootConfig(storageConfig);
+        break;
+      case 'jupiterone':
+        await writeJ1RootConfig(storageConfig);
+        break;
+      default:
+        throw new StarbaseConfigurationError(
+          `Invalid storage engine supplied: '${storageConfig.engine}'.`,
+        );
+    }
+  }
 }
 
 type OnSkipIntegrationExecutionFunctionParams = {
@@ -48,11 +61,16 @@ async function executeStarbase(
     await writeIntegrationConfig(integration);
     await executeIntegration(integration, starbaseConfig);
   }
-  console.log(
-    `open ${
-      process.env.NEO4J_BROWSER_URI ?? 'http://localhost:7474/browser/'
-    } to browse your Neo4J graph.`,
-  );
+
+  const engines = (starbaseConfig.storage || []).map((s) => s.engine);
+
+  if (engines.includes('neo4j')) {
+    console.log(
+      `open ${
+        process.env.NEO4J_BROWSER_URI ?? 'http://localhost:7474/browser/'
+      } to browse your Neo4J graph.`,
+    );
+  }
 }
 
 export { executeStarbase, OnSkipIntegrationExecutionFunctionParams };

src/starbase/integration.ts

@@ -1,5 +1,10 @@
-import { StarbaseConfig, StarbaseIntegration } from './types';
+import {
+  JupiterOneStorage,
+  StarbaseConfig,
+  StarbaseIntegration,
+} from './types';
 import { executeWithLogging } from './process';
+import { StarbaseConfigurationError } from './error';
 
 async function collectIntegrationData(integrationDirectory: string) {
   await executeWithLogging(
@@ -17,19 +22,49 @@ async function writeIntegrationDataToNeo4j(
   );
 }
 
+async function writeIntegrationDataToJupiterOne(
+  integrationInstanceId: string,
+  integrationDirectory: string,
+  storageConfig: JupiterOneStorage,
+) {
+  const isDev = storageConfig.config.apiBaseUrl?.match(/dev/) || false;
+  const optDevFlags = isDev
+    ? `--api-base-url ${storageConfig.config.apiBaseUrl}`
+    : '';
+  await executeWithLogging(
+    `yarn j1-integration sync -i ${integrationInstanceId} -p ${integrationDirectory} ${optDevFlags}`,
+  );
+}
+
 async function executeIntegration<TConfig>(
   integration: StarbaseIntegration<TConfig>,
   starbaseConfig: StarbaseConfig,
 ) {
   await collectIntegrationData(integration.directory);
 
   // TODO: Remove this in favor of custom storage engine handler functions.
-  if (starbaseConfig.storage?.engine === 'neo4j') {
-    await writeIntegrationDataToNeo4j(
-      integration.instanceId,
-      integration.directory,
-      starbaseConfig.storage?.config?.database,
-    );
+
+  for (const storageConfig of starbaseConfig.storage || []) {
+    switch (storageConfig.engine) {
+      case 'neo4j':
+        await writeIntegrationDataToNeo4j(
+          integration.instanceId,
+          integration.directory,
+          storageConfig.config?.database,
+        );
+        break;
+      case 'jupiterone':
+        await writeIntegrationDataToJupiterOne(
+          integration.instanceId,
+          integration.directory,
+          storageConfig,
+        );
+        break;
+      default:
+        throw new StarbaseConfigurationError(
+          `Invalid storage engine supplied: '${storageConfig.engine}'.`,
+        );
+    }
   }
 }
 

src/starbase/types.ts

@@ -11,16 +11,33 @@ export type StarbaseStorage<TStorageConfig = any> = {
   config: TStorageConfig;
 };
 
-interface Neo4jStorageConfig {
+interface Neo4jStorageEngineConfig {
   username: string;
   password: string;
   uri: string;
   database?: string;
 }
 
-export type Neo4jStorage = StarbaseStorage<Neo4jStorageConfig>;
+interface JupiterOneStorageEngineConfig {
+  /**
+   * The JupiterOne API key to authenticate with
+   */
+  apiKey: string;
+  /**
+   * The JupiterOne account ID to target
+   */
+  accountId: string;
+  /**
+   * If provided this option specifies which base URL to use for synchronization (default: `https://api.us.jupiterone.io`)
+   */
+  apiBaseUrl?: string;
+}
+
+export type Neo4jStorage = StarbaseStorage<Neo4jStorageEngineConfig>;
+
+export type JupiterOneStorage = StarbaseStorage<JupiterOneStorageEngineConfig>;
 
 export type StarbaseConfig = {
   integrations: StarbaseIntegration[];
-  storage?: StarbaseStorage;
+  storage?: StarbaseStorage[];
 };