diff --git a/packages/mdctl-cli/lib/package/index.js b/packages/mdctl-cli/lib/package/index.js new file mode 100644 index 00000000..41fe8025 --- /dev/null +++ b/packages/mdctl-cli/lib/package/index.js @@ -0,0 +1,7 @@ +const publishPkg = require('./publish'), + installPkg = require('./install') + +module.exports = { + publishPkg, + installPkg +} diff --git a/packages/mdctl-cli/lib/package/install.js b/packages/mdctl-cli/lib/package/install.js new file mode 100644 index 00000000..6753754f --- /dev/null +++ b/packages/mdctl-cli/lib/package/install.js @@ -0,0 +1,37 @@ +const Package = require('../../../mdctl-packages'), + { Cortex } = require('../package/source'), + installPkg = async(name, params) => { + let tmpName = name + + if (name && name.startsWith('--')) { + // No package name specified after `mdctl package install`, but an option + // So assign name to an empty string to install a local package at the current + // working directory where the mdctl command is executed. + tmpName = '' + } + + // If pkgName is empty, then this is a local package. + // Otherwise, it is a remote package. + const { + registryUrl, registryProjectId, registryToken, client + } = params, + options = { registryUrl, registryProjectId, registryToken }, + [pkgName, pkgVersion] = tmpName.split('@'), + isLocalPkg = pkgName === '', + pkg = new Package(pkgName, isLocalPkg ? '.' : pkgVersion || 'latest', null, options) + + await pkg.evaluate() + + // eslint-disable-next-line one-var + const srcClient = new Cortex(pkg.name, pkg.version, { + client + }) + + try { + await srcClient.installPackage(pkg) + } catch (err) { + throw err + } + } + +module.exports = installPkg diff --git a/packages/mdctl-cli/lib/package/publish.js b/packages/mdctl-cli/lib/package/publish.js new file mode 100644 index 00000000..8241ce5d --- /dev/null +++ b/packages/mdctl-cli/lib/package/publish.js @@ -0,0 +1,32 @@ +const Package = require('../../../mdctl-packages'), + { Registry, Cortex } = require('../package/source'), + publishPkg = async(name, params) => { + const { + source, registryUrl, registryProjectId, registryToken, client + } = params, + pkg = new Package(name, '.') + + await pkg.evaluate() + + let srcClient + + if (source === 'registry') { + srcClient = new Registry(pkg.name, pkg.version, { + registryUrl, + registryProjectId, + registryToken + }) + } else { + srcClient = new Cortex(pkg.name, pkg.version, { + client + }) + } + + try { + await srcClient.publishPackage(await pkg.getPackageStream()) + } catch (err) { + throw err + } + } + +module.exports = publishPkg diff --git a/packages/mdctl-cli/lib/package/source/cortex.js b/packages/mdctl-cli/lib/package/source/cortex.js new file mode 100644 index 00000000..95ef0972 --- /dev/null +++ b/packages/mdctl-cli/lib/package/source/cortex.js @@ -0,0 +1,96 @@ +const { URL } = require('url'), + FormData = require('form-data') + + +class Cortex { + + constructor(name, version, options) { + this.name = name + this.version = version + this.client = options.client + this.publishPath = process.env.PACKAGE_PUBLISH_PATH || '/developer/packages/publish' + this.installPath = process.env.PACKAGE_INSTALL_PATH || '/developer/packages/install' + } + + async installPackage(pkg) { + const url = new URL(this.installPath, this.client.environment.url), + dependencies = pkg.dependenciesPackages || [], + install = body => this.client.call(url.pathname, { method: 'POST', body }) + + dependencies.forEach(async(dependency) => { + try { + await install(await dependency.getPackageStream()) + } catch (err) { + throw new Error('Failed to install one of the package dependencies. Please try it again!!!') + } + }) + + await install(await pkg.getPackageStream()) + } + + async publishPackage(zipStream) { + // Publishing a package to cortex has 2 phases + // 1. Create a facet + // 2. Upload the package + const url = new URL(this.publishPath, this.client.environment.url), + filename = `${this.name}_${this.version}.zip`, + facet = await this.client.call(url.pathname, { + method: 'PUT', + body: { + content: filename + } + }), + upload = facet.uploads[0], + { + uploadUrl, uploadKey, fields + } = upload, + form = new FormData(), + zipToBuffer = () => new Promise((resolve, reject) => { + const data = [] + + zipStream.on('data', (chunk) => { + data.push(chunk) + }) + + zipStream.on('end', () => { + resolve(Buffer.concat(data)) + }) + + zipStream.on('error', (error) => { + reject(error) + }) + }), + data = await zipToBuffer() + + fields.forEach((field) => { + const { key, value } = field + form.append(key, value) + }) + + form.append( + uploadKey, + data, + { + filename + } + ) + + await new Promise((resolve, reject) => { + form.submit(uploadUrl, (err, response) => { + if (err) { + console.error(err) + reject(err) + } else if ([200, 201].includes(response.statusCode)) { + console.log(`Successfully published package ${this.name}@${this.version} to cortex`) + resolve() + } else { + console.error(`Publishing package failed with status code ${response.statusCode} and status message ${response.statusMessage}`) + reject() + } + }) + }) + } + +} + +module.exports = Cortex diff --git a/packages/mdctl-cli/lib/package/source/index.js b/packages/mdctl-cli/lib/package/source/index.js new file mode 100644 index 00000000..da3eb115 --- /dev/null +++ b/packages/mdctl-cli/lib/package/source/index.js @@ -0,0 +1,7 @@ +const Registry = require('./registry'), + Cortex = require('./cortex') + +module.exports = { + Registry, + Cortex +} diff --git a/packages/mdctl-cli/lib/package/source/registry.js b/packages/mdctl-cli/lib/package/source/registry.js new file mode 100644 index 00000000..adb6bdca --- /dev/null +++ b/packages/mdctl-cli/lib/package/source/registry.js @@ -0,0 +1,19 @@ +const { RegistrySource } = require('../../../../mdctl-packages/lib') + +class Registry { + + constructor(name, version, options) { + + this.source = new RegistrySource(name, version, options) + + } + + async publishPackage(zipStream) { + + await this.source.publishPackage(zipStream) + + } + +} + +module.exports = Registry diff --git a/packages/mdctl-cli/package.json b/packages/mdctl-cli/package.json index 1a09e23e..047daf0f 100644 --- a/packages/mdctl-cli/package.json +++ b/packages/mdctl-cli/package.json @@ -25,7 +25,8 @@ "test:watch": "npm run test:only -- --watch", "test:examples": "node examples/", "cover": "istanbul cover ../../node_modules/mocha/bin/_mocha -- --recursive --timeout 10000", - "lint": "eslint . --ext .js" + "lint": "eslint . --ext .js", + "test:package": "mocha --timeout 1200000 ./test/lib/package" }, "dependencies": { "@medable/mdctl-api": "^1.0.62", @@ -43,6 +44,7 @@ "async": "^2.6.3", "cli-table": "^0.3.1", "clone": "^2.1.2", + "form-data": "^4.0.0", "globby": "^9.1.0", "inflection": "^1.12.0", "inquirer": "^6.5.2", diff --git a/packages/mdctl-cli/tasks/package.js b/packages/mdctl-cli/tasks/package.js index cb2635c8..3569a456 100644 --- a/packages/mdctl-cli/tasks/package.js +++ b/packages/mdctl-cli/tasks/package.js @@ -1,17 +1,9 @@ /* eslint-disable class-methods-use-this */ -const fs = require('fs'), - _ = require('lodash'), - pump = require('pump'), - ndjson = require('ndjson'), - { isSet, parseString, rString } = require('@medable/mdctl-core-utils/values'), - Packages = require('packages/mdctl-packages'), - ImportStream = require('@medable/mdctl-core/streams/import_stream'), - ImportFileTreeAdapter = require('@medable/mdctl-import-adapter'), - { - createConfig, loadDefaults - } = require('../lib/config'), - Task = require('../lib/task') +const _ = require('lodash'), + { isSet } = require('@medable/mdctl-core-utils/values'), + Task = require('../lib/task'), + { publishPkg, installPkg } = require('../lib/package') class Package extends Task { @@ -51,33 +43,45 @@ class Package extends Task { throw new Error('Invalid command') } - const config = createConfig() - config.update(await loadDefaults()) return this[handler](cli) + + } + + async 'package@get'(cli) { + throw Error('Not Implemented') } async 'package@list'(cli) { - const result = await this.registry.getPackages() - console.log(result) + // const result = await this.registry.getPackages() + // console.log(result) + throw Error('Not Implemented') } async 'package@publish'(cli) { - // this will build package artifact - const params = await cli.getArguments(this.optionKeys), - inputDir = params.dir || process.cwd(), - packageJson = parseString(fs.readFileSync(`${inputDir}/package.json`)), - pkg = this.args('2') || `${packageJson.name}@${packageJson.version}`, - fileAdapter = new ImportFileTreeAdapter(`${inputDir}/${packageJson.mdEnvPath || 'configuration'}`, 'json'), - importStream = new ImportStream(fileAdapter), - ndjsonStream = ndjson.stringify(), - streamList = [importStream, ndjsonStream], - [name, version] = pkg.split('@') - await this.registry.publishPackage(name, version, pump(streamList), packageJson.mdDependencies) - console.log(`${name}@${version} has been published!`) + // Determine where to publish the package i.e either cortex or registry (default) + const name = this.args('name') || '', + source = this.args('source') || 'registry', + registryUrl = this.args('registryUrl') || process.env.REGISTRY_URL, + registryProjectId = this.args('registryProjectId') || process.env.REGISTRY_PROJECT_ID, + registryToken = this.args('registryToken') || process.env.REGISTRY_TOKEN, + client = source === 'cortex' ? await cli.getApiClient({ credentials: await cli.getAuthOptions() }) : null + + await publishPkg(name, { + source, registryUrl, registryProjectId, registryToken, client + }) } async 'package@install'(cli) { // this will install a package in target organization + const name = this.args('2') || '', + registryUrl = this.args('registryUrl') || process.env.REGISTRY_URL, + registryProjectId = this.args('registryProjectId') || process.env.REGISTRY_PROJECT_ID, + registryToken = this.args('registryToken') || process.env.REGISTRY_TOKEN, + client = await cli.getApiClient({ credentials: await cli.getAuthOptions() }) + + await installPkg(name, { + registryUrl, registryProjectId, registryToken, client + }) } // ---------------------------------------------------------------------------------------------- diff --git a/packages/mdctl-cli/test/lib/package/test.cortex.js b/packages/mdctl-cli/test/lib/package/test.cortex.js new file mode 100644 index 00000000..6841f5f9 --- /dev/null +++ b/packages/mdctl-cli/test/lib/package/test.cortex.js @@ -0,0 +1,105 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +const sinon = require('sinon'), + path = require('path'), + fs = require('fs'), + FormData = require('form-data'), + { Client } = require('@medable/mdctl-api'), + ZipTree = require('../../../../mdctl-packages/lib/zip_tree'), + { Cortex } = require('../../../lib/package/source') + +describe('Cortex Test', () => { + + let cortex, + sandbox, + client + + beforeEach(() => { + sandbox = sinon.createSandbox() + client = new Client({ + strictSSL: false, + environment: { + endpoint: 'https://localhost', + env: 'test' + }, + credentials: { + type: 'password', + apiKey: 'abcdefghijklmnopqrstuv', + username: 'test@medable.com', + password: 'password' + } + }) + cortex = new Cortex('TestPackage', 'latest', { client }) + }) + + afterEach(() => { + cortex = null + client = null + sandbox.restore() + }) + + it('Test publish package to cortex', async() => { + let isZipStreamDrained = false, + isFormSubmitCalled = false + + const packageZipTree = new ZipTree(path.resolve(__dirname, 'test_pkg'), { fs }), + packageZipStream = await packageZipTree.compress(), + clientFacetStub = sandbox.stub(client, 'call').resolves({ + uploads: [ + { + uploadUrl: 'test_upload_url', + uploadKey: 'test_upload_key', + fields: [{ + key: 'x-amz-credential', + value: 'x-amz-credential-test' + }, { + key: 'x-amz-date', + value: '20220118T041333Z' + }, { + key: 'x-amz-server-side-encryption', + value: 'AES256' + }, { + key: 'x-amz-signature', + value: 'x-amz-signature-test' + }, { + key: 'x-amz-algorithm', + value: 'AWS4-HMAC-SHA256' + }, { + key: 'success_action_status', + value: '201' + }, { + key: 'content-type', + value: 'application/zip' + }, { + key: 'key', + value: 'test_key' + }, { + key: 'policy', + value: 'policy_test' + } + ] + } + ] + }) + + packageZipStream.on = (message, handler) => { + if (message === 'data') { + handler(Buffer.from('test_data_begin')) + } else if (message === 'end') { + isZipStreamDrained = true + handler(Buffer.from('test_data_end')) + } + } + + FormData.prototype.submit = (uploadUrl, callback) => { + isFormSubmitCalled = true + callback(null, { statusCode: 200 }) + } + + await cortex.publishPackage(packageZipStream) + + sinon.assert.calledOnce(clientFacetStub) + sinon.assert.match(isZipStreamDrained, true) + sinon.assert.match(isFormSubmitCalled, true) + }) + +}) diff --git a/packages/mdctl-cli/test/lib/package/test.install.js b/packages/mdctl-cli/test/lib/package/test.install.js new file mode 100644 index 00000000..ec856b1f --- /dev/null +++ b/packages/mdctl-cli/test/lib/package/test.install.js @@ -0,0 +1,71 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +const sinon = require('sinon'), + { Client } = require('@medable/mdctl-api'), + Package = require('../../../../mdctl-packages'), + { Cortex } = require('../../../lib/package/source'), + { installPkg } = require('../../../lib/package/index') + +describe('Install Package Test', () => { + + let sandbox + + beforeEach(() => { + sandbox = sinon.createSandbox() + }) + + afterEach(() => { + sandbox.restore() + }) + + it('Test install local package into cortex', async() => { + const packageEvaluateStub = sandbox.stub(Package.prototype, 'evaluate').resolves(), + cortexPublishStub = sandbox.stub(Cortex.prototype, 'installPackage').resolves(), + client = new Client({ + strictSSL: false, + environment: { + endpoint: 'https://localhost', + env: 'test' + }, + credentials: { + type: 'password', + apiKey: 'abcdefghijklmnopqrstuv', + username: 'test@medable.com', + password: 'password' + } + }) + + await installPkg('', { client }) + + sinon.assert.calledOnce(packageEvaluateStub) + sinon.assert.calledOnce(cortexPublishStub) + }) + + it('Test install registry package into cortex', async() => { + const packageEvaluateStub = sandbox.stub(Package.prototype, 'evaluate').resolves(), + cortexPublishStub = sandbox.stub(Cortex.prototype, 'installPackage').resolves(), + client = new Client({ + strictSSL: false, + environment: { + endpoint: 'https://localhost', + env: 'test' + }, + credentials: { + type: 'password', + apiKey: 'abcdefghijklmnopqrstuv', + username: 'test@medable.com', + password: 'password' + } + }) + + await installPkg('TestPackage', { + registryUrl: 'test_registry_url', + registryProjectId: 'test_registry_project_id', + registryToken: 'test_registry_token', + client, + }) + + sinon.assert.calledOnce(packageEvaluateStub) + sinon.assert.calledOnce(cortexPublishStub) + }) + +}) diff --git a/packages/mdctl-cli/test/lib/package/test.publish.js b/packages/mdctl-cli/test/lib/package/test.publish.js new file mode 100644 index 00000000..b41e5d66 --- /dev/null +++ b/packages/mdctl-cli/test/lib/package/test.publish.js @@ -0,0 +1,69 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +const sinon = require('sinon'), + path = require('path'), + fs = require('fs'), + { Client } = require('@medable/mdctl-api'), + Package = require('../../../../mdctl-packages'), + { Registry, Cortex } = require('../../../lib/package/source'), + ZipTree = require('../../../../mdctl-packages/lib/zip_tree'), + { publishPkg } = require('../../../lib/package/index') + +describe('Publish Package Test', () => { + + let sandbox + + beforeEach(() => { + sandbox = sinon.createSandbox() + }) + + afterEach(() => { + sandbox.restore() + }) + + it('Test publish package to registry', async() => { + const packageZipTree = new ZipTree(path.resolve(__dirname, 'test_pkg'), { fs }), + packageZipStream = await packageZipTree.compress(), + packageEvaluateStub = sandbox.stub(Package.prototype, 'evaluate').resolves(), + packageGetStreamStub = sandbox.stub(Package.prototype, 'getPackageStream').resolves(packageZipStream), + registryPublishStub = sandbox.stub(Registry.prototype, 'publishPackage').resolves() + + await publishPkg('TestPackage', { + source: 'registry', + registryUrl: 'test_registry_url', + registryProjectId: 'test_registry_project_id', + registryToken: 'test_registry_token' + }) + + sinon.assert.calledOnce(packageEvaluateStub) + sinon.assert.calledOnce(packageGetStreamStub) + sinon.assert.calledOnce(registryPublishStub) + }) + + it('Test publish package to cortex', async() => { + const packageZipTree = new ZipTree(path.resolve(__dirname, 'test_pkg'), { fs }), + packageZipStream = await packageZipTree.compress(), + packageEvaluateStub = sandbox.stub(Package.prototype, 'evaluate').resolves(), + packageGetStreamStub = sandbox.stub(Package.prototype, 'getPackageStream').resolves(packageZipStream), + cortexPublishStub = sandbox.stub(Cortex.prototype, 'publishPackage').resolves(), + client = new Client({ + strictSSL: false, + environment: { + endpoint: 'https://localhost', + env: 'test' + }, + credentials: { + type: 'password', + apiKey: 'abcdefghijklmnopqrstuv', + username: 'test@medable.com', + password: 'password' + } + }) + + await publishPkg('TestPackage', { source: 'cortex', client }) + + sinon.assert.calledOnce(packageEvaluateStub) + sinon.assert.calledOnce(packageGetStreamStub) + sinon.assert.calledOnce(cortexPublishStub) + }) + +}) diff --git a/packages/mdctl-cli/test/lib/package/test.registry.js b/packages/mdctl-cli/test/lib/package/test.registry.js new file mode 100644 index 00000000..c4caca6d --- /dev/null +++ b/packages/mdctl-cli/test/lib/package/test.registry.js @@ -0,0 +1,38 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +const sinon = require('sinon'), + path = require('path'), + fs = require('fs'), + ZipTree = require('../../../../mdctl-packages/lib/zip_tree'), + { RegistrySource } = require('../../../../mdctl-packages/lib'), + { Registry } = require('../../../lib/package/source') + +describe('Registry Test', () => { + + let registry, + sandbox + + beforeEach(() => { + sandbox = sinon.createSandbox() + registry = new Registry('TestPackage', 'latest', { + registryUrl: 'http://registry.com', + registryProjectId: '100', + registryToken: 'test_token' + }) + }) + + afterEach(() => { + registry = null + sandbox.restore() + }) + + it('Test publish package to registry', async() => { + const packageZipTree = new ZipTree(path.resolve(__dirname, 'test_pkg'), { fs }), + packageZipStream = await packageZipTree.compress(), + publishPackageStub = sandbox.stub(RegistrySource.prototype, 'publishPackage').resolves({}) + + await registry.publishPackage(packageZipStream) + + sinon.assert.calledOnce(publishPackageStub) + }) + +}) diff --git a/packages/mdctl-cli/test/lib/package/test_pkg/.mpmrc b/packages/mdctl-cli/test/lib/package/test_pkg/.mpmrc new file mode 100644 index 00000000..ca786b8b --- /dev/null +++ b/packages/mdctl-cli/test/lib/package/test_pkg/.mpmrc @@ -0,0 +1,5 @@ +{ + "package": { + "root": "." + } +} diff --git a/packages/mdctl-cli/test/lib/package/test_pkg/env/objects/tp__test.json b/packages/mdctl-cli/test/lib/package/test_pkg/env/objects/tp__test.json new file mode 100644 index 00000000..b4909186 --- /dev/null +++ b/packages/mdctl-cli/test/lib/package/test_pkg/env/objects/tp__test.json @@ -0,0 +1,13 @@ +{ + "name": "tp__test", + "label": "Test Object", + "object": "object", + "properties": [ + { + "name": "c_test_property", + "label": "Test Property", + "type": "String", + "indexed": true + } + ] +} diff --git a/packages/mdctl-cli/test/lib/package/test_pkg/package.json b/packages/mdctl-cli/test/lib/package/test_pkg/package.json new file mode 100644 index 00000000..b2a639ae --- /dev/null +++ b/packages/mdctl-cli/test/lib/package/test_pkg/package.json @@ -0,0 +1,11 @@ +{ + "name": "TestPackage", + "version": "1.0.5", + "engines": { + "cortex": "> 2.15.8" + }, + "dependencies": { + "axon": "git+https://gitlab.medable.com/axon/org.git#test_pkg", + "data-transfers": "git+https://gitlab.medable.com/platform/environments/data-transfers.git#test_pkg" + } +} diff --git a/packages/mdctl-packages/lib/index.js b/packages/mdctl-packages/lib/index.js index 75d31e3a..29bbffd7 100644 --- a/packages/mdctl-packages/lib/index.js +++ b/packages/mdctl-packages/lib/index.js @@ -12,16 +12,13 @@ const Fault = require('@medable/mdctl-core/fault'), }, resolveSource = (name, path, options) => { let sourceType = 'registry' - if(options.ndjsonStream) { + if (options.ndjsonStream) { sourceType = 'ndjson' } else if (path.indexOf('file://') > -1 || path === '.') { sourceType = 'file' } else if (path.indexOf('git+https://') > -1) { sourceType = 'git' } - if (sourceType === 'registry') { - throw new Error('Registry source is not implemented yet') - } return new sources[sourceType](name, path, options) } diff --git a/packages/mdctl-packages/lib/sources/file.js b/packages/mdctl-packages/lib/sources/file.js index 994b1680..dcce8d27 100644 --- a/packages/mdctl-packages/lib/sources/file.js +++ b/packages/mdctl-packages/lib/sources/file.js @@ -1,4 +1,4 @@ -const path = require("path"), +const path = require('path'), fs = require('fs'), { privatesAccessor } = require('@medable/mdctl-core-utils/privates'), Source = require('./source'), @@ -8,33 +8,52 @@ class FileSource extends Source { constructor(name, path, options = {}) { path = path.replace('file://', '') - super(name, path, options); + super(name, path, options) } async readConfigFiles() { const rcFile = fs.readFileSync(path.join(this.path, '.mpmrc'), 'utf8') - if(rcFile) { + if (rcFile) { const rcData = JSON.parse(rcFile.toString()), - pkgFile = fs.readFileSync(path.join(this.path, path.join( rcData.package.root, 'package.json')), 'utf8') - return JSON.parse(pkgFile) + pkgFile = fs.readFileSync(path.join(this.path, rcData.package.root, 'package.json'), 'utf8'), + pkgInfo = JSON.parse(pkgFile), + manifestEntry = pkgInfo.manifest + + if (manifestEntry) { + const manifestPath = path.join(this.path, rcData.package.root, manifestEntry) + + if (!fs.existsSync(manifestPath)) { + throw new Error('Manifest not found. Not a valid package.') + } + } else { + const manifestJsonPath = path.join(this.path, rcData.package.root, 'manifest.json'), + manifestYmlPath = path.join(this.path, rcData.package.root, 'manifest.yml'), + manifestYamlPath = path.join(this.path, rcData.package.root, 'manifest.yaml') + + if (!fs.existsSync(manifestJsonPath) && !fs.existsSync(manifestYmlPath) && !fs.existsSync(manifestYamlPath)) { + throw new Error('Manifest not found. Not a valid package.') + } + } + + privatesAccessor(this).rootDir = rcData.package.root + return pkgInfo } throw new Error('No config file found') - } async loadPackageInfo() { const info = await this.readConfigFiles(), - packageInfo = { - name: info.name, - version: info.version, - dependencies: info.dependencies || {}, - engine: info.engine || {} - } + packageInfo = { + name: info.name, + version: info.version, + dependencies: info.dependencies || {}, + engines: info.engines || {} + } Object.assign(privatesAccessor(this), packageInfo) } async getStream() { - const zip = new ZipTree(`/${this.path}`, { fs }) + const zip = new ZipTree(path.join(this.path, privatesAccessor(this).rootDir), { fs }) return zip.compress() } diff --git a/packages/mdctl-packages/lib/sources/git.js b/packages/mdctl-packages/lib/sources/git.js index cbfee6fc..dec0558e 100644 --- a/packages/mdctl-packages/lib/sources/git.js +++ b/packages/mdctl-packages/lib/sources/git.js @@ -91,7 +91,7 @@ class GitSource extends Source { dependencies: info.dependencies || {}, version: info.version, name: info.name, - engine: info.engine || {} + engines: info.engines || {} } Object.assign(privatesAccessor(this), packageInfo) } catch (ex) { diff --git a/packages/mdctl-packages/lib/sources/registry.js b/packages/mdctl-packages/lib/sources/registry.js index cd540418..2b7c510a 100644 --- a/packages/mdctl-packages/lib/sources/registry.js +++ b/packages/mdctl-packages/lib/sources/registry.js @@ -1,19 +1,237 @@ -const { SemverResolver } = require('semver-resolver'), +const Axios = require('axios'), + { privatesAccessor } = require('@medable/mdctl-core-utils/privates'), + { isSet } = require('@medable/mdctl-core-utils/values'), + unzip = require('unzip-stream'), + semverMaxSatisfying = require('semver/ranges/max-satisfying'), + semverSort = require('semver/functions/sort'), Source = require('./source') + +class RegistryClient { + + constructor(options) { + + if (!isSet(options.registryUrl)) { + throw new Error('Missing an option --registryUrl or an environment variable REGISTRY_URL.') + } + + if (!isSet(options.registryProjectId)) { + throw new Error('Missing an option --registryProjectId or an environment variable REGISTRY_PROJECT_ID.') + } + + if (!isSet(options.registryToken)) { + throw new Error('Missing an option --registryToken or an environment variable REGISTRY_TOKEN.') + } + + this.client = Axios.default.create({ + baseURL: `${options.registryUrl}/${options.registryProjectId}/packages`, + maxContentLength: Infinity, + maxBodyLength: Infinity, + headers: { + 'PRIVATE-TOKEN': options.registryToken + } + }) + + } + + async publishPackage(name, version, content) { + + try { + + await this.client.put(`/generic/${name}/${version}/${name}_${version}.zip`, content) + + console.log(`Successfully published package ${name}@${version} to registry`) + + } catch (err) { + + console.error(`Failed to publish package ${name}@${version} to registry`) + + throw err + + } + + } + + async getPackage(name, version) { + + try { + + const { data } = await this.client.get(`/generic/${name}/${version}/${name}_${version}.zip`, { + responseType: 'stream' + }) + + return data + + } catch (err) { + + console.error(`Failed to get package ${name}@${version} from registry`) + + // TODO: How do we want to handle package not found here? + + throw err + + } + + } + + async getPackageInfo(name) { + // read the package.json from the package + try { + + const { data } = await this.client.get(''), + packages = data.filter(pkg => pkg.name === name).map((pkg) => { + + const result = { + name: pkg.name, + version: pkg.version + } + + return result + + }) + + return packages + + } catch (err) { + + throw err + + } + + } + +} + class RegistrySource extends Source { + // options should contain version, registry url, registry token, and registry project id + constructor(name, version, options = {}) { + + super(name, null, options) + + this.registryClient = new RegistryClient(options) - async getPackageInfo() { - throw Error('Not Implemented') + privatesAccessor(this).version = version } - async getStream() { + get version() { + const { correctVersion, version } = privatesAccessor(this) + return correctVersion || version + } + + async getPackage() { + + return this.registryClient.getPackage(this.name, this.version) + + } + + async publishPackage(content) { + + await this.registryClient.publishPackage(this.name, this.version, content) + + } + + async resolvePackageVersion() { + const packages = await this.registryClient.getPackageInfo(this.name), + versions = packages.filter(pkg => pkg).map(pkg => pkg.version), + sortedVersions = semverSort(versions), + correctVersion = this.version === 'latest' ? sortedVersions[sortedVersions.length - 1] : semverMaxSatisfying(sortedVersions, this.version) + + if (correctVersion) { + privatesAccessor(this).correctVersion = correctVersion + } else { + throw new Error(`Package ${this.name} has no version ${this.version}`) + } + } + + async getPackageJson(pkgZipStream) { + const streamToBuffer = stream => new Promise((resolve, reject) => { + const data = [] + + stream.on('data', (chunk) => { + data.push(chunk) + }) + stream.on('end', () => { + resolve(Buffer.concat(data)) + }) + + stream.on('error', (error) => { + reject(error) + }) + }) + + return new Promise((resolve, reject) => { + // Note: unzip-stream parses the zipped package file by file so it might take some time + // to find package.json in the zip. Is there any better library to parse it??? + const unzipStream = pkgZipStream.pipe(unzip.Parse()).on('entry', async(entry) => { + if (entry.path === 'package.json') { + try { + const data = await streamToBuffer(entry) + resolve(data.toString()) + } catch (err) { + reject(err) + } finally { + pkgZipStream.unpipe() + pkgZipStream.destroy() + unzipStream.destroy() + } + } else { + entry.autodrain() + } + }) + }) + } + + async loadPackageJson() { + const pkgZipStream = await this.getPackage(), + packageJson = await this.getPackageJson(pkgZipStream) + + privatesAccessor(this).packageJson = packageJson + } + + async loadContent() { + const { loadedZipStream } = privatesAccessor(this) + + if (!loadedZipStream) { + await this.loadPackageJson() + privatesAccessor(this).loadedZipStream = true + } + } + + async readConfigFiles() { + await this.resolvePackageVersion() + + await this.loadContent() + + const { packageJson } = privatesAccessor(this) + + if (packageJson) { + return JSON.parse(packageJson) + } + + throw new Error('No package.json found') + } + + async loadPackageInfo() { + try { + const info = await this.readConfigFiles(), + packageInfo = { + name: info.name, + version: info.version, + dependencies: info.dependencies || {}, + engines: info.engines || {} + } + Object.assign(privatesAccessor(this), packageInfo) + } catch (err) { + throw err + } + } + + async getStream() { + return this.getPackage() } - // TODO: implement some version resolution - // SemverResolver } module.exports = RegistrySource diff --git a/packages/mdctl-packages/lib/sources/source.js b/packages/mdctl-packages/lib/sources/source.js index f6a8db49..040e8acc 100644 --- a/packages/mdctl-packages/lib/sources/source.js +++ b/packages/mdctl-packages/lib/sources/source.js @@ -1,4 +1,4 @@ -const { privatesAccessor } = require('@medable/mdctl-core-utils/privates') +const { privatesAccessor } = require('@medable/mdctl-core-utils/privates') class Source { @@ -31,15 +31,15 @@ class Source { return privatesAccessor(this).dependencies } - get engine() { - return privatesAccessor(this).engine + get engines() { + return privatesAccessor(this).engines } get options() { return privatesAccessor(this).options } - get type(){ + get type() { return this.constructor.name } @@ -47,7 +47,6 @@ class Source { throw new Error('Must be implement on inherited sources') } - async getStream() { throw new Error('Must be implement on inherited sources') } diff --git a/packages/mdctl-packages/package-lock.json b/packages/mdctl-packages/package-lock.json index a09842c8..5a116ac8 100644 --- a/packages/mdctl-packages/package-lock.json +++ b/packages/mdctl-packages/package-lock.json @@ -403,6 +403,41 @@ "fastq": "^1.6.0" } }, + "@sinonjs/commons": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", + "integrity": "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==", + "dev": true, + "requires": { + "type-detect": "4.0.8" + } + }, + "@sinonjs/fake-timers": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-8.1.0.tgz", + "integrity": "sha512-OAPJUAtgeINhh/TAlUID4QTs53Njm7xzddaVlEs/SXwgtiD1tW22zAB/W1wdqfrpmikgaWQ9Fw6Ws+hsiRm5Vg==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.7.0" + } + }, + "@sinonjs/samsam": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-6.0.2.tgz", + "integrity": "sha512-jxPRPp9n93ci7b8hMfJOFDPRLFYadN6FSpeROFTR4UNF4i5b+EK6m4QXPO46BDhFgRy1JuS87zAnFOzCUwMJcQ==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.6.0", + "lodash.get": "^4.4.2", + "type-detect": "^4.0.8" + } + }, + "@sinonjs/text-encoding": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", + "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", + "dev": true + }, "@types/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", @@ -3363,6 +3398,12 @@ } } }, + "just-extend": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz", + "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", + "dev": true + }, "jwa": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", @@ -3495,6 +3536,12 @@ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", + "dev": true + }, "lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -3916,6 +3963,30 @@ } } }, + "nise": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.0.tgz", + "integrity": "sha512-W5WlHu+wvo3PaKLsJJkgPup2LrsXCcm7AWwyNZkUnn5rwPkuPBi3Iwk5SQtN0mv+K65k7nKKjwNQ30wg3wLAQQ==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.7.0", + "@sinonjs/fake-timers": "^7.0.4", + "@sinonjs/text-encoding": "^0.7.1", + "just-extend": "^4.0.2", + "path-to-regexp": "^1.7.0" + }, + "dependencies": { + "@sinonjs/fake-timers": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-7.1.2.tgz", + "integrity": "sha512-iQADsW4LBMISqZ6Ci1dupJL9pprqwcVFTcOsEmQOEhW+KLCVn/Y4Jrvg2k19fIHCp+iFprriYPTdRcQR8NbUPg==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.7.0" + } + } + } + }, "node-abi": { "version": "2.30.1", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz", @@ -4271,6 +4342,23 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dev": true, + "requires": { + "isarray": "0.0.1" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "dev": true + } + } + }, "path-type": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", @@ -5417,6 +5505,20 @@ "simple-concat": "^1.0.0" } }, + "sinon": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-12.0.1.tgz", + "integrity": "sha512-iGu29Xhym33ydkAT+aNQFBINakjq69kKO6ByPvTsm3yyIACfyQttRTP03aBP/I8GfhFmLzrnKwNNkr0ORb1udg==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.8.3", + "@sinonjs/fake-timers": "^8.1.0", + "@sinonjs/samsam": "^6.0.2", + "diff": "^5.0.0", + "nise": "^5.1.0", + "supports-color": "^7.2.0" + } + }, "slash": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", diff --git a/packages/mdctl-packages/package.json b/packages/mdctl-packages/package.json index 965b2005..79577bf2 100644 --- a/packages/mdctl-packages/package.json +++ b/packages/mdctl-packages/package.json @@ -17,6 +17,7 @@ "license": "MIT", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", + "test:package": "mocha --timeout 1200000 --ui bdd ./test", "lint": "eslint . --ext .js" }, "dependencies": { @@ -48,6 +49,7 @@ "eslint-plugin-node": "^8.0.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.1", - "mocha": "^9.1.3" + "mocha": "^9.1.3", + "sinon": "^12.0.1" } } diff --git a/packages/mdctl-packages/test/test.package.js b/packages/mdctl-packages/test/test.package.js index 4fcefb23..86dc630a 100644 --- a/packages/mdctl-packages/test/test.package.js +++ b/packages/mdctl-packages/test/test.package.js @@ -16,7 +16,7 @@ describe('CLI - Pkg - Install package', () => { console.log(p.dependenciesPackages) }) - it('test package', async() => { + it.skip('test package', async() => { const pkg = new Package('my-study-1022992',{ name: 'my-study-1022992', version: '1.0.0-rc.1', @@ -42,7 +42,7 @@ describe('CLI - Pkg - Install package', () => { // stream.resume() }) - it('test package export', async() => { + it.skip('test package export', async() => { const stream = fs.createReadStream(path.resolve('data.ndjson')) const pkg = new Package('exported','1.0.0-rc.1', { ndjsonStream: stream.pipe(ndjson.stringify()) diff --git a/packages/mdctl-packages/test/test.registry.js b/packages/mdctl-packages/test/test.registry.js new file mode 100644 index 00000000..106e3e45 --- /dev/null +++ b/packages/mdctl-packages/test/test.registry.js @@ -0,0 +1,65 @@ +const sinon = require('sinon'), + path = require('path'), + fs = require('fs'), + { RegistrySource } = require('../lib/index'), + ZipTree = require('../lib/zip_tree') + +describe('Registry Source Test', () => { + + let rs, + sandbox + + beforeEach(() => { + sandbox = sinon.createSandbox() + rs = new RegistrySource('TestPackage', 'latest', { + registryUrl: 'http://registry.com', + registryProjectId: '100', + registryToken: 'test_token' + }) + }) + + afterEach(() => { + rs = null + sandbox.restore() + }) + + it('Test registry load package info', async() => { + const packageJson = require(path.resolve(__dirname, 'test_pkg', 'package.json')), + readConfigFilesStub = sandbox.stub(rs, 'readConfigFiles').resolves(packageJson) + + await rs.loadPackageInfo() + + sinon.assert.calledOnce(readConfigFilesStub) + sinon.assert.match(rs.name, 'TestPackage') + sinon.assert.match(rs.version, '1.0.5') + sinon.assert.match(rs.engines, { + "cortex": "> 2.15.8" + }) + sinon.assert.match(rs.dependencies, { + "axon": "git+https://gitlab.medable.com/axon/org.git#test_pkg", + "data-transfers": "git+https://gitlab.medable.com/platform/environments/data-transfers.git#test_pkg" + }) + }) + + it('Test registry get package stream', async() => { + const packageZipTree = new ZipTree(path.resolve(__dirname, 'test_pkg'), { fs }), + packageZipStream = await packageZipTree.compress(), + getStreamStub = sandbox.stub(rs, 'getStream').resolves(packageZipStream), + getPackageJsonStub = sandbox.stub(rs, 'getPackageJson').resolves(require(path.resolve(__dirname, 'test_pkg', 'package.json'))), + zipStream = await rs.getStream(), + packageJson = await rs.getPackageJson(zipStream) + + sinon.assert.calledOnce(getStreamStub) + sinon.assert.calledOnce(getPackageJsonStub) + sinon.assert.match(packageJson.name, 'TestPackage') + sinon.assert.match(packageJson.version, '1.0.5') + sinon.assert.match(packageJson.engines, { + "cortex": "> 2.15.8" + }) + sinon.assert.match(packageJson.dependencies, { + "axon": "git+https://gitlab.medable.com/axon/org.git#test_pkg", + "data-transfers": "git+https://gitlab.medable.com/platform/environments/data-transfers.git#test_pkg" + }) + }) + +}) \ No newline at end of file