diff --git a/CHANGELOG.md b/CHANGELOG.md index b0422c3f..92867d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ 4.2.0 / ================ * new API `LanguageService.getLanguageStatus` + * support for $ref with $id 4.1.6 / 2021-07-16 ================ diff --git a/src/services/jsonSchemaService.ts b/src/services/jsonSchemaService.ts index f4f2accf..272b77fd 100644 --- a/src/services/jsonSchemaService.ts +++ b/src/services/jsonSchemaService.ts @@ -129,7 +129,7 @@ class SchemaHandle implements ISchemaHandle { public readonly uri: string; public readonly dependencies: SchemaDependencies; - + public readonly anchors: Map; private resolvedSchema: Thenable | undefined; private unresolvedSchema: Thenable | undefined; private readonly service: JSONSchemaService; @@ -138,6 +138,7 @@ class SchemaHandle implements ISchemaHandle { this.service = service; this.uri = uri; this.dependencies = new Set(); + this.anchors = new Map(); if (unresolvedSchemaContent) { this.unresolvedSchema = this.service.promise.resolve(new UnresolvedSchema(unresolvedSchemaContent)); } @@ -153,7 +154,7 @@ class SchemaHandle implements ISchemaHandle { public getResolvedSchema(): Thenable { if (!this.resolvedSchema) { this.resolvedSchema = this.getUnresolvedSchema().then(unresolved => { - return this.service.resolveSchemaContent(unresolved, this.uri, this.dependencies); + return this.service.resolveSchemaContent(unresolved, this); }); } return this.resolvedSchema; @@ -164,6 +165,7 @@ class SchemaHandle implements ISchemaHandle { this.resolvedSchema = undefined; this.unresolvedSchema = undefined; this.dependencies.clear(); + this.anchors.clear(); return hasChanges; } } @@ -404,7 +406,7 @@ export class JSONSchemaService implements IJSONSchemaService { ); } - public resolveSchemaContent(schemaToResolve: UnresolvedSchema, schemaURL: string, dependencies: SchemaDependencies): Thenable { + public resolveSchemaContent(schemaToResolve: UnresolvedSchema, handle: SchemaHandle): Thenable { const resolveErrors: string[] = schemaToResolve.errors.slice(0); const schema = schemaToResolve.schema; @@ -438,38 +440,91 @@ export class JSONSchemaService implements IJSONSchemaService { return current; }; - const merge = (target: JSONSchema, sourceRoot: JSONSchema, sourceURI: string, refSegment: string | undefined): void => { + const merge = (target: JSONSchema, section: any): void => { + for (const key in section) { + if (section.hasOwnProperty(key) && !target.hasOwnProperty(key)) { + (target)[key] = section[key]; + } + } + }; + + const mergeByJsonPointer = (target: JSONSchema, sourceRoot: JSONSchema, sourceURI: string, refSegment: string | undefined): void => { const path = refSegment ? decodeURIComponent(refSegment) : undefined; const section = findSection(sourceRoot, path); if (section) { - for (const key in section) { - if (section.hasOwnProperty(key) && !target.hasOwnProperty(key)) { - (target)[key] = section[key]; - } - } + merge(target, section); } else { resolveErrors.push(localize('json.schema.invalidref', '$ref \'{0}\' in \'{1}\' can not be resolved.', path, sourceURI)); } }; - const resolveExternalLink = (node: JSONSchema, uri: string, refSegment: string | undefined, parentSchemaURL: string, parentSchemaDependencies: SchemaDependencies): Thenable => { + const isSubSchemaRef = (refSegment?: string): boolean => { + // Check if the first character is not '/' to determine whether it's a sub schema reference or a JSON Pointer + return !!refSegment && refSegment.charAt(0) !== '/'; + }; + + const reconstructRefURI = (uri: string, fragment?: string, separator: string = '#'): string => { + return normalizeId(`${uri}${separator}${fragment}`); + }; + + // To find which $refs point to which $ids we keep two maps: + // pendingSubSchemas '$id' we expect to encounter (if they exist) + // handle.anchors for the ones we have encountered + const pendingSubSchemas: Map = new Map(); + + const tryMergeSubSchema = (target: JSONSchema, id: string, handle: SchemaHandle): boolean => { + // Get the full URI for the current schema to avoid matching schema1#hello and schema2#hello to the same + // reference by accident + const fullId = reconstructRefURI(handle.uri, id); + const resolved = handle.anchors.get(fullId); + if (resolved) { + merge(target, resolved); + return true; // return success + } + + // This subschema has not been resolved yet + // Remember the target to merge later once resolved + let pending = pendingSubSchemas.get(fullId); + if (!pending) { + pending = []; + pendingSubSchemas.set(fullId, pending); + } + pending.push(target); + return false; // return failure - merge didn't occur + }; + + const resolveExternalLink = (node: JSONSchema, uri: string, refSegment: string | undefined, parentHandle: SchemaHandle): Thenable => { if (contextService && !/^[A-Za-z][A-Za-z0-9+\-.+]*:\/\/.*/.test(uri)) { - uri = contextService.resolveRelativePath(uri, parentSchemaURL); + uri = contextService.resolveRelativePath(uri, parentHandle.uri); } uri = normalizeId(uri); const referencedHandle = this.getOrAddSchemaHandle(uri); return referencedHandle.getUnresolvedSchema().then(unresolvedSchema => { - parentSchemaDependencies.add(uri); + parentHandle.dependencies.add(uri); if (unresolvedSchema.errors.length) { const loc = refSegment ? uri + '#' + refSegment : uri; resolveErrors.push(localize('json.schema.problemloadingref', 'Problems loading reference \'{0}\': {1}', loc, unresolvedSchema.errors[0])); } - merge(node, unresolvedSchema.schema, uri, refSegment); - return resolveRefs(node, unresolvedSchema.schema, uri, referencedHandle.dependencies); + + // A placeholder promise that might execute later a ref resolution for the newly resolved schema + let externalLinkPromise: Thenable = Promise.resolve(true); + if (refSegment === undefined || !isSubSchemaRef(refSegment)) { + // This is not a sub schema, merge the regular way + mergeByJsonPointer(node, unresolvedSchema.schema, uri, refSegment); + } else { + // This is a reference to a subschema + if (!tryMergeSubSchema(node, refSegment, referencedHandle)) { + // We weren't able to merge currently so we'll try to resolve this schema first to obtain subschemas + // that could be missed + // to improve: it would be enough to find the nodes, no need to resolve the full schema + externalLinkPromise = resolveRefs(unresolvedSchema.schema, unresolvedSchema.schema, referencedHandle); + } + } + return externalLinkPromise.then(() => resolveRefs(node, unresolvedSchema.schema, referencedHandle)); }); }; - const resolveRefs = (node: JSONSchema, parentSchema: JSONSchema, parentSchemaURL: string, parentSchemaDependencies: SchemaDependencies): Thenable => { + const resolveRefs = (node: JSONSchema, parentSchema: JSONSchema, parentHandle: SchemaHandle): Thenable => { if (!node || typeof node !== 'object') { return Promise.resolve(null); } @@ -517,11 +572,18 @@ export class JSONSchemaService implements IJSONSchemaService { const segments = ref.split('#', 2); delete next.$ref; if (segments[0].length > 0) { - openPromises.push(resolveExternalLink(next, segments[0], segments[1], parentSchemaURL, parentSchemaDependencies)); + // This is a reference to an external schema + openPromises.push(resolveExternalLink(next, segments[0], segments[1], parentHandle)); return; } else { + // This is a reference inside the current schema if (!seenRefs.has(ref)) { - merge(next, parentSchema, parentSchemaURL, segments[1]); // can set next.$ref again, use seenRefs to avoid circle + const id = segments[1]; + if (id !== undefined && isSubSchemaRef(id)) { // A $ref to a sub-schema with an $id (i.e #hello) + tryMergeSubSchema(next, id, handle); + } else { // A $ref to a JSON Pointer (i.e #/definitions/foo) + mergeByJsonPointer(next, parentSchema, parentHandle.uri, id); // can set next.$ref again, use seenRefs to avoid circle + } seenRefs.add(ref); } } @@ -532,18 +594,54 @@ export class JSONSchemaService implements IJSONSchemaService { collectArrayEntries(next.anyOf, next.allOf, next.oneOf, next.items); }; + const handleId = (next: JSONSchema) => { + // TODO figure out should loops be preventse + const id = next.$id || next.id; + if (typeof id === 'string' && id.charAt(0) === '#') { + delete next.$id; + delete next.id; + // Use a blank separator, as the $id already has the '#' + const fullId = reconstructRefURI(parentHandle.uri, id, ''); + + const resolved = parentHandle.anchors.get(fullId); + if (!resolved) { + // it's resolved now + parentHandle.anchors.set(fullId, next); + } else if (resolved !== next) { + // Duplicate may occur in recursive $refs, but as long as they are the same object + // it's ok, otherwise report and error + resolveErrors.push(localize('json.schema.duplicateid', 'Duplicate id declaration: \'{0}\'', id)); + } + + // Resolve all pending requests and cleanup the queue list + const pending = pendingSubSchemas.get(fullId); + if (pending) { + for (const target of pending) { + merge(target, next); + } + pendingSubSchemas.delete(fullId); + } + } + }; + while (toWalk.length) { const next = toWalk.pop()!; if (seen.has(next)) { continue; } seen.add(next); + handleId(next); handleRef(next); } return this.promise.all(openPromises); }; - return resolveRefs(schema, schema, schemaURL, dependencies).then(_ => new ResolvedSchema(schema, resolveErrors)); + return resolveRefs(schema, schema, handle).then(_ => { + for (const unresolvedSubschemaId in pendingSubSchemas) { + resolveErrors.push(localize('json.schema.idnotfound', 'Subschema with id \'{0}\' was not found', unresolvedSubschemaId)); + } + return new ResolvedSchema(schema, resolveErrors); + }); } private getSchemaFromProperty(resource: string, document: Parser.JSONDocument): string | undefined { if (document.root?.type === 'object') { @@ -618,7 +716,8 @@ export class JSONSchemaService implements IJSONSchemaService { public getMatchingSchemas(document: TextDocument, jsonDocument: Parser.JSONDocument, schema?: JSONSchema): Thenable { if (schema) { const id = schema.id || ('schemaservice://untitled/matchingSchemas/' + idCounter++); - return this.resolveSchemaContent(new UnresolvedSchema(schema), id, new Set()).then(resolvedSchema => { + const handle = this.addSchemaHandle(id, schema); + return handle.getResolvedSchema().then(resolvedSchema => { return jsonDocument.getMatchingSchemas(resolvedSchema.schema).filter(s => !s.inverted); }); } diff --git a/src/services/jsonValidation.ts b/src/services/jsonValidation.ts index 25412021..aacc01b8 100644 --- a/src/services/jsonValidation.ts +++ b/src/services/jsonValidation.ts @@ -103,7 +103,8 @@ export class JSONValidation { if (schema) { const id = schema.id || ('schemaservice://untitled/' + idCounter++); - return this.jsonSchemaService.resolveSchemaContent(new UnresolvedSchema(schema), id, new Set()).then(resolvedSchema => { + const handle = this.jsonSchemaService.registerExternalSchema(id, [], schema); + return handle.getResolvedSchema().then(resolvedSchema => { return getDiagnostics(resolvedSchema); }); } diff --git a/src/test/schema.test.ts b/src/test/schema.test.ts index 6f33cc35..1b852bdc 100644 --- a/src/test/schema.test.ts +++ b/src/test/schema.test.ts @@ -138,7 +138,7 @@ suite('JSON Schema', () => { service.setSchemaContributions({ schemas: { "https://myschemastore/main/schema1.json": { - id: 'https://myschemastore/schema1.json', + id: 'https://myschemastore/main/schema1.json', type: 'object', properties: { p1: { @@ -181,12 +181,12 @@ suite('JSON Schema', () => { }); - test('Resolving $refs 3', async function () { + test('Resolving $refs 4', async function () { const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); service.setSchemaContributions({ schemas: { "https://myschemastore/main/schema1.json": { - id: 'https://myschemastore/schema1.json', + id: 'https://myschemastore/main/schema1.json', type: 'object', properties: { p1: { @@ -233,7 +233,7 @@ suite('JSON Schema', () => { service.setSchemaContributions({ schemas: { "https://myschemastore/main/schema1.json": { - id: 'https://myschemastore/schema1.json', + id: 'https://myschemastore/main/schema1.json', type: 'object', properties: { p1: { @@ -272,6 +272,378 @@ suite('JSON Schema', () => { }); + test('Resolving $refs to local $ids', async function () { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + + service.setSchemaContributions({ + schemas: { + "https://myschemastore/main/schema1.json": { + id: 'https://myschemastore/main/schema1.json', + definitions:{ + hello: { + id: '#hello', + type: 'string', + const: 'hello' + }, + world: { + $id: '#world', + type: 'string', + const: 'world' + } + }, + type: 'object', + properties: { + p1: { + $ref: '#hello' + }, + p2: { + $ref: '#world' + } + } + } + } + }); + + return service.getResolvedSchema('https://myschemastore/main/schema1.json').then(fs => { + assert.deepEqual(fs?.schema.properties?.p1, { + type: 'string', + const: 'hello' + }); + assert.deepEqual(fs?.schema.properties?.p2, { + type: 'string', + const: 'world' + }); + }); + + }); + + test('Resolving $refs to external $ids', async function () { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + service.setSchemaContributions({ + schemas: { + "https://myschemastore/main/schema1.json": { + id: 'https://myschemastore/main/schema1.json', + type: 'object', + properties: { + p1: { + '$ref': 'schema2.json#hello' + }, + p2: { + '$ref': './schema2.json#/definitions/hello' + }, + p3: { + '$ref': '/main/schema2.json#/definitions/hello' + } + } + }, + "https://myschemastore/main/schema2.json": { + id: 'https://myschemastore/main/schema2.json', + definitions: { + "hello": { + $id: "#hello", + "type": "string", + "enum": ["object"], + } + } + } + } + }); + + return service.getResolvedSchema('https://myschemastore/main/schema1.json').then(fs => { + assert.deepEqual(fs?.schema.properties?.['p1'], { + type: 'string', + enum: ["object"] + }); + assert.deepEqual(fs?.schema.properties?.['p2'], { + type: 'string', + enum: ["object"] + }); + assert.deepEqual(fs?.schema.properties?.['p3'], { + type: 'string', + enum: ["object"] + }); + }); + + + }); + + test('Resolving $refs to external $ids with same as local', async function () { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + service.setSchemaContributions({ + schemas: { + "https://myschemastore/main/schema1.json": { + id: 'https://myschemastore/main/schema1.json', + definitions: { + "hello": { + $id: "#hello", + "type": "string", + "const": "wrong", + } + }, + type: 'object', + properties: { + p1: { + '$ref': 'schema2.json#hello' + } + } + }, + "https://myschemastore/main/schema2.json": { + id: 'https://myschemastore/main/schema2.json', + definitions: { + "hello": { + $id: "#hello", + "type": "string", + "const": "correct" + } + } + } + } + }); + + return service.getResolvedSchema('https://myschemastore/main/schema1.json').then(fs => { + assert.deepEqual(fs?.schema.properties?.['p1'], { + type: 'string', + const: 'correct' + }); + }); + + + }); + + + test('Resolving external $ref two levels', async function () { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + service.setSchemaContributions({ + schemas: { + "https://myschemastore/main/schema1.json": { + id: 'https://myschemastore/main/schema1.json', + definitions: { + "world": { + $id: '#world', + type: 'string', + const: 'world' + } + }, + type: 'object', + properties: { + p1: { + $ref: 'schema2.json#blue' + } + } + }, + "https://myschemastore/main/schema3.json": { + id: 'https://myschemastore/main/schema3.json', + definitions: { + "world": { + $id: '#world', + type: 'string', + const: 'world' + } + } + }, + "https://myschemastore/main/schema2.json": { + id: 'https://myschemastore/main/schema2.json', + definitions: { + "blue": { + $id: '#blue', + $ref: 'schema3.json#world' + } + } + } + } + }); + + const resolvedSchema = await service.getResolvedSchema('https://myschemastore/main/schema1.json'); + assert.deepStrictEqual(resolvedSchema?.schema.properties?.p1, { + type: 'string', + const: 'world' + }); + }); + + test('Resolving external $ref recursive', async function () { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + service.setSchemaContributions({ + schemas: { + "https://myschemastore/main/schema1.json": { + id: 'https://myschemastore/main/schema1.json', + definitions: { + "world": { + $id: '#world', + type: 'string', + const: 'world' + } + }, + type: 'object', + properties: { + p1: { + $ref: 'schema2.json#blue' + } + } + }, + "https://myschemastore/main/schema2.json": { + id: 'https://myschemastore/main/schema2.json', + definitions: { + "blue": { + $id: '#blue', + $ref: 'schema1.json#world' + } + } + } + } + }); + + const resolvedSchema = await service.getResolvedSchema('https://myschemastore/main/schema1.json'); + assert.deepStrictEqual(resolvedSchema?.schema.properties?.p1, { + type: 'string', + const: 'world' + }); + }); + + + test('Resolving external $ref to already resolved schema', async function () { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + service.setSchemaContributions({ + schemas: { + "https://myschemastore/main/schema1.json": { + type: 'object', + properties: { + p1: { + $ref: 'schema2.json#blue' + } + } + }, + "https://myschemastore/main/schema3.json": { + type: 'object', + properties: { + p1: { + $ref: 'schema2.json#blue' + } + } + }, + "https://myschemastore/main/schema2.json": { + definitions: { + "blue": { + $id: '#blue', + type: 'string', + const: 'blue' + } + } + } + } + }); + + const resolvedSchema1 = await service.getResolvedSchema('https://myschemastore/main/schema1.json'); + assert.deepStrictEqual(resolvedSchema1?.schema.properties?.p1, { + type: 'string', + const: 'blue' + }); + const resolvedSchema3 = await service.getResolvedSchema('https://myschemastore/main/schema3.json'); + assert.deepStrictEqual(resolvedSchema3?.schema.properties?.p1, { + type: 'string', + const: 'blue' + }); + }); + + + test('Resolving $refs 5', async function() { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + service.setSchemaContributions({ + schemas:{ + "https://myschemastore/main/schema1.json": { + "type": "object", + "properties": { + "p1": { + "$ref": "#hello" + }, + "p2": { + "$ref": "#world" + }, + "p3": { + "id": "#hello", + "type": "string", + "const": "hello" + }, + "p4": { + "type": "object", + "properties": { + "deep": { + "$id": "#world", + "type": "string", + "const": "world" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + } + }); + + return service.getResolvedSchema('https://myschemastore/main/schema1.json').then(fs => { + assert.deepEqual(fs?.schema.properties?.['p1'], { + type: 'string', + const: 'hello' + }); + + assert.deepEqual(fs?.schema.properties?.['p2'], { + "type": "string", + "const": "world" + }); + }); + }); + + test('Recursive $refs to $ids', async function() { + const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext); + service.setSchemaContributions({ + schemas: { + "https://myschemastore/main/schema1.json": { + "type": "object", + "definitions": { + "foo": { + "id": "#foo", + "type": "object", + "properties": { + "bar": { + "type": "string", + "const": "hello" + }, + "foo": { + "$ref": "#foo" + } + }, + "additionalProperties": false + } + }, + "properties": { + "foo": { + "$ref": "#foo" + } + }, + "additionalProperties": false + } + } + }); + + return service.getResolvedSchema('https://myschemastore/main/schema1.json').then(fs => { + assert.deepEqual(fs?.schema.properties?.['foo'], { + "type": "object", + "properties": { + "bar": { + "type": "string", + "const": "hello" + }, + "foo": { + "additionalProperties": false, + properties: fs?.schema.definitions?.['foo'].properties, + type:"object" + } + }, + "additionalProperties": false + }); + }); + }); + test('FileSchema', async function () { const service = new SchemaService.JSONSchemaService(newMockRequestService(), workspaceContext);