diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index ec8860ea4e..89ab4ea48f 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -312,12 +312,13 @@ function buildPipelineInternal( end, name, true, + partitionKey, ); } } if (ast.where && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) { - end = applyWhere(end, ast.where, delegate, name); + end = applyWhere(end, ast.where, delegate, name, partitionKey); } if (ast.limit !== undefined) { @@ -334,7 +335,15 @@ function buildPipelineInternal( if (ast.related) { for (const csq of ast.related) { - end = applyCorrelatedSubQuery(csq, delegate, queryID, end, name, false); + end = applyCorrelatedSubQuery( + csq, + delegate, + queryID, + end, + name, + false, + partitionKey, + ); } } @@ -346,6 +355,7 @@ function applyWhere( condition: Condition, delegate: BuilderDelegate, name: string, + partitionKey: CompoundKey | undefined, ): Input { if (!conditionIncludesFlippedSubqueryAtAnyLevel(condition)) { return buildFilterPipeline(input, delegate, filterInput => @@ -353,7 +363,7 @@ function applyWhere( ); } - return applyFilterWithFlips(input, condition, delegate, name); + return applyFilterWithFlips(input, condition, delegate, name, partitionKey); } function applyFilterWithFlips( @@ -361,6 +371,7 @@ function applyFilterWithFlips( condition: Condition, delegate: BuilderDelegate, name: string, + partitionKey: CompoundKey | undefined, ): Input { let end = input; assert(condition.type !== 'simple', 'Simple conditions cannot have flips'); @@ -386,7 +397,7 @@ function applyFilterWithFlips( } assert(withFlipped.length > 0, 'Impossible to have no flips here'); for (const cond of withFlipped) { - end = applyFilterWithFlips(end, cond, delegate, name); + end = applyFilterWithFlips(end, cond, delegate, name, partitionKey); } break; } @@ -419,7 +430,9 @@ function applyFilterWithFlips( } for (const cond of withFlipped) { - branches.push(applyFilterWithFlips(end, cond, delegate, name)); + branches.push( + applyFilterWithFlips(end, cond, delegate, name, partitionKey), + ); } const ufi = new UnionFanIn(ufo, branches); @@ -439,11 +452,15 @@ function applyFilterWithFlips( `${name}.${sq.subquery.alias}`, sq.correlation.childField, ); + + const flippedJoinName = `${name}:flipped-join(${sq.subquery.alias})`; const flippedJoin = new FlippedJoin({ parent: end, child, + storage: delegate.createStorage(flippedJoinName), parentKey: sq.correlation.parentField, childKey: sq.correlation.childField, + partitionKey, relationshipName: must( sq.subquery.alias, 'Subquery must have an alias', @@ -453,10 +470,7 @@ function applyFilterWithFlips( }); delegate.addEdge(end, flippedJoin); delegate.addEdge(child, flippedJoin); - end = delegate.decorateInput( - flippedJoin, - `${name}:flipped-join(${sq.subquery.alias})`, - ); + end = delegate.decorateInput(flippedJoin, flippedJoinName); break; } } @@ -598,6 +612,7 @@ function applyCorrelatedSubQuery( end: Input, name: string, fromCondition: boolean, + partitionKey: CompoundKey | undefined, ) { // TODO: we only omit the join if the CSQ if from a condition since // we want to create an empty array for `related` fields that are `limit(0)` @@ -621,6 +636,7 @@ function applyCorrelatedSubQuery( storage: delegate.createStorage(joinName), parentKey: sq.correlation.parentField, childKey: sq.correlation.childField, + partitionKey, relationshipName: sq.subquery.alias, hidden: sq.hidden ?? false, system: sq.system ?? 'client', diff --git a/packages/zql/src/ivm/flipped-join.more-fetch.test.ts b/packages/zql/src/ivm/flipped-join.more-fetch.test.ts index d29da7824b..8cce781242 100644 --- a/packages/zql/src/ivm/flipped-join.more-fetch.test.ts +++ b/packages/zql/src/ivm/flipped-join.more-fetch.test.ts @@ -169,7 +169,14 @@ suite('one:many:one', () => { ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ".issueLabels_1_0:flipped-join(labels_1)": {}, + ".issueLabels_2_1:flipped-join(labels_2)": {}, + ":flipped-join(issueLabels_1_0)": {}, + ":flipped-join(issueLabels_2_1)": {}, + } + `); expect(log).toMatchInlineSnapshot(` [ diff --git a/packages/zql/src/ivm/flipped-join.push.test.ts b/packages/zql/src/ivm/flipped-join.push.test.ts index cf81c5cebd..70abb1c327 100644 --- a/packages/zql/src/ivm/flipped-join.push.test.ts +++ b/packages/zql/src/ivm/flipped-join.push.test.ts @@ -97,7 +97,11 @@ suite('push one:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(`[]`); }); @@ -138,7 +142,11 @@ suite('push one:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(`[]`); }); @@ -211,7 +219,11 @@ suite('push one:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -314,7 +326,11 @@ suite('push one:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -383,7 +399,11 @@ suite('push one:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(`[]`); }); @@ -457,7 +477,11 @@ suite('push one:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -520,7 +544,11 @@ suite('push one:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(`[]`); }); @@ -579,7 +607,11 @@ suite('push one:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -663,7 +695,11 @@ suite('push one:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -907,7 +943,11 @@ suite('push one:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -1099,7 +1139,11 @@ suite('push one:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -1218,7 +1262,11 @@ suite('push one:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -1399,7 +1447,11 @@ suite('push one:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -1590,7 +1642,11 @@ suite('push one:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -1756,7 +1812,11 @@ suite('push many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -1819,7 +1879,11 @@ suite('push many:one', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(`[]`); }); @@ -1891,7 +1955,11 @@ suite('push many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -2017,7 +2085,11 @@ suite('push many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -2216,7 +2288,11 @@ suite('push many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -2604,7 +2680,14 @@ suite('push many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ".issues:flipped-join(comments)": { + ""partition","[\\"i2\\"]","[\\"u1\\"]","[\\"i2\\"]",": true, + }, + ":flipped-join(issues)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ @@ -2727,7 +2810,11 @@ suite('push many:one', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(`[]`); }); @@ -2855,7 +2942,11 @@ suite('push many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -2980,7 +3071,7 @@ suite('push one:many:many', () => { }, } as const; - test('fetch one parent, one child, add grandchild', () => { + test.only('fetch one parent, one child, add grandchild', () => { const {log, data, actualStorage, pushes} = runPushTest({ sources, sourceContents: { @@ -3006,78 +3097,6 @@ suite('push one:many:many', () => { "type": "add", }, ], - [ - ".comments:source(comment)", - "fetch", - { - "constraint": { - "id": "c1", - }, - }, - ], - [ - ".comments.revisions:source(revision)", - "fetch", - { - "constraint": { - "commentID": "c1", - }, - }, - ], - [ - ".comments:flipped-join(revisions)", - "push", - { - "row": { - "id": "c1", - "issueID": "i1", - }, - "type": "add", - }, - ], - [ - ":source(issue)", - "fetch", - { - "constraint": { - "id": "i1", - }, - }, - ], - [ - ".comments:flipped-join(revisions)", - "fetch", - { - "constraint": { - "issueID": "i1", - }, - }, - ], - [ - ".comments.revisions:source(revision)", - "fetch", - {}, - ], - [ - ".comments:source(comment)", - "fetch", - { - "constraint": { - "id": "c1", - "issueID": "i1", - }, - }, - ], - [ - ":flipped-join(comments)", - "push", - { - "row": { - "id": "i1", - }, - "type": "add", - }, - ], ] `); expect(data).toMatchInlineSnapshot(` @@ -3102,7 +3121,7 @@ suite('push one:many:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(); expect(pushes).toMatchInlineSnapshot(` [ { @@ -3260,7 +3279,14 @@ suite('push one:many:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ".comments:flipped-join(revisions)": { + ""partition","[\\"c1\\"]","[\\"i1\\"]","[\\"c1\\"]",": true, + }, + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -3403,7 +3429,12 @@ suite('push one:many:many', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ".comments:flipped-join(revisions)": {}, + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -3525,7 +3556,12 @@ suite('push one:many:many', () => { ] `); expect(data).toMatchInlineSnapshot(`[]`); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ".comments:flipped-join(revisions)": {}, + ":flipped-join(comments)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -3757,7 +3793,7 @@ suite('push one:many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(); expect(pushes).toMatchInlineSnapshot(` [ { @@ -3913,7 +3949,14 @@ suite('push one:many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ".issueLabels:flipped-join(labels)": { + ""partition","[\\"l1\\"]","[\\"i1\\"]","[\\"i1\\",\\"l1\\"]",": true, + }, + ":flipped-join(issueLabels)": {}, + } + `); expect(pushes).toMatchInlineSnapshot(` [ { @@ -4147,7 +4190,7 @@ suite('push one:many:one', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(); expect(pushes).toMatchInlineSnapshot(` [ { @@ -4542,7 +4585,12 @@ describe('edit assignee', () => { ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(assignee_1)": {}, + ":flipped-join(creator_0)": {}, + } + `); }); test('from none to many', () => { @@ -4882,7 +4930,12 @@ describe('edit assignee', () => { ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(assignee_1)": {}, + ":flipped-join(creator_0)": {}, + } + `); }); test('from one to none', () => { @@ -5105,7 +5158,12 @@ describe('edit assignee', () => { ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(assignee_1)": {}, + ":flipped-join(creator_0)": {}, + } + `); }); test('from many to none', () => { @@ -5411,7 +5469,12 @@ describe('edit assignee', () => { ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(assignee_1)": {}, + ":flipped-join(creator_0)": {}, + } + `); }); }); @@ -5657,7 +5720,11 @@ describe('joins with compound join keys', () => { ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(ab)": {}, + } + `); }); test('edit child with moving it', () => { @@ -5788,7 +5855,11 @@ describe('joins with compound join keys', () => { ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(ab)": {}, + } + `); }); }); @@ -5968,7 +6039,11 @@ suite('test overlay on many:one pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushesWithFetch).toMatchInlineSnapshot(` [ { @@ -6252,7 +6327,11 @@ suite('test overlay on many:one pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); // TODO: there is a bug here, the following // should be in the first fetch... we need to include /** @@ -6562,7 +6641,11 @@ suite('test overlay on many:one pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(owner)": {}, + } + `); expect(pushesWithFetch).toMatchInlineSnapshot(` [ { @@ -7112,7 +7195,7 @@ suite('test overlay on many:one pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(); expect(pushesWithFetch).toMatchInlineSnapshot(` [ { @@ -7790,7 +7873,11 @@ suite('test overlay on many:many (no junction) pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(ownerByName)": {}, + } + `); expect(pushesWithFetch).toMatchInlineSnapshot(` [ { @@ -8213,7 +8300,11 @@ suite('test overlay on many:many (no junction) pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(ownerByName)": {}, + } + `); expect(pushesWithFetch).toMatchInlineSnapshot(` [ { @@ -8645,7 +8736,11 @@ suite('test overlay on many:many (no junction) pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(` + { + ":flipped-join(ownerByName)": {}, + } + `); // TODO double check this test result expect(pushesWithFetch).toMatchInlineSnapshot(` [ @@ -9356,7 +9451,7 @@ suite('test overlay on many:many (no junction) pushes', () => { }, ] `); - expect(actualStorage).toMatchInlineSnapshot(`{}`); + expect(actualStorage).toMatchInlineSnapshot(); expect(pushesWithFetch).toMatchInlineSnapshot(` [ { diff --git a/packages/zql/src/ivm/flipped-join.ts b/packages/zql/src/ivm/flipped-join.ts index 192b905600..1457f7d6f4 100644 --- a/packages/zql/src/ivm/flipped-join.ts +++ b/packages/zql/src/ivm/flipped-join.ts @@ -8,8 +8,10 @@ import type {Change} from './change.ts'; import {constraintsAreCompatible, type Constraint} from './constraint.ts'; import type {Node} from './data.ts'; import { + generateParentNodesForChildRow, generateWithOverlay, isJoinMatch, + KeySet, rowEqualsForCompoundKey, type JoinChangeOverlay, } from './join-utils.ts'; @@ -18,6 +20,7 @@ import { type FetchRequest, type Input, type Output, + type Storage, } from './operator.ts'; import type {SourceSchema} from './schema.ts'; import {first, type Stream} from './stream.ts'; @@ -25,10 +28,11 @@ import {first, type Stream} from './stream.ts'; type Args = { parent: Input; child: Input; + storage: Storage; // The nth key in childKey corresponds to the nth key in parentKey. parentKey: CompoundKey; childKey: CompoundKey; - + partitionKey: CompoundKey | undefined; relationshipName: string; hidden: boolean; system: System; @@ -49,6 +53,7 @@ export class FlippedJoin implements Input { readonly #childKey: CompoundKey; readonly #relationshipName: string; readonly #schema: SourceSchema; + readonly #partitionKeySet: KeySet | undefined; #output: Output = throwOutput; @@ -57,8 +62,10 @@ export class FlippedJoin implements Input { constructor({ parent, child, + storage, parentKey, childKey, + partitionKey, relationshipName, hidden, system, @@ -88,6 +95,16 @@ export class FlippedJoin implements Input { }, }; + this.#partitionKeySet = partitionKey + ? new KeySet( + storage, + 'partition', + this.#parentKey, + parentSchema.primaryKey, + partitionKey, + ) + : undefined; + parent.setOutput({ push: (change: Change) => this.#pushParent(change), }); @@ -214,9 +231,13 @@ export class FlippedJoin implements Input { relatedChildNodes.push(childNodes[minParentNodeChildIndex]); const iter = parentIterators[minParentNodeChildIndex]; const result = iter.next(); - nextParentNodes[minParentNodeChildIndex] = result.done - ? null - : result.value; + if (result.done) { + nextParentNodes[minParentNodeChildIndex] = null; + } else { + const node: Node = result.value; + nextParentNodes[minParentNodeChildIndex] = node; + this.#partitionKeySet?.add(node.row); + } } let overlaidRelatedChildNodes = relatedChildNodes; if ( @@ -300,15 +321,14 @@ export class FlippedJoin implements Input { position: undefined, }; try { - const parentNodeStream = this.#parent.fetch({ - constraint: Object.fromEntries( - this.#parentKey.map((key, i) => [ - key, - change.node.row[this.#childKey[i]], - ]), - ), - }); - for (const parentNode of parentNodeStream) { + const parentNodes: Stream = generateParentNodesForChildRow( + this.#parentKey, + this.#childKey, + this.#partitionKeySet, + this.#parent, + change.node.row, + ); + for (const parentNode of parentNodes) { this.#inprogressChildChange = { change, position: parentNode.row, @@ -412,6 +432,13 @@ export class FlippedJoin implements Input { }, }); + if (change.type === 'add') { + this.#partitionKeySet?.add(change.node.row); + } + if (change.type === 'remove') { + this.#partitionKeySet?.delete(change.node.row); + } + // If no related child don't push as this is an inner join. if (first(childNodeStream(change.node)()) === undefined) { return; diff --git a/packages/zql/src/ivm/join-utils.test.ts b/packages/zql/src/ivm/join-utils.test.ts new file mode 100644 index 0000000000..c3bd14b36f --- /dev/null +++ b/packages/zql/src/ivm/join-utils.test.ts @@ -0,0 +1,201 @@ +import {beforeEach, describe, expect, test} from 'vitest'; +import {MemoryStorage} from './memory-storage.ts'; +import {KeySet} from './join-utils.ts'; +import type {CompoundKey} from '../../../zero-protocol/src/ast.ts'; +import type {Row} from '../../../zero-protocol/src/data.ts'; + +describe('KeySet', () => { + let storage: MemoryStorage; + + beforeEach(() => { + storage = new MemoryStorage(); + }); + + describe('with valueKey', () => { + let keySet: KeySet; + const setName = 'userPosts'; + const setKey: CompoundKey = ['userId']; + const primaryKey: CompoundKey = ['postId']; + const valueKey: CompoundKey = ['postTitle', 'category']; + + // Test data + const row1: Row = { + userId: 'u1', + postId: 'p1', + postTitle: 'Hello', + category: 'Tech', + }; + const row2: Row = { + userId: 'u1', + postId: 'p2', + postTitle: 'World', + category: 'Life', + }; + const row3: Row = { + userId: 'u2', + postId: 'p3', + postTitle: 'Other', + category: 'Misc', + }; + const row4: Row = { + userId: 'u1', + postId: 'p4', + postTitle: 'Hello', + category: 'Tech', + }; + + beforeEach(() => { + keySet = new KeySet(storage, setName, setKey, primaryKey, valueKey); + }); + + test('should add a row by constructing the correct key', () => { + keySet.add(row1); + expect(storage.cloneData()).toMatchInlineSnapshot(` + { + ""userPosts","[\\"u1\\"]","[\\"Hello\\",\\"Tech\\"]","[\\"p1\\"]",": true, + } + `); + }); + + test('should delete a row by constructing the correct key', () => { + keySet.add(row1); + expect(storage.cloneData()).toMatchInlineSnapshot(` + { + ""userPosts","[\\"u1\\"]","[\\"Hello\\",\\"Tech\\"]","[\\"p1\\"]",": true, + } + `); + + keySet.delete(row1); + expect(storage.cloneData()).toMatchInlineSnapshot(`{}`); + }); + + test('should correctly report if a set is empty', () => { + // Check set "u1" + expect(keySet.isEmpty({userId: 'u1'})).toBe(true); + keySet.add(row1); + expect(keySet.isEmpty({userId: 'u1'})).toBe(false); + + // Check set "u2" (should still be empty) + expect(keySet.isEmpty({userId: 'u2'})).toBe(true); + keySet.add(row3); + expect(keySet.isEmpty({userId: 'u2'})).toBe(false); + + // Deleting row 1 should not empty set "u1" if row 2 is present + keySet.add(row2); + keySet.delete(row1); + expect(keySet.isEmpty({userId: 'u1'})).toBe(false); + + // Deleting row 2 should now empty set "u1" + keySet.delete(row2); + expect(keySet.isEmpty({userId: 'u1'})).toBe(true); + }); + + test('should get unique values for a set, respecting deduplication', () => { + keySet.add(row1); // "u1" -> { Hello, Tech } (via p1) + keySet.add(row2); // "u1" -> { World, Life } (via p2) + keySet.add(row3); // "u2" -> { Other, Misc } (via p3) + keySet.add(row4); // "u1" -> { Hello, Tech } (via p4) + + // Check the results. Should be sorted by key, then deduplicated. + // key1 and key4 have the same value, should only appear once. + // key2 has a different value. + expect(Array.from(keySet.getValues({userId: 'u1'}))).toEqual([ + {postTitle: 'Hello', category: 'Tech'}, + {postTitle: 'World', category: 'Life'}, + ]); + expect(Array.from(keySet.getValues({userId: 'u2'}))).toEqual([ + {postTitle: 'Other', category: 'Misc'}, + ]); + expect(Array.from(keySet.getValues({userId: 'u3'}))).toEqual([]); + + keySet.delete(row4); + expect(Array.from(keySet.getValues({userId: 'u1'}))).toEqual([ + {postTitle: 'Hello', category: 'Tech'}, + {postTitle: 'World', category: 'Life'}, + ]); + expect(Array.from(keySet.getValues({userId: 'u2'}))).toEqual([ + {postTitle: 'Other', category: 'Misc'}, + ]); + expect(Array.from(keySet.getValues({userId: 'u3'}))).toEqual([]); + + keySet.delete(row3); + expect(Array.from(keySet.getValues({userId: 'u1'}))).toEqual([ + {postTitle: 'Hello', category: 'Tech'}, + {postTitle: 'World', category: 'Life'}, + ]); + expect(Array.from(keySet.getValues({userId: 'u2'}))).toEqual([]); + expect(Array.from(keySet.getValues({userId: 'u3'}))).toEqual([]); + + keySet.delete(row2); + expect(Array.from(keySet.getValues({userId: 'u1'}))).toEqual([ + {postTitle: 'Hello', category: 'Tech'}, + ]); + expect(Array.from(keySet.getValues({userId: 'u2'}))).toEqual([]); + expect(Array.from(keySet.getValues({userId: 'u3'}))).toEqual([]); + + keySet.delete(row1); + expect(Array.from(keySet.getValues({userId: 'u1'}))).toEqual([]); + expect(Array.from(keySet.getValues({userId: 'u2'}))).toEqual([]); + expect(Array.from(keySet.getValues({userId: 'u3'}))).toEqual([]); + }); + }); + + describe('without valueKey (undefined)', () => { + let keySet: KeySet; + const setName = 'userSet'; + const setKey: CompoundKey = ['userId']; + const primaryKey: CompoundKey = ['itemId']; + const valueKey = undefined; + + const row1: Row = {userId: 'u1', itemId: 'i1'}; + const row2: Row = {userId: 'u1', itemId: 'i2'}; + const row3: Row = {userId: 'u2', itemId: 'i3'}; + + beforeEach(() => { + keySet = new KeySet(storage, setName, setKey, primaryKey, valueKey); + }); + + test('should add a row with the correct (simpler) key', () => { + keySet.add(row1); + expect(storage.cloneData()).toMatchInlineSnapshot(` + { + ""userSet","[\\"u1\\"]","[\\"i1\\"]",": true, + } + `); + }); + + test('should delete a row with the correct (simpler) key', () => { + keySet.add(row1); + expect(storage.cloneData()).toMatchInlineSnapshot(` + { + ""userSet","[\\"u1\\"]","[\\"i1\\"]",": true, + } + `); + + keySet.delete(row1); + expect(storage.cloneData()).toMatchInlineSnapshot(`{}`); + }); + + test('should correctly report if a set is empty', () => { + expect(keySet.isEmpty({userId: 'u1'})).toBe(true); + keySet.add(row1); + expect(keySet.isEmpty({userId: 'u1'})).toBe(false); + + keySet.add(row2); + keySet.delete(row1); + expect(keySet.isEmpty({userId: 'u1'})).toBe(false); // row2 still exists + + keySet.delete(row2); + expect(keySet.isEmpty({userId: 'u1'})).toBe(true); + }); + + test('should return an empty iterable from getValues', () => { + keySet.add(row1); + keySet.add(row2); + keySet.add(row3); + + const values = Array.from(keySet.getValues({userId: 'u1'})); + expect(values).toEqual([]); + }); + }); +}); diff --git a/packages/zql/src/ivm/join-utils.ts b/packages/zql/src/ivm/join-utils.ts index 1faef9357b..c114d5ca59 100644 --- a/packages/zql/src/ivm/join-utils.ts +++ b/packages/zql/src/ivm/join-utils.ts @@ -1,10 +1,12 @@ -import type {Row} from '../../../zero-protocol/src/data.ts'; +import type {Row, Value} from '../../../zero-protocol/src/data.ts'; import type {Change} from './change.ts'; import type {SourceSchema} from './schema.ts'; -import type {Stream} from './stream.ts'; +import {take, type Stream} from './stream.ts'; import {compareValues, valuesEqual, type Node} from './data.ts'; import {assert} from '../../../shared/src/asserts.ts'; import type {CompoundKey} from '../../../zero-protocol/src/ast.ts'; +import {type Input, type Storage} from './operator.ts'; +import type {Constraint} from './constraint.ts'; export type JoinChangeOverlay = { change: Change; @@ -100,6 +102,34 @@ export function* generateWithOverlay( assert(applied); } +export function* generateParentNodesForChildRow( + parentKey: CompoundKey, + childKey: CompoundKey, + partitionKeySet: KeySet | undefined, + parent: Input, + childRow: Row, +): Stream { + const parentKeyConstraint = Object.fromEntries( + parentKey.map((key, i) => [key, childRow[childKey[i]]]), + ); + if (!partitionKeySet) { + yield* parent.fetch({ + constraint: parentKeyConstraint, + }); + return; + } + for (const partitionKeyConstraint of partitionKeySet.getValues( + parentKeyConstraint, + )) { + yield* parent.fetch({ + constraint: { + ...parentKeyConstraint, + ...partitionKeyConstraint, + }, + }); + } +} + export function rowEqualsForCompoundKey( a: Row, b: Row, @@ -126,3 +156,112 @@ export function isJoinMatch( } return true; } + +export class KeySet { + readonly #storage: Storage; + readonly #name: string; + readonly #setKey: CompoundKey; + readonly #primaryKey: CompoundKey; + readonly #valueKey: V; + + /*** + * @param storage The underlying key-value storage implementation. + * @param name A unique name for this set (used as the first part of the key). + * @param setKey The row properties that define the "partition" or "set". + * @param primaryKey The row properties that uniquely identify an entry *within* the set. + * @param valueKey (Optional) The row properties to be stored *in* the key, + * which can be retrieved with `getValues`. + */ + constructor( + storage: Storage, + name: string, + setKey: CompoundKey, + primaryKey: CompoundKey, + valueKey: V, + ) { + this.#storage = storage; + this.#name = name; + this.#setKey = setKey; + this.#primaryKey = primaryKey; + this.#valueKey = valueKey; + } + + add(row: Row): void { + this.#storage.set(this.#makeKeySetStorageKey(row), true); + } + + delete(row: Row): void { + this.#storage.del(this.#makeKeySetStorageKey(row)); + } + + *getValues(row: Row): Iterable<{ + readonly [key: string]: Value; + }> { + if (this.#valueKey === undefined) { + return; + } + const prefix = this.#makeKeySetStorageKeyPrefix(row); + let lastValuesStringified = undefined; + for (const [key] of this.#storage.scan({prefix})) { + const valuesStringified = JSON.parse( + '[' + key.substring(prefix.length, key.length - 1) + ']', + )[0]; + if (valuesStringified === lastValuesStringified) { + continue; + } + lastValuesStringified = valuesStringified; + const values = JSON.parse(valuesStringified); + yield Object.fromEntries( + this.#valueKey.map((key, i) => [key, values[i]]), + ); + } + } + + isEmpty(row: Row): boolean { + const prefix = this.#makeKeySetStorageKeyPrefix(row); + const iterator = this.#storage.scan({prefix})[Symbol.iterator](); + return !!iterator.next().done; + } + + #makeKeySetStorageKey(row: Row): string { + const setKeyValues: Value[] = this.#setKey.map(k => row[k]); + + const primaryKeyValues: Value[] = []; + for (const key of this.#primaryKey) { + primaryKeyValues.push(row[key]); + } + + if (this.#valueKey === undefined) { + return KeySet.#makeKeySetStorageKeyForValues(this.#name, [ + setKeyValues, + primaryKeyValues, + ]); + } + const valueKeyValues: Value[] = []; + for (const key of this.#valueKey) { + valueKeyValues.push(row[key]); + } + return KeySet.#makeKeySetStorageKeyForValues(this.#name, [ + setKeyValues, + valueKeyValues, + primaryKeyValues, + ]); + } + + #makeKeySetStorageKeyPrefix(row: Row): string { + return KeySet.#makeKeySetStorageKeyForValues(this.#name, [ + this.#setKey.map(k => row[k]), + ]); + } + + static #makeKeySetStorageKeyForValues( + setName: string, + valueArrays: readonly Value[][], + ): string { + const stringified = valueArrays.map(v => JSON.stringify(v)); + const json = JSON.stringify([setName, ...stringified]); + // Removes leading '[' and trailing ']' and appends a comma + // to create the prefix or full key + return json.substring(1, json.length - 1) + ','; + } +} diff --git a/packages/zql/src/ivm/join.fetch.test.ts b/packages/zql/src/ivm/join.fetch.test.ts index 196591d144..f81d11cf03 100644 --- a/packages/zql/src/ivm/join.fetch.test.ts +++ b/packages/zql/src/ivm/join.fetch.test.ts @@ -7,7 +7,7 @@ import type {PrimaryKey} from '../../../zero-protocol/src/primary-key.ts'; import type {SchemaValue} from '../../../zero-schema/src/table-schema.ts'; import {Catch, type CaughtNode} from './catch.ts'; import {SetOfConstraint} from './constraint.ts'; -import {Join, makeStorageKey, makeStorageKeyPrefix} from './join.ts'; +import {Join} from './join.ts'; import {MemoryStorage} from './memory-storage.ts'; import type {SourceSchema} from './schema.ts'; import {Snitch, type SnitchMessage} from './snitch.ts'; @@ -118,7 +118,7 @@ suite('fetch one:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, ] `); @@ -171,7 +171,7 @@ suite('fetch one:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, ] `); @@ -216,7 +216,7 @@ suite('fetch one:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, ] `); @@ -275,7 +275,7 @@ suite('fetch one:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, ] `); @@ -375,8 +375,8 @@ suite('fetch one:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -462,7 +462,7 @@ suite('fetch many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","u1","i1",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, }, ] `); @@ -538,7 +538,7 @@ suite('fetch many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","u1","i1",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, }, ] `); @@ -622,8 +622,8 @@ suite('fetch many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, }, ] `); @@ -707,8 +707,8 @@ suite('fetch many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","u1","i1",": true, - ""pKeySet","u2","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -848,7 +848,7 @@ suite('fetch one:many:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, {}, ] @@ -925,10 +925,10 @@ suite('fetch one:many:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, { - ""pKeySet","c1","c1",": true, + ""primary","[\\"c1\\"]","[\\"c1\\"]",": true, }, ] `); @@ -1142,14 +1142,14 @@ suite('fetch one:many:many', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","c1","c1",": true, - ""pKeySet","c2","c2",": true, - ""pKeySet","c3","c3",": true, - ""pKeySet","c4","c4",": true, + ""primary","[\\"c1\\"]","[\\"c1\\"]",": true, + ""primary","[\\"c2\\"]","[\\"c2\\"]",": true, + ""primary","[\\"c3\\"]","[\\"c3\\"]",": true, + ""primary","[\\"c4\\"]","[\\"c4\\"]",": true, }, ] `); @@ -1276,7 +1276,7 @@ suite('fetch one:many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, {}, ] @@ -1342,10 +1342,10 @@ suite('fetch one:many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, { - ""pKeySet","l1","i1","l1",": true, + ""primary","[\\"l1\\"]","[\\"i1\\",\\"l1\\"]",": true, }, ] `); @@ -1417,10 +1417,10 @@ suite('fetch one:many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, { - ""pKeySet","l1","i1","l1",": true, + ""primary","[\\"l1\\"]","[\\"i1\\",\\"l1\\"]",": true, }, ] `); @@ -1524,11 +1524,11 @@ suite('fetch one:many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, { - ""pKeySet","l1","i1","l1",": true, - ""pKeySet","l2","i1","l2",": true, + ""primary","[\\"l1\\"]","[\\"i1\\",\\"l1\\"]",": true, + ""primary","[\\"l2\\"]","[\\"i1\\",\\"l2\\"]",": true, }, ] `); @@ -1702,14 +1702,14 @@ suite('fetch one:many:one', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","l1","i1","l1",": true, - ""pKeySet","l1","i2","l1",": true, - ""pKeySet","l2","i1","l2",": true, - ""pKeySet","l2","i2","l2",": true, + ""primary","[\\"l1\\"]","[\\"i1\\",\\"l1\\"]",": true, + ""primary","[\\"l1\\"]","[\\"i2\\",\\"l1\\"]",": true, + ""primary","[\\"l2\\"]","[\\"i1\\",\\"l2\\"]",": true, + ""primary","[\\"l2\\"]","[\\"i2\\",\\"l2\\"]",": true, }, ] `); @@ -1831,7 +1831,7 @@ suite('compound join keys', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet",1,2,0,": true, + ""primary","[1,2]","[0]",": true, }, ] `); @@ -1890,7 +1890,7 @@ suite('compound join keys', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet",1,2,0,": true, + ""primary","[1,2]","[0]",": true, }, ] `); @@ -1940,7 +1940,7 @@ suite('compound join keys', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet",1,2,0,": true, + ""primary","[1,2]","[0]",": true, }, ] `); @@ -2005,7 +2005,7 @@ suite('compound join keys', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet",1,2,0,": true, + ""primary","[1,2]","[0]",": true, }, ] `); @@ -2124,8 +2124,8 @@ suite('compound join keys', () => { expect(results.storage).toMatchInlineSnapshot(` [ { - ""pKeySet",1,2,0,": true, - ""pKeySet",4,5,1,": true, + ""primary","[1,2]","[0]",": true, + ""primary","[4,5]","[1]",": true, }, ] `); @@ -2174,6 +2174,7 @@ function fetchTest(t: FetchTest): FetchTestResults { const join = new Join({ parent, child, + partitionKey: undefined, storage, ...info, hidden: false, @@ -2285,41 +2286,3 @@ type FetchTestResults = { hydrate: CaughtNode[]; storage: Record[]; }; - -test('createPrimaryKeySetStorageKey', () => { - const row123 = {a: 123, b: true, id: 'id1'}; - const row1234 = {a: 1234, b: true, id: 'id1'}; - const k123 = makeStorageKey(['a'], ['id'], row123); - const kp123 = makeStorageKeyPrefix(row123, ['a']); - const k1234 = makeStorageKey(['a'], ['id'], row1234); - const kp1234 = makeStorageKeyPrefix(row1234, ['a']); - - expect(k123).toEqual('"pKeySet",123,"id1",'); - expect(kp123).toEqual('"pKeySet",123,'); - expect(k123.startsWith(kp123)).true; - - expect(k1234).toEqual('"pKeySet",1234,"id1",'); - expect(kp1234).toEqual('"pKeySet",1234,'); - expect(k1234.startsWith(kp1234)).true; - - expect(k123.startsWith(kp1234)).false; - expect(k1234.startsWith(kp123)).false; - - const row456 = {a: 456, b: true, id: 'id1', id2: 'id2'}; - const row4567 = {a: 4567, b: true, id: 'id1', id2: 'id2'}; - const k456 = makeStorageKey(['b', 'a'], ['id', 'id2'], row456); - const kp456 = makeStorageKeyPrefix(row456, ['b', 'a']); - const k4567 = makeStorageKey(['b', 'a'], ['id', 'id2'], row4567); - const kp4567 = makeStorageKeyPrefix(row4567, ['b', 'a']); - - expect(k456).toEqual('"pKeySet",true,456,"id1","id2",'); - expect(kp456).toEqual('"pKeySet",true,456,'); - expect(k456.startsWith(kp456)).true; - - expect(k4567).toEqual('"pKeySet",true,4567,"id1","id2",'); - expect(kp4567).toEqual('"pKeySet",true,4567,'); - expect(k4567.startsWith(kp4567)).true; - - expect(k456.startsWith(kp4567)).false; - expect(k4567.startsWith(kp456)).false; -}); diff --git a/packages/zql/src/ivm/join.push.test.ts b/packages/zql/src/ivm/join.push.test.ts index 67a3bbad07..881c2b27f9 100644 --- a/packages/zql/src/ivm/join.push.test.ts +++ b/packages/zql/src/ivm/join.push.test.ts @@ -226,7 +226,7 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -326,7 +326,7 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -419,7 +419,7 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i2","i2",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, } `); @@ -511,7 +511,7 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -588,7 +588,7 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -1150,7 +1150,7 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -1275,7 +1275,7 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -1452,8 +1452,8 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, } `); @@ -1632,8 +1632,8 @@ suite('push one:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(comments)": { - ""pKeySet","i2","i2",": true, - ""pKeySet","i3","i3",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, + ""primary","[\\"i3\\"]","[\\"i3\\"]",": true, }, } `); @@ -1793,7 +1793,7 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u1","i1",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, }, } `); @@ -1882,7 +1882,7 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u2","i1",": true, + ""primary","[\\"u2\\"]","[\\"i1\\"]",": true, }, } `); @@ -1972,7 +1972,7 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u1","i1",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, }, } `); @@ -2098,8 +2098,8 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, }, } `); @@ -2300,8 +2300,8 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, }, } `); @@ -2617,12 +2617,14 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".issues:join(comments)": { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""partition","[\\"i1\\"]","[\\"u1\\"]","[\\"i1\\"]",": true, + ""partition","[\\"i2\\"]","[\\"u1\\"]","[\\"i2\\"]",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, ":join(issues)": { - ""pKeySet","u1","u1",": true, - ""pKeySet","u2","u2",": true, + ""primary","[\\"u1\\"]","[\\"u1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"u2\\"]",": true, }, } `); @@ -2768,8 +2770,8 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, }, } `); @@ -2903,8 +2905,8 @@ suite('push many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, }, } `); @@ -3058,6 +3060,7 @@ suite('push one:many:many', () => { { "constraint": { "id": "c1", + "issueID": "i1", }, }, ], @@ -3139,10 +3142,11 @@ suite('push one:many:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".comments:join(revisions)": { - ""pKeySet","c1","c1",": true, + ""partition","[\\"c1\\"]","[\\"i1\\"]","[\\"c1\\"]",": true, + ""primary","[\\"c1\\"]","[\\"c1\\"]",": true, }, ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -3280,10 +3284,11 @@ suite('push one:many:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".comments:join(revisions)": { - ""pKeySet","c1","c1",": true, + ""partition","[\\"c1\\"]","[\\"i1\\"]","[\\"c1\\"]",": true, + ""primary","[\\"c1\\"]","[\\"c1\\"]",": true, }, ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -3411,10 +3416,11 @@ suite('push one:many:many', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".comments:join(revisions)": { - ""pKeySet","c1","c1",": true, + ""partition","[\\"c1\\"]","[\\"i1\\"]","[\\"c1\\"]",": true, + ""primary","[\\"c1\\"]","[\\"c1\\"]",": true, }, ":join(comments)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -3661,6 +3667,7 @@ suite('push one:many:one', () => { "fetch", { "constraint": { + "issueID": "i1", "labelID": "l1", }, }, @@ -3738,10 +3745,11 @@ suite('push one:many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".issueLabels:join(labels)": { - ""pKeySet","l1","i1","l1",": true, + ""partition","[\\"l1\\"]","[\\"i1\\"]","[\\"i1\\",\\"l1\\"]",": true, + ""primary","[\\"l1\\"]","[\\"i1\\",\\"l1\\"]",": true, }, ":join(issueLabels)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -3877,10 +3885,11 @@ suite('push one:many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".issueLabels:join(labels)": { - ""pKeySet","l1","i1","l1",": true, + ""partition","[\\"l1\\"]","[\\"i1\\"]","[\\"i1\\",\\"l1\\"]",": true, + ""primary","[\\"l1\\"]","[\\"i1\\",\\"l1\\"]",": true, }, ":join(issueLabels)": { - ""pKeySet","i1","i1",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, }, } `); @@ -3951,6 +3960,7 @@ suite('push one:many:one', () => { "fetch", { "constraint": { + "issueID": "i1", "labelID": "l1", }, }, @@ -4004,6 +4014,16 @@ suite('push one:many:one', () => { "type": "child", }, ], + [ + ".issueLabels:source(issueLabel)", + "fetch", + { + "constraint": { + "issueID": "i2", + "labelID": "l1", + }, + }, + ], [ ".issueLabels:join(labels)", "push", @@ -4092,12 +4112,14 @@ suite('push one:many:one', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".issueLabels:join(labels)": { - ""pKeySet","l1","i1","l1",": true, - ""pKeySet","l1","i2","l1",": true, + ""partition","[\\"l1\\"]","[\\"i1\\"]","[\\"i1\\",\\"l1\\"]",": true, + ""partition","[\\"l1\\"]","[\\"i2\\"]","[\\"i2\\",\\"l1\\"]",": true, + ""primary","[\\"l1\\"]","[\\"i1\\",\\"l1\\"]",": true, + ""primary","[\\"l1\\"]","[\\"i2\\",\\"l1\\"]",": true, }, ":join(issueLabels)": { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, } `); @@ -4503,12 +4525,12 @@ describe('edit assignee', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(assignee)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u2","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, }, ":join(creator)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u2","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, }, } `); @@ -4868,12 +4890,12 @@ describe('edit assignee', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(assignee)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u2","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, }, ":join(creator)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u2","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, }, } `); @@ -5135,12 +5157,12 @@ describe('edit assignee', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(assignee)": { - ""pKeySet","u2","i2",": true, - ""pKeySet",null,"i1",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, + ""primary","[null]","[\\"i1\\"]",": true, }, ":join(creator)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u2","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, }, } `); @@ -5488,12 +5510,12 @@ describe('edit assignee', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(assignee)": { - ""pKeySet","u2","i2",": true, - ""pKeySet",null,"i1",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, + ""primary","[null]","[\\"i1\\"]",": true, }, ":join(creator)": { - ""pKeySet","u1","i1",": true, - ""pKeySet","u2","i2",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i2\\"]",": true, }, } `); @@ -5768,9 +5790,9 @@ describe('joins with compound join keys', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(ab)": { - ""pKeySet",1,2,0,": true, - ""pKeySet",4,5,1,": true, - ""pKeySet",7,8,2,": true, + ""primary","[1,2]","[0]",": true, + ""primary","[4,5]","[1]",": true, + ""primary","[7,8]","[2]",": true, }, } `); @@ -5897,8 +5919,8 @@ describe('joins with compound join keys', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(ab)": { - ""pKeySet",1,2,0,": true, - ""pKeySet",4,5,1,": true, + ""primary","[1,2]","[0]",": true, + ""primary","[4,5]","[1]",": true, }, } `); @@ -6077,10 +6099,10 @@ suite('test overlay on many:one pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u0","i0",": true, - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, - ""pKeySet","u2","i3",": true, + ""primary","[\\"u0\\"]","[\\"i0\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i3\\"]",": true, }, } `); @@ -6387,10 +6409,10 @@ suite('test overlay on many:one pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u0","i0",": true, - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, - ""pKeySet","u2","i3",": true, + ""primary","[\\"u0\\"]","[\\"i0\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i3\\"]",": true, }, } `); @@ -6710,10 +6732,10 @@ suite('test overlay on many:one pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(owner)": { - ""pKeySet","u0","i0",": true, - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, - ""pKeySet","u2","i3",": true, + ""primary","[\\"u0\\"]","[\\"i0\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i3\\"]",": true, }, } `); @@ -7032,6 +7054,7 @@ suite('test overlay on many:one pushes', () => { "fetch", { "constraint": { + "id": "u0", "stateID": "s0", }, }, @@ -7098,6 +7121,16 @@ suite('test overlay on many:one pushes', () => { "type": "child", }, ], + [ + ".owner:source(user)", + "fetch", + { + "constraint": { + "id": "u1", + "stateID": "s0", + }, + }, + ], [ ".owner:join(state)", "push", @@ -7263,15 +7296,18 @@ suite('test overlay on many:one pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".owner:join(state)": { - ""pKeySet","s0","u0",": true, - ""pKeySet","s0","u1",": true, - ""pKeySet","s1","u2",": true, + ""partition","[\\"s0\\"]","[\\"u0\\"]","[\\"u0\\"]",": true, + ""partition","[\\"s0\\"]","[\\"u1\\"]","[\\"u1\\"]",": true, + ""partition","[\\"s1\\"]","[\\"u2\\"]","[\\"u2\\"]",": true, + ""primary","[\\"s0\\"]","[\\"u0\\"]",": true, + ""primary","[\\"s0\\"]","[\\"u1\\"]",": true, + ""primary","[\\"s1\\"]","[\\"u2\\"]",": true, }, ":join(owner)": { - ""pKeySet","u0","i0",": true, - ""pKeySet","u1","i1",": true, - ""pKeySet","u1","i2",": true, - ""pKeySet","u2","i3",": true, + ""primary","[\\"u0\\"]","[\\"i0\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"u1\\"]","[\\"i2\\"]",": true, + ""primary","[\\"u2\\"]","[\\"i3\\"]",": true, }, } `); @@ -7943,10 +7979,10 @@ suite('test overlay on many:many (no junction) pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(ownerByName)": { - ""pKeySet","Aaron","i1",": true, - ""pKeySet","Aaron","i2",": true, - ""pKeySet","Arv","i3",": true, - ""pKeySet","Fritz","i0",": true, + ""primary","[\\"Aaron\\"]","[\\"i1\\"]",": true, + ""primary","[\\"Aaron\\"]","[\\"i2\\"]",": true, + ""primary","[\\"Arv\\"]","[\\"i3\\"]",": true, + ""primary","[\\"Fritz\\"]","[\\"i0\\"]",": true, }, } `); @@ -8366,10 +8402,10 @@ suite('test overlay on many:many (no junction) pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(ownerByName)": { - ""pKeySet","Aaron","i1",": true, - ""pKeySet","Aaron","i2",": true, - ""pKeySet","Arv","i3",": true, - ""pKeySet","Fritz","i0",": true, + ""primary","[\\"Aaron\\"]","[\\"i1\\"]",": true, + ""primary","[\\"Aaron\\"]","[\\"i2\\"]",": true, + ""primary","[\\"Arv\\"]","[\\"i3\\"]",": true, + ""primary","[\\"Fritz\\"]","[\\"i0\\"]",": true, }, } `); @@ -8807,10 +8843,10 @@ suite('test overlay on many:many (no junction) pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ":join(ownerByName)": { - ""pKeySet","Aaron","i1",": true, - ""pKeySet","Aaron","i2",": true, - ""pKeySet","Arv","i3",": true, - ""pKeySet","Fritz","i0",": true, + ""primary","[\\"Aaron\\"]","[\\"i1\\"]",": true, + ""primary","[\\"Aaron\\"]","[\\"i2\\"]",": true, + ""primary","[\\"Arv\\"]","[\\"i3\\"]",": true, + ""primary","[\\"Fritz\\"]","[\\"i0\\"]",": true, }, } `); @@ -9207,6 +9243,7 @@ suite('test overlay on many:many (no junction) pushes', () => { "fetch", { "constraint": { + "name": "Aaron", "stateID": "s0", }, }, @@ -9520,17 +9557,22 @@ suite('test overlay on many:many (no junction) pushes', () => { expect(actualStorage).toMatchInlineSnapshot(` { ".ownerByName:join(state)": { - ""pKeySet","s0","u2",": true, - ""pKeySet","s0","u3",": true, - ""pKeySet","s1","u0",": true, - ""pKeySet","s1","u1",": true, - ""pKeySet","s1","u4",": true, + ""partition","[\\"s0\\"]","[\\"Aaron\\"]","[\\"u2\\"]",": true, + ""partition","[\\"s0\\"]","[\\"Aaron\\"]","[\\"u3\\"]",": true, + ""partition","[\\"s1\\"]","[\\"Aaron\\"]","[\\"u1\\"]",": true, + ""partition","[\\"s1\\"]","[\\"Arv\\"]","[\\"u4\\"]",": true, + ""partition","[\\"s1\\"]","[\\"Fritz\\"]","[\\"u0\\"]",": true, + ""primary","[\\"s0\\"]","[\\"u2\\"]",": true, + ""primary","[\\"s0\\"]","[\\"u3\\"]",": true, + ""primary","[\\"s1\\"]","[\\"u0\\"]",": true, + ""primary","[\\"s1\\"]","[\\"u1\\"]",": true, + ""primary","[\\"s1\\"]","[\\"u4\\"]",": true, }, ":join(ownerByName)": { - ""pKeySet","Aaron","i1",": true, - ""pKeySet","Aaron","i2",": true, - ""pKeySet","Arv","i3",": true, - ""pKeySet","Fritz","i0",": true, + ""primary","[\\"Aaron\\"]","[\\"i1\\"]",": true, + ""primary","[\\"Aaron\\"]","[\\"i2\\"]",": true, + ""primary","[\\"Arv\\"]","[\\"i3\\"]",": true, + ""primary","[\\"Fritz\\"]","[\\"i0\\"]",": true, }, } `); diff --git a/packages/zql/src/ivm/join.sibling.test.ts b/packages/zql/src/ivm/join.sibling.test.ts index f1350cdedb..dca0b9a3af 100644 --- a/packages/zql/src/ivm/join.sibling.test.ts +++ b/packages/zql/src/ivm/join.sibling.test.ts @@ -113,14 +113,14 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, - ""pKeySet","i3","i3",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, + ""primary","[\\"i3\\"]","[\\"i3\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, - ""pKeySet","o2","i3",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i3\\"]",": true, }, ] `); @@ -204,12 +204,12 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -293,12 +293,12 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -382,12 +382,12 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -471,12 +471,12 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -580,12 +580,12 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -665,12 +665,12 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, }, ] `); @@ -759,12 +759,12 @@ suite('sibling relationships tests with issues, comments, and owners', () => { expect(storage).toMatchInlineSnapshot(` [ { - ""pKeySet","i1","i1",": true, - ""pKeySet","i2","i2",": true, + ""primary","[\\"i1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"i2\\"]","[\\"i2\\"]",": true, }, { - ""pKeySet","o1","i1",": true, - ""pKeySet","o2","i2",": true, + ""primary","[\\"o1\\"]","[\\"i1\\"]",": true, + ""primary","[\\"o2\\"]","[\\"i2\\"]",": true, }, ] `); diff --git a/packages/zql/src/ivm/join.ts b/packages/zql/src/ivm/join.ts index 0cf2940e5e..07b1d229d0 100644 --- a/packages/zql/src/ivm/join.ts +++ b/packages/zql/src/ivm/join.ts @@ -1,7 +1,6 @@ import {assert, unreachable} from '../../../shared/src/asserts.ts'; import type {CompoundKey, System} from '../../../zero-protocol/src/ast.ts'; -import type {Row, Value} from '../../../zero-protocol/src/data.ts'; -import type {PrimaryKey} from '../../../zero-protocol/src/primary-key.ts'; +import type {Row} from '../../../zero-protocol/src/data.ts'; import type {Change, ChildChange} from './change.ts'; import type {Node} from './data.ts'; import { @@ -9,6 +8,8 @@ import { isJoinMatch, rowEqualsForCompoundKey, type JoinChangeOverlay, + KeySet, + generateParentNodesForChildRow, } from './join-utils.ts'; import { throwOutput, @@ -18,7 +19,7 @@ import { type Storage, } from './operator.ts'; import type {SourceSchema} from './schema.ts'; -import {take, type Stream} from './stream.ts'; +import {type Stream} from './stream.ts'; type Args = { parent: Input; @@ -27,6 +28,7 @@ type Args = { // The nth key in parentKey corresponds to the nth key in childKey. parentKey: CompoundKey; childKey: CompoundKey; + partitionKey: CompoundKey | undefined; relationshipName: string; hidden: boolean; system: System; @@ -45,11 +47,12 @@ type Args = { export class Join implements Input { readonly #parent: Input; readonly #child: Input; - readonly #storage: Storage; readonly #parentKey: CompoundKey; readonly #childKey: CompoundKey; readonly #relationshipName: string; readonly #schema: SourceSchema; + readonly #primaryKeySet: KeySet; + readonly #partitionKeySet: KeySet | undefined; #output: Output = throwOutput; @@ -61,6 +64,7 @@ export class Join implements Input { storage, parentKey, childKey, + partitionKey, relationshipName, hidden, system, @@ -72,7 +76,6 @@ export class Join implements Input { ); this.#parent = parent; this.#child = child; - this.#storage = storage; this.#parentKey = parentKey; this.#childKey = childKey; this.#relationshipName = relationshipName; @@ -91,6 +94,23 @@ export class Join implements Input { }, }; + this.#primaryKeySet = new KeySet( + storage, + 'primary', + this.#parentKey, + parentSchema.primaryKey, + undefined, + ); + this.#partitionKeySet = partitionKey + ? new KeySet( + storage, + 'partition', + this.#parentKey, + parentSchema.primaryKey, + partitionKey, + ) + : undefined; + parent.setOutput({ push: (change: Change) => this.#pushParent(change), }); @@ -214,12 +234,13 @@ export class Join implements Input { position: undefined, }; try { - const parentNodes = this.#parent.fetch({ - constraint: Object.fromEntries( - this.#parentKey.map((key, i) => [key, childRow[this.#childKey[i]]]), - ), - }); - + const parentNodes: Stream = generateParentNodesForChildRow( + this.#parentKey, + this.#childKey, + this.#partitionKeySet, + this.#parent, + childRow, + ); for (const parentNode of parentNodes) { this.#inprogressChildChange.position = parentNode.row; const childChange: ChildChange = { @@ -276,22 +297,8 @@ export class Join implements Input { const childStream = () => { if (!storageUpdated) { if (mode === 'cleanup') { - this.#storage.del( - makeStorageKey( - this.#parentKey, - this.#parent.getSchema().primaryKey, - parentNodeRow, - ), - ); - const empty = - [ - ...take( - this.#storage.scan({ - prefix: makeStorageKeyPrefix(parentNodeRow, this.#parentKey), - }), - 1, - ), - ].length === 0; + this.#primaryKeySet.delete(parentNodeRow); + const empty = this.#primaryKeySet.isEmpty(parentNodeRow); method = empty ? 'cleanup' : 'fetch'; } @@ -299,14 +306,7 @@ export class Join implements Input { // Defer the work to update storage until the child stream // is actually accessed if (mode === 'fetch') { - this.#storage.set( - makeStorageKey( - this.#parentKey, - this.#parent.getSchema().primaryKey, - parentNodeRow, - ), - true, - ); + this.#primaryKeySet.add(parentNodeRow); } } @@ -342,6 +342,15 @@ export class Join implements Input { return stream; }; + if (this.#partitionKeySet) { + if (mode === 'fetch') { + this.#partitionKeySet.add(parentNodeRow); + } else { + mode satisfies 'cleanup'; + this.#partitionKeySet.delete(parentNodeRow); + } + } + return { row: parentNodeRow, relationships: { @@ -353,30 +362,3 @@ export class Join implements Input { } type ProcessParentMode = 'fetch' | 'cleanup'; - -/** Exported for testing. */ -export function makeStorageKeyForValues(values: readonly Value[]): string { - const json = JSON.stringify(['pKeySet', ...values]); - return json.substring(1, json.length - 1) + ','; -} - -/** Exported for testing. */ -export function makeStorageKeyPrefix(row: Row, key: CompoundKey): string { - return makeStorageKeyForValues(key.map(k => row[k])); -} - -/** Exported for testing. - * This storage key tracks the primary keys seen for each unique - * value joined on. This is used to know when to cleanup a child's state. - */ -export function makeStorageKey( - key: CompoundKey, - primaryKey: PrimaryKey, - row: Row, -): string { - const values: Value[] = key.map(k => row[k]); - for (const key of primaryKey) { - values.push(row[key]); - } - return makeStorageKeyForValues(values); -} diff --git a/packages/zql/src/ivm/take.ts b/packages/zql/src/ivm/take.ts index fcaac99065..324607e237 100644 --- a/packages/zql/src/ivm/take.ts +++ b/packages/zql/src/ivm/take.ts @@ -1,8 +1,8 @@ import {assert, unreachable} from '../../../shared/src/asserts.ts'; import {hasOwn} from '../../../shared/src/has-own.ts'; import {must} from '../../../shared/src/must.ts'; +import type {CompoundKey} from '../../../zero-protocol/src/ast.ts'; import type {Row, Value} from '../../../zero-protocol/src/data.ts'; -import type {PrimaryKey} from '../../../zero-protocol/src/primary-key.ts'; import {assertOrderingIncludesPK} from '../builder/builder.ts'; import {type Change, type EditChange, type RemoveChange} from './change.ts'; import type {Constraint} from './constraint.ts'; @@ -33,7 +33,7 @@ interface TakeStorage { del(key: string): void; } -export type PartitionKey = PrimaryKey; +export type PartitionKey = CompoundKey; /** * The Take operator is for implementing limit queries. It takes the first n @@ -117,6 +117,9 @@ export class Take implements Operator { } return; } + if (this.#partitionKey) { + throw new Error('no partition!!!!!!!!' + JSON.stringify(req)); + } // There is a partition key, but the fetch is not constrained or constrained // on a different key. Thus we don't have a single take state to bound by. // This currently only happens with nested sub-queries