fix: indexeddb multimap dupes

yarn-project/kv-store/package.json

@@ -32,6 +32,7 @@
     "idb": "^8.0.0",
     "lmdb": "^3.2.0",
     "msgpackr": "^1.11.2",
+    "ohash": "^2.0.11",
     "ordered-binary": "^1.5.3"
   },
   "devDependencies": {

yarn-project/kv-store/src/indexeddb/array.ts

@@ -1,12 +1,14 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
 
 import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
  * A persistent array backed by IndexedDB.
  */
-export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
+export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T> {
   #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
   #rootDB: IDBPDatabase<AztecIDBSchema>;
   #container: string;
@@ -39,6 +41,7 @@ export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
     for (const val of vals) {
       await this.db.put({
         value: val,
+        hash: hash(val),
         container: this.#container,
         key: this.#name,
         keyCount: length + 1,
@@ -86,6 +89,7 @@ export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
 
     await this.db.put({
       value: val,
+      hash: hash(val),
       container: this.#container,
       key: this.#name,
       keyCount: index + 1,

yarn-project/kv-store/src/indexeddb/map.ts

@@ -1,13 +1,14 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
 
-import type { Key, Range } from '../interfaces/common.js';
+import type { Key, Range, Value } from '../interfaces/common.js';
 import type { AztecAsyncMap } from '../interfaces/map.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
  * A map backed by IndexedDB.
  */
-export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMap<K, V> {
+export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
   protected name: string;
   protected container: string;
 
@@ -41,6 +42,7 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMap<K, V>
   async set(key: K, val: V): Promise<void> {
     await this.db.put({
       value: val,
+      hash: hash(val),
       container: this.container,
       key: this.normalizeKey(key),
       keyCount: 1,

yarn-project/kv-store/src/indexeddb/multi_map.ts

@@ -1,20 +1,34 @@
-import type { Key } from '../interfaces/common.js';
+import { hash } from 'ohash';
+
+import type { Key, Value } from '../interfaces/common.js';
 import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
 import { IndexedDBAztecMap } from './map.js';
 
 /**
  * A multi map backed by IndexedDB.
  */
-export class IndexedDBAztecMultiMap<K extends Key, V>
+export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
   extends IndexedDBAztecMap<K, V>
   implements AztecAsyncMultiMap<K, V>
 {
   override async set(key: K, val: V): Promise<void> {
+    const exists = !!(await this.db
+      .index('hash')
+      .get(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      ));
+    if (exists) {
+      return;
+    }
     const count = await this.db
       .index('key')
       .count(IDBKeyRange.bound([this.container, this.normalizeKey(key)], [this.container, this.normalizeKey(key)]));
     await this.db.put({
       value: val,
+      hash: hash(val),
       container: this.container,
       key: this.normalizeKey(key),
       keyCount: count + 1,
@@ -36,18 +50,16 @@ export class IndexedDBAztecMultiMap<K extends Key, V>
   }
 
   async deleteValue(key: K, val: V): Promise<void> {
-    const index = this.db.index('keyCount');
-    const rangeQuery = IDBKeyRange.bound(
-      [this.container, this.normalizeKey(key), 0],
-      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
-      false,
-      false,
-    );
-    for await (const cursor of index.iterate(rangeQuery)) {
-      if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
-        await cursor.delete();
-        return;
-      }
+    const fullKey = await this.db
+      .index('hash')
+      .getKey(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      );
+    if (fullKey) {
+      await this.db.delete(fullKey);
     }
   }
 }

yarn-project/kv-store/src/indexeddb/singleton.ts

@@ -1,12 +1,14 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
 
+import type { Value } from '../interfaces/common.js';
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
  * Stores a single value in IndexedDB.
  */
-export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
+export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
   #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
   #rootDB: IDBPDatabase<AztecIDBSchema>;
   #container: string;
@@ -38,6 +40,7 @@ export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
       key: this.#slot,
       keyCount: 1,
       value: val,
+      hash: hash(val),
     });
     return result !== undefined;
   }

yarn-project/kv-store/src/indexeddb/store.ts

@@ -1,9 +1,10 @@
 import type { Logger } from '@aztec/foundation/log';
 
 import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
+import type { NotUndefined } from 'object-hash';
 
 import type { AztecAsyncArray } from '../interfaces/array.js';
-import type { Key, StoreSize } from '../interfaces/common.js';
+import type { Key, StoreSize, Value } from '../interfaces/common.js';
 import type { AztecAsyncCounter } from '../interfaces/counter.js';
 import type { AztecAsyncMap } from '../interfaces/map.js';
 import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
@@ -16,13 +17,20 @@ import { IndexedDBAztecMultiMap } from './multi_map.js';
 import { IndexedDBAztecSet } from './set.js';
 import { IndexedDBAztecSingleton } from './singleton.js';
 
-export type StoredData<V> = { value: V; container: string; key: string; keyCount: number; slot: string };
+export type StoredData<V extends NotUndefined> = {
+  value: V;
+  container: string;
+  key: string;
+  keyCount: number;
+  slot: string;
+  hash: string;
+};
 
 export interface AztecIDBSchema extends DBSchema {
   data: {
     value: StoredData<any>;
     key: string;
-    indexes: { container: string; key: string; keyCount: number };
+    indexes: { container: string; key: string; keyCount: number; hash: string };
   };
 }
 
@@ -67,6 +75,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
 
         objectStore.createIndex('key', ['container', 'key'], { unique: false });
         objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: false });
+        objectStore.createIndex('hash', ['container', 'key', 'hash'], { unique: true });
       },
     });
 
@@ -79,7 +88,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param name - Name of the map
    * @returns A new AztecMap
    */
-  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
+  openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
     const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
     this.#containers.add(map);
     return map;
@@ -101,7 +110,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param name - Name of the map
    * @returns A new AztecMultiMap
    */
-  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
     const multimap = new IndexedDBAztecMultiMap<K, V>(this.#rootDB, name);
     this.#containers.add(multimap);
     return multimap;
@@ -116,7 +125,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param name - Name of the array
    * @returns A new AztecArray
    */
-  openArray<T>(name: string): AztecAsyncArray<T> {
+  openArray<T extends Value>(name: string): AztecAsyncArray<T> {
     const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
     this.#containers.add(array);
     return array;
@@ -127,7 +136,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param name - Name of the singleton
    * @returns A new AztecSingleton
    */
-  openSingleton<T>(name: string): AztecAsyncSingleton<T> {
+  openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
     const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
     this.#containers.add(singleton);
     return singleton;

yarn-project/kv-store/src/interfaces/array.ts

@@ -1,7 +1,9 @@
+import type { Value } from './common.js';
+
 /**
  * An array backed by a persistent store. Can not have any holes in it.
  */
-interface BaseAztecArray<T> {
+interface BaseAztecArray<T extends Value> {
   /**
    * Pushes values to the end of the array
    * @param vals - The values to push to the end of the array
@@ -27,7 +29,7 @@ interface BaseAztecArray<T> {
 /**
  * An array backed by a persistent store. Can not have any holes in it.
  */
-export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
+export interface AztecAsyncArray<T extends Value> extends BaseAztecArray<T> {
   /**
    * The size of the array
    */
@@ -58,7 +60,7 @@ export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
   [Symbol.asyncIterator](): AsyncIterableIterator<T>;
 }
 
-export interface AztecArray<T> extends BaseAztecArray<T> {
+export interface AztecArray<T extends Value> extends BaseAztecArray<T> {
   /**
    * The size of the array
    */

yarn-project/kv-store/src/interfaces/common.ts

@@ -3,6 +3,8 @@
  */
 export type Key = string | number | Array<string | number>;
 
+export type Value = NonNullable<any>;
+
 /**
  * A range of keys to iterate over.
  */

yarn-project/kv-store/src/interfaces/map.ts

@@ -1,9 +1,9 @@
-import type { Key, Range } from './common.js';
+import type { Key, Range, Value } from './common.js';
 
 /**
  * A map backed by a persistent store.
  */
-interface AztecBaseMap<K extends Key, V> {
+interface AztecBaseMap<K extends Key, V extends Value> {
   /**
    * Sets the value at the given key.
    * @param key - The key to set the value at
@@ -24,7 +24,7 @@ interface AztecBaseMap<K extends Key, V> {
    */
   delete(key: K): Promise<void>;
 }
-export interface AztecMap<K extends Key, V> extends AztecBaseMap<K, V> {
+export interface AztecMap<K extends Key, V extends Value> extends AztecBaseMap<K, V> {
   /**
    * Gets the value at the given key.
    * @param key - The key to get the value from
@@ -65,7 +65,7 @@ export interface AztecMap<K extends Key, V> extends AztecBaseMap<K, V> {
 /**
  * A map backed by a persistent store.
  */
-export interface AztecAsyncMap<K extends Key, V> extends AztecBaseMap<K, V> {
+export interface AztecAsyncMap<K extends Key, V extends Value> extends AztecBaseMap<K, V> {
   /**
    * Gets the value at the given key.
    * @param key - The key to get the value from

yarn-project/kv-store/src/interfaces/multi_map.ts

@@ -1,10 +1,10 @@
-import type { Key } from './common.js';
+import type { Key, Value } from './common.js';
 import type { AztecAsyncMap, AztecMap } from './map.js';
 
 /**
  * A map backed by a persistent store that can have multiple values for a single key.
  */
-export interface AztecMultiMap<K extends Key, V> extends AztecMap<K, V> {
+export interface AztecMultiMap<K extends Key, V extends Value> extends AztecMap<K, V> {
   /**
    * Gets all the values at the given key.
    * @param key - The key to get the values from
@@ -22,7 +22,7 @@ export interface AztecMultiMap<K extends Key, V> extends AztecMap<K, V> {
 /**
  * A map backed by a persistent store that can have multiple values for a single key.
  */
-export interface AztecAsyncMultiMap<K extends Key, V> extends AztecAsyncMap<K, V> {
+export interface AztecAsyncMultiMap<K extends Key, V extends Value> extends AztecAsyncMap<K, V> {
   /**
    * Gets all the values at the given key.
    * @param key - The key to get the values from

yarn-project/kv-store/src/interfaces/multi_map_test_suite.ts

@@ -116,6 +116,13 @@ export function describeAztecMultiMap(
       expect(await getValues('foo')).to.deep.equal(['bar', 'baz']);
     });
 
+    it('should ignore multiple identical values', async () => {
+      await multiMap.set('foo', 'bar');
+      await multiMap.set('foo', 'bar');
+
+      expect(await getValues('foo')).to.deep.equal(['bar']);
+    });
+
     it('should be able to delete individual values for a single key', async () => {
       await multiMap.set('foo', 'bar');
       await multiMap.set('foo', 'baz');