Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/core/src/storages/key_value_store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ export const maybeStringify = <T>(value: T, options: { contentType?: string }) =
export class KeyValueStore {
readonly id: string;
readonly name?: string;
readonly storageObject?: Record<string, unknown>;
private readonly client: KeyValueStoreClient;
private persistStateEventStarted = false;

Expand All @@ -121,6 +122,7 @@ export class KeyValueStore {
) {
this.id = options.id;
this.name = options.name;
this.storageObject = options.storageObject;
this.client = options.client.keyValueStore(this.id);
}

Expand Down Expand Up @@ -708,6 +710,7 @@ export interface KeyValueStoreOptions {
id: string;
name?: string;
client: StorageClient;
storageObject?: Record<string, unknown>;
}

export interface RecordOptions {
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/storages/storage_manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ export class StorageManager<T extends IStorage = IStorage> {
storage = new this.StorageConstructor({
id: storageObject.id,
name: storageObject.name,
storageObject,
client,
});

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"actorSpecification": 1,
"name": "test-kv-open-return-storage-object",
"version": "0.0",
"buildTag": "latest",
"env": null
}
7 changes: 7 additions & 0 deletions test/e2e/kv-open-return-storage-object/actor/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
.idea
.DS_Store
node_modules
package-lock.json
apify_storage
crawlee_storage
storage
16 changes: 16 additions & 0 deletions test/e2e/kv-open-return-storage-object/actor/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
FROM apify/actor-node:20-beta

COPY packages ./packages
COPY package*.json ./

RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
&& npm --version

COPY . ./
14 changes: 14 additions & 0 deletions test/e2e/kv-open-return-storage-object/actor/main.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { Actor, KeyValueStore } from 'apify';

const mainOptions = {
exit: Actor.isAtHome(),
storage:
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
? new (await import('@apify/storage-local')).ApifyStorageLocal()
: undefined,
};

await Actor.main(async () => {
const kv = await KeyValueStore.open();
kv.setValue('storageObject', { storeObject: kv.storageObject });
}, mainOptions);
25 changes: 25 additions & 0 deletions test/e2e/kv-open-return-storage-object/actor/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{
"name": "test-kv-open-return-storage-object",
"version": "0.0.1",
"description": "Key-Value Store - Return storage object on open",
"dependencies": {
"apify": "next",
"@apify/storage-local": "^2.1.3",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/core": "file:./packages/core",
"@crawlee/memory-storage": "file:./packages/memory-storage",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
},
"overrides": {
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
}
},
"scripts": {
"start": "node main.js"
},
"type": "module",
"license": "ISC"
}
25 changes: 25 additions & 0 deletions test/e2e/kv-open-return-storage-object/test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';

/* This test verifies that the storageObject is correctly returned when the KeyValueStore is opened.
* The storageObject is the result of the KeyValueStoreClient.get() method,
* containing properties such as name, id, and other custom attributes.
*/

const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);

const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);

await expect(items !== undefined, 'Key value store exists');

const item = items.find((kvItem) => kvItem.name === 'storageObject');

const parsed = JSON.parse(item.raw.toString());

await expect(
typeof parsed === 'object' && parsed !== null,
'Key-value contains key "storeObject" and it\'s value is a non-nullable object',
);
await expect(parsed.id !== null, 'storeObject contains id');
await expect(parsed.name !== null, 'storeObject contains name');
await expect(parsed.userId !== null, 'storeObject contains userId');
Loading