From 17241fe5fcfc7e31f3ffd1db56d52b3a33961d70 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:00:05 +0100 Subject: [PATCH 01/10] feat: sign public url --- package-lock.json | 193 +++++++++++++++++++++++--- packages/apify/package.json | 10 +- packages/apify/src/key_value_store.ts | 7 +- 3 files changed, 188 insertions(+), 22 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6c9facee4b..41fd0e8410 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,9 +44,9 @@ } }, "node_modules/@apify/consts": { - "version": "2.33.0", - "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.33.0.tgz", - "integrity": "sha512-Du4unQM2w1Hdt9ZT5AOMetP9jrTeYJelFRaIBDKNSF6ynZ+sbwBswHWAsxQfuy0gCHRftocexz5xNLzfV7tg4g==", + "version": "2.37.0", + "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.37.0.tgz", + "integrity": "sha512-GCK7wZcUPnHMUpoxd1tgbDgojdgcdxpnaF8UX5h5/hq5ZPdlik7EP2CkU6MpRfYxYl8T5LOsANUJvdB6bGthKA==", "license": "Apache-2.0" }, "node_modules/@apify/datastructures": { @@ -174,12 +174,12 @@ } }, "node_modules/@apify/log": { - "version": "2.5.9", - "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.9.tgz", - "integrity": "sha512-HVmRHPMacUiWA7HYqgIq9sJywFEsyexZBUazfwCvmrOSHIt8kRCpvMNAgm+aMDNNnNm6e+pyA2pBQST7xXCGWQ==", + "version": "2.5.13", + "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.13.tgz", + "integrity": "sha512-gxvWyD9JWBkLoTX7UfQ4s0F32/UfF4T8TG4jAl2CE0vNKr0COOJFkLmmyWnTNSVDL+yGC+LZp3mtbPnB+l6Sog==", "license": "Apache-2.0", "dependencies": { - "@apify/consts": "^2.33.0", + "@apify/consts": "^2.37.0", "ansi-colors": "^4.1.1" } }, @@ -225,13 +225,13 @@ "license": "Apache-2.0" }, "node_modules/@apify/utilities": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.10.10.tgz", - "integrity": "sha512-oJT2oRoTLJAhiTD/uQAQvPetLEiwkNKkhJSNkKi3TUsnxdRmHwxJz6S+evouOAt6Snipe9sb9PBune6DUAXGoA==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.13.1.tgz", + "integrity": "sha512-czQz2oANACVuwE7KrJKB8LrbhvMZyB1bvwO9jTL2SCmKQHxzY34XLQnSfblq/qsLWrwKynKxWXSvAVRZH2oIlA==", "license": "Apache-2.0", "dependencies": { - "@apify/consts": "^2.33.0", - "@apify/log": "^2.5.9" + "@apify/consts": "^2.37.0", + "@apify/log": "^2.5.13" } }, "node_modules/@babel/code-frame": { @@ -3191,6 +3191,24 @@ "node": ">=14.16" } }, + "node_modules/@tokenizer/inflate": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.6.tgz", + "integrity": "sha512-SdR/i05U7Xhnsq36iyIq/ZiGGw4PKzw4ww3bOq80Pjj4wyXpqyTcgrgdDdGlcatnlvzNJx8CQw3hp6QZvkUwhA==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.7", + "fflate": "^0.8.2", + "token-types": "^6.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/@tokenizer/token": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", @@ -8234,6 +8252,12 @@ "pend": "~1.2.0" } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, "node_modules/figlet": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/figlet/-/figlet-1.8.0.tgz", @@ -18408,10 +18432,10 @@ "@apify/input_secrets": "^1.1.40", "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.9.3", - "@crawlee/core": "^3.9.0", - "@crawlee/types": "^3.9.0", - "@crawlee/utils": "^3.9.0", + "@apify/utilities": "^2.13.0", + "@crawlee/core": "^3.12.3-beta.12", + "@crawlee/types": "^3.12.3-beta.12", + "@crawlee/utils": "^3.12.3-beta.12", "apify-client": "^2.11.1", "fs-extra": "^11.2.0", "ow": "^0.28.2", @@ -18423,6 +18447,143 @@ "node": ">=16.0.0" } }, + "packages/apify/node_modules/@crawlee/core": { + "version": "3.12.3-beta.12", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.12.3-beta.12.tgz", + "integrity": "sha512-Roz4b1Q/h83r3S8Sx3jCXxXGp+9pCLwpUUpQyAtt6InYr3i8yFBBMJis0PS3hbaL1pn5dbzzIuREHZs4aBEXCg==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.20.0", + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@apify/pseudo_url": "^2.0.30", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/memory-storage": "3.12.3-beta.12", + "@crawlee/types": "3.12.3-beta.12", + "@crawlee/utils": "3.12.3-beta.12", + "@sapphire/async-queue": "^1.5.1", + "@vladfrangu/async_event_emitter": "^2.2.2", + "csv-stringify": "^6.2.0", + "fs-extra": "^11.0.0", + "got-scraping": "^4.0.0", + "json5": "^2.2.3", + "minimatch": "^9.0.0", + "ow": "^0.28.1", + "stream-json": "^1.8.0", + "tldts": "^6.0.0", + "tough-cookie": "^5.0.0", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "packages/apify/node_modules/@crawlee/memory-storage": { + "version": "3.12.3-beta.12", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.12.3-beta.12.tgz", + "integrity": "sha512-JJ8WxjrV0afPIdiwiofa+xjXItmbcl2ATfRheRFVPJ+xoI97hLlrotkONF9bcsIbff5JfMh6d1RqhTa7b3su4A==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@crawlee/types": "3.12.3-beta.12", + "@sapphire/async-queue": "^1.5.0", + "@sapphire/shapeshift": "^3.0.0", + "content-type": "^1.0.4", + "fs-extra": "^11.0.0", + "json5": "^2.2.3", + "mime-types": "^2.1.35", + "proper-lockfile": "^4.1.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">= 16" + } + }, + "packages/apify/node_modules/@crawlee/types": { + "version": "3.12.3-beta.12", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.12.3-beta.12.tgz", + "integrity": "sha512-pmN+VKwhd3rv7Tr/RfbIEp42S/1NvLNditWp48M3aL+sUbLCxyERNBV0Dq5hLUDCTqnfFUMqLla8kBs2NGOgtw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "packages/apify/node_modules/@crawlee/utils": { + "version": "3.12.3-beta.12", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.12.3-beta.12.tgz", + "integrity": "sha512-URtKXpcnOHI/QjrzHJtf2hU2iP1viYpfb8a28H3+hF8aWD6Q1ppnvP5RMtm1sBmj8qBrTDK3RnwKJxq+T9zqOA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/ps-tree": "^1.2.0", + "@crawlee/types": "3.12.3-beta.12", + "@types/sax": "^1.2.7", + "cheerio": "1.0.0-rc.12", + "file-type": "^20.0.0", + "got-scraping": "^4.0.3", + "ow": "^0.28.1", + "robots-parser": "^3.0.1", + "sax": "^1.4.1", + "tslib": "^2.4.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "packages/apify/node_modules/file-type": { + "version": "20.1.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.1.0.tgz", + "integrity": "sha512-XoxU+lETfCf+bYK3SXkxFusAvmtYQl1u/ZC4zw1DBLEsHUvh339uwYucgQnnSMz1mRCWYJrCzsbJJ95hsQbZ8A==", + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.2.6", + "strtok3": "^10.2.0", + "token-types": "^6.0.0", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "packages/apify/node_modules/peek-readable": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-6.1.1.tgz", + "integrity": "sha512-7QmvgRKhxM0E2PGV4ocfROItVode+ELI27n4q+lpufZ+tRKBu/pBP8WOmw9HXn2ui/AUizqtvaVQhcJrOkRqYg==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "packages/apify/node_modules/strtok3": { + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.2.1.tgz", + "integrity": "sha512-Q2dTnW3UXokAvXmXvrvMoUj/me3LyJI76HNHeuGMh2o0As/vzd7eHV3ncLOyvu928vQIDbE7Vf9ldEnC7cwy1w==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^6.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "packages/scraper-tools": { "name": "@apify/scraper-tools", "version": "1.1.4", diff --git a/packages/apify/package.json b/packages/apify/package.json index 6e7b294585..4b56e1a8a2 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -58,10 +58,10 @@ "@apify/input_secrets": "^1.1.40", "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.9.3", - "@crawlee/core": "^3.9.0", - "@crawlee/types": "^3.9.0", - "@crawlee/utils": "^3.9.0", + "@apify/utilities": "^2.13.0", + "@crawlee/core": "^3.12.3-beta.12", + "@crawlee/types": "^3.12.3-beta.12", + "@crawlee/utils": "^3.12.3-beta.12", "apify-client": "^2.11.1", "fs-extra": "^11.2.0", "ow": "^0.28.2", @@ -69,4 +69,4 @@ "tslib": "^2.6.2", "ws": "^8.18.0" } -} \ No newline at end of file +} diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index e19b713d89..10f4f298ab 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -1,3 +1,4 @@ +import { createHmacSignature } from '@apify/utilities'; import type { StorageManagerOptions } from '@crawlee/core'; import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core'; @@ -19,7 +20,11 @@ export class KeyValueStore extends CoreKeyValueStore { return getPublicUrl.call(this, key); } - return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`; + if (!this.storageObject?.urlSigningSecretKea) { + return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`; + } + + return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}?signature=${createHmacSignature(this.storageObject.urlSigningSecretKea as string, key)}`; } /** From 3b40e5768fd61ab32c91657309364f24e555ac67 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:24:45 +0100 Subject: [PATCH 02/10] refactor: compose url with URL() --- packages/apify/src/key_value_store.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index 10f4f298ab..361a1c0653 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -20,11 +20,13 @@ export class KeyValueStore extends CoreKeyValueStore { return getPublicUrl.call(this, key); } - if (!this.storageObject?.urlSigningSecretKea) { - return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`; + const publicUrl = new URL(`https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`, 'https://api.apify.com'); + + if (this.storageObject?.urlSigningSecretKey) { + publicUrl.searchParams.append('signature', createHmacSignature(this.storageObject.urlSigningSecretKey as string, key)); } - return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}?signature=${createHmacSignature(this.storageObject.urlSigningSecretKea as string, key)}`; + return publicUrl.toString(); } /** From 13cdc6389ea8562444c68d58ed3ecda3ff6985a1 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:27:07 +0100 Subject: [PATCH 03/10] refactor: clean up --- packages/apify/src/key_value_store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index 361a1c0653..89f7500e5c 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -20,7 +20,7 @@ export class KeyValueStore extends CoreKeyValueStore { return getPublicUrl.call(this, key); } - const publicUrl = new URL(`https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`, 'https://api.apify.com'); + const publicUrl = new URL(`https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`); if (this.storageObject?.urlSigningSecretKey) { publicUrl.searchParams.append('signature', createHmacSignature(this.storageObject.urlSigningSecretKey as string, key)); From 7a74be948df7d79378ea7619ef225155d25d1446 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:31:31 +0100 Subject: [PATCH 04/10] chore: added test for publicUrl --- package-lock.json | 644 ++++++++++++++++------------ package.json | 2 +- test/e2e/sdk/publicUrl/src/main.mjs | 16 + test/e2e/sdk/publicUrl/test.mjs | 22 + 4 files changed, 405 insertions(+), 279 deletions(-) create mode 100644 test/e2e/sdk/publicUrl/src/main.mjs create mode 100644 test/e2e/sdk/publicUrl/test.mjs diff --git a/package-lock.json b/package-lock.json index 7964b56e68..43ffc4d2ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.11.5", + "crawlee": "^3.13", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", @@ -234,6 +234,19 @@ "@apify/log": "^2.5.13" } }, + "node_modules/@asamuzakjp/css-color": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-2.8.3.tgz", + "integrity": "sha512-GIc76d9UI1hCvOATjZPyHFmE5qhRccp3/zGfMPapK3jBi+yocEzp6BBB0UnfRYP9NP4FANqUZYb0hnfs3TM3hw==", + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.1", + "@csstools/css-color-parser": "^3.0.7", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" + } + }, "node_modules/@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", @@ -515,17 +528,17 @@ } }, "node_modules/@crawlee/basic": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.12.0.tgz", - "integrity": "sha512-ltA1Z8zrmsgcCbiwa5PwqG4Pw4MdfaXqaXjdM6WYKdOHa7P0meDSrTaVtTrBBvNOpqODKFef94x5zo7/7Ya1Fg==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.13.0.tgz", + "integrity": "sha512-OJu+o/sK0L/St1qeK2w8e6ypZv/dVRqCuCYOPNQ4xl6VC8F9DlPZRg/iJyDFImj+SXZIWDOJWJEHPmPPjD+eKA==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/core": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/core": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "csv-stringify": "^6.2.0", "fs-extra": "^11.0.0", "got-scraping": "^4.0.0", @@ -539,16 +552,16 @@ } }, "node_modules/@crawlee/browser": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.12.0.tgz", - "integrity": "sha512-VL/Dfs0Pdr/duNJKNEg9b9ndYskRvG/Bwc/VHOf2j2K/De4pk+eRZhlt8FnQItIljhAb55EGoya90ZYck4o8cQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.13.0.tgz", + "integrity": "sha512-UnAawFfzzOWzX2g9WL5+Iapz3stVL3iG8kf0UQi65JAjZj48coOyt0KQb8WgrRMj3Xmz+P/IxR/55qeiCS9/dQ==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", - "@crawlee/basic": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/basic": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "ow": "^0.28.1", "tslib": "^2.4.0", "type-fest": "^4.0.0" @@ -570,15 +583,15 @@ } }, "node_modules/@crawlee/browser-pool": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.12.0.tgz", - "integrity": "sha512-URModIQb033TC8cqfgJLmwVpueR4dAdyRBOPhyzOiahZSXd3PC2ZLvZ8ALyAUhW5Qjqlxr6vpEuZpEo/MBrReg==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.13.0.tgz", + "integrity": "sha512-e3LbgsAmb2au+KVgcDMCxkRs4XWBFJjQSFcQuSPIy0ObYeVWhdeY+9g21gJoj/oHL+VFEX5GYhuIvIr3FY7cNA==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.0", - "@crawlee/core": "3.12.0", - "@crawlee/types": "3.12.0", + "@crawlee/core": "3.13.0", + "@crawlee/types": "3.13.0", "fingerprint-generator": "^2.0.6", "fingerprint-injector": "^2.0.5", "lodash.merge": "^4.6.2", @@ -607,14 +620,14 @@ } }, "node_modules/@crawlee/cheerio": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.12.0.tgz", - "integrity": "sha512-OHLUjyoSaW1TYpquigU440UHuOmEPiyYImxPPykaY5j3OPOBDFKR1BsHA+dVT9A7ZOKcpy2abnnv4lsjz60L7A==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.13.0.tgz", + "integrity": "sha512-VlK6BX5cGZzTOGDoAF/Z0DVav1KjiYpD2GXP3kpuJiGFFN97GNIIVoMPnnoTT0XnYQ7S1b1DyyJwNn4vkA0QdA==", "license": "Apache-2.0", "dependencies": { - "@crawlee/http": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/http": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "cheerio": "1.0.0-rc.12", "htmlparser2": "^9.0.0", "tslib": "^2.4.0" @@ -624,12 +637,12 @@ } }, "node_modules/@crawlee/cli": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.12.0.tgz", - "integrity": "sha512-YGgL605IgXGJfJRtjN52S7GOWBzY8kuHMsKkAZKqRPsCnvoOtqEdJlIoXrMOKJbQfFb1u34UqMvDlozrh3udEA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.13.0.tgz", + "integrity": "sha512-AzXlmJKRQ9EQGjDr+Vc4pYe0VrE1DTHaFt0Cf9x16HGIKMWrkKN9mX1vwvuL0Exv4Za/QXF5i8xnV0ralmtlNQ==", "license": "Apache-2.0", "dependencies": { - "@crawlee/templates": "3.12.0", + "@crawlee/templates": "3.13.0", "ansi-colors": "^4.1.3", "fs-extra": "^11.0.0", "inquirer": "^8.2.4", @@ -645,9 +658,9 @@ } }, "node_modules/@crawlee/core": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.12.0.tgz", - "integrity": "sha512-64TM+/gBh2r15SjLhPMTmTQrgdtfiQT96c0XKtgH5oATFytn83NkUzjpGiJRyKzKmfcTby3xUhOhSDeRo2qhtQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.13.0.tgz", + "integrity": "sha512-oDP1tumieNPcKORQfTtolVP6fkOQovTKT4LISAJ/P02iu4bBBfWMxno4uJncskgw6eMUW5xUAhAdPJi52qa6uQ==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.20.0", @@ -656,9 +669,9 @@ "@apify/pseudo_url": "^2.0.30", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/memory-storage": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/memory-storage": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "@sapphire/async-queue": "^1.5.1", "@vladfrangu/async_event_emitter": "^2.2.2", "csv-stringify": "^6.2.0", @@ -678,16 +691,16 @@ } }, "node_modules/@crawlee/http": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.12.0.tgz", - "integrity": "sha512-kaSz5FmnVChfZkGJNblX2l2vUgcHHHpntagBr3dAKW+fkTTPqH4/G3f6dQcyBotW33gR0nq/e3AhC/T+mYjn+Q==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.13.0.tgz", + "integrity": "sha512-IJAMXXBOhjqKPYRAH4rBeVLeX/juCbbKkVmsrxmbmctl9cQc4WbTRPjJkYDjx82Lb6BbN1OZwFWHEuJNm44TAw==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/basic": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/basic": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "@types/content-type": "^1.1.5", "cheerio": "1.0.0-rc.12", "content-type": "^1.0.4", @@ -703,19 +716,19 @@ } }, "node_modules/@crawlee/jsdom": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.12.0.tgz", - "integrity": "sha512-y2+wYuKKD8q6tmJTUujQw4jz7EixxYGKYc0pNNm57Gu/5VAYO7w5OvLMEqdYIB6HmQknAZ8n46KxBzqbcHlokA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.13.0.tgz", + "integrity": "sha512-SkAltVDH0cvQKdS/Y7xBM0oJsQX0CNe9dXbTfgX+Tm092ZceR1gtncUJvXXoD+Dx8twhga9s5mrOi4KeowPqVg==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/http": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "@types/jsdom": "^21.0.0", "cheerio": "1.0.0-rc.12", - "jsdom": "^25.0.0", + "jsdom": "^26.0.0", "ow": "^0.28.2", "tslib": "^2.4.0" }, @@ -723,16 +736,62 @@ "node": ">=16.0.0" } }, + "node_modules/@crawlee/jsdom/node_modules/jsdom": { + "version": "26.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.0.0.tgz", + "integrity": "sha512-BZYDGVAIriBWTpIxYzrXjv3E/4u8+/pSG5bQdIYCbNCGOvsPkDQfTVLAIXAf9ETdCpduCVTkDe2NNZ8NIwUVzw==", + "license": "MIT", + "dependencies": { + "cssstyle": "^4.2.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.4.3", + "form-data": "^4.0.1", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.16", + "parse5": "^7.2.1", + "rrweb-cssom": "^0.8.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.0.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.1.0", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/@crawlee/jsdom/node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "license": "MIT" + }, "node_modules/@crawlee/linkedom": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.12.0.tgz", - "integrity": "sha512-1Y+tDLT+802O2NGi3rhzluW7nbI5wFCvCEiYjHBt/fKM/rcaSakhNwfycG8QZn217PEw/SCy/V9Z5oymtpqoBQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.13.0.tgz", + "integrity": "sha512-ZCsw7ydIhVPdc4SWtkBh/uhPMDiP1xzah/ECKNxCn1AYgR+E8mcEHl/J9XxnPl0jV87g/qXIVwHKPWMLxkkc4A==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.12.0", - "@crawlee/types": "3.12.0", + "@crawlee/http": "3.13.0", + "@crawlee/types": "3.13.0", "linkedom": "^0.18.0", "ow": "^0.28.2", "tslib": "^2.4.0" @@ -742,13 +801,13 @@ } }, "node_modules/@crawlee/memory-storage": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.12.0.tgz", - "integrity": "sha512-PrIWH/tAw1+8kxH9+Y1eKwDuq2T+JKRiv+ltOIog0n/3tu7HTGU2Uom/1GZHHXag6jWDyBQsrHFMGSneTfhJ/w==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.13.0.tgz", + "integrity": "sha512-WC+zG2yucSL7lstrijwzm/ZkX0f/iN42x79wDkTG11ZOEGJyCFR3dmPGuNiyhK2fdGIZapLJiUolJnGOAsNtxg==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", - "@crawlee/types": "3.12.0", + "@crawlee/types": "3.13.0", "@sapphire/async-queue": "^1.5.0", "@sapphire/shapeshift": "^3.0.0", "content-type": "^1.0.4", @@ -763,19 +822,19 @@ } }, "node_modules/@crawlee/playwright": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.12.0.tgz", - "integrity": "sha512-FQYUH3oy6BNkQfB9b88ATS67e2qZcITXQSQxI/sezx6KPakRfIGUmhVuGuDrnZAe6atGBWiJDR/NRINF7hEnhA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.13.0.tgz", + "integrity": "sha512-C2mdkOp/hZQyq1/EUdovXLFmDRrWa4zGRTVaYZ178FmxwdZnVe95hpWd2ZXL5CagHCifVqhbrMDp7M4lKz28nA==", "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.0", "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.1", - "@crawlee/browser": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/core": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/browser": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/core": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "cheerio": "1.0.0-rc.12", "idcac-playwright": "^0.1.2", "jquery": "^3.6.0", @@ -799,17 +858,17 @@ } }, "node_modules/@crawlee/puppeteer": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.12.0.tgz", - "integrity": "sha512-YZTAnvLJ1HRjw02TICesWI0ZNbmu4XPdIS/Wuv+82MUTjqJ1sqiRa+pBfx3KNgiOOcOj1/P9bNmINqOmPqA+aA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.13.0.tgz", + "integrity": "sha512-UaMsmHGRtKGT4ru0gupemzIMgLt36Vb0369lRp3zRLeZrLux4cDyh7AzlHwv1cudW0gQaClujaQUP1YK2x3pXw==", "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.0", "@apify/log": "^2.4.0", - "@crawlee/browser": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/browser": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "cheerio": "1.0.0-rc.12", "devtools-protocol": "*", "idcac-playwright": "^0.1.2", @@ -830,9 +889,9 @@ } }, "node_modules/@crawlee/templates": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.12.0.tgz", - "integrity": "sha512-pr8qOmfAB7oYLYBXvxLThRV9S/Kpj74qG3vPGdJc2ax6RskooQtaa7Jqgr8vJFlTbCnbgquiko0QhvLamC0aRg==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.13.0.tgz", + "integrity": "sha512-5VM2ukcM/G7FvklnlmSMd/Th4F/Cr+lUVWSQsFJI2GKK0DnovQ1T4OhcSPFmGAjKqaIL/MT2L7KjIJNV+dA21w==", "license": "Apache-2.0", "dependencies": { "ansi-colors": "^4.1.3", @@ -896,9 +955,9 @@ } }, "node_modules/@crawlee/types": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.12.0.tgz", - "integrity": "sha512-W52s8H3jZ9H7pjxygC7mhqwdNIV29mHxJr5yujnHYJyH1iWLU6p22h0M55l11KZmUBwaRq5LqmPX6LbHNRqyoQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.13.0.tgz", + "integrity": "sha512-CdUecVRN2TkqwmVqJ+DiiKBj/jBBlUzsIaCZYDaRwAYEZmzVmWQ7GkcQju6YYbqtRTS+3HLyxW4bgTZdRQW98g==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.4.0" @@ -908,17 +967,17 @@ } }, "node_modules/@crawlee/utils": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.12.0.tgz", - "integrity": "sha512-8BILVHixIzAvOhHjychVnfch/ptt7dcJJ1pRLWdYvhF6Y+LmXg2KKPWltRajx0St9pXOkJaNQ11HzSK1n29wfw==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.13.0.tgz", + "integrity": "sha512-hiL+VrJ7jrExwuMHVm28EvfloonrZshhzIaV3MnocURM9B/ByILdEjuYr2bb+lRlsuk3czuhkprYbE+AsEXT9A==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/ps-tree": "^1.2.0", - "@crawlee/types": "3.12.0", + "@crawlee/types": "3.13.0", "@types/sax": "^1.2.7", "cheerio": "1.0.0-rc.12", - "file-type": "^19.0.0", + "file-type": "^20.0.0", "got-scraping": "^4.0.3", "ow": "^0.28.1", "robots-parser": "^3.0.1", @@ -930,6 +989,116 @@ "node": ">=16.0.0" } }, + "node_modules/@csstools/color-helpers": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.0.2.tgz", + "integrity": "sha512-JqWH1vsgdGcw2RR6VliXXdA0/59LttzlU8UlRT/iUUsEeWfYq8I+K0yhihEUTTHLRm1EXvpsCx3083EU15ecsA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.2.tgz", + "integrity": "sha512-TklMyb3uBB28b5uQdxjReG4L80NxAqgrECqLZFQbyLekwwlcDDS8r3f07DKqeo8C4926Br0gf/ZDe17Zv4wIuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.0.8.tgz", + "integrity": "sha512-pdwotQjCCnRPuNi06jFuP68cykU1f3ZWExLe/8MQ1LOs8Xq+fTkYgd+2V8mWUWMrOn9iS2HftPVaMZDaXzGbhQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.0.2", + "@csstools/css-calc": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.4.tgz", + "integrity": "sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.3.tgz", + "integrity": "sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@emnapi/core": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.3.1.tgz", @@ -1560,9 +1729,9 @@ } }, "node_modules/@inquirer/figures": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.8.tgz", - "integrity": "sha512-tKd+jsmhq21AP1LhexC0pPwsCxEhGgAkg28byjJAd+xhmIs8LUX8JbUc3vBf3PhLxWiB5EvyBE5X7JSPAqMAqg==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.10.tgz", + "integrity": "sha512-Ey6176gZmeqZuY/W/nZiUyvmb1/qInjcpiZjXWi6nON+nxJpD1bxtSoBxNliGISae32n6OwbY+TSXPZ1CfS4bw==", "license": "MIT", "engines": { "node": ">=18" @@ -3277,23 +3446,40 @@ } }, "node_modules/@tokenizer/inflate": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.6.tgz", - "integrity": "sha512-SdR/i05U7Xhnsq36iyIq/ZiGGw4PKzw4ww3bOq80Pjj4wyXpqyTcgrgdDdGlcatnlvzNJx8CQw3hp6QZvkUwhA==", + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", + "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", "license": "MIT", "dependencies": { - "debug": "^4.3.7", + "debug": "^4.4.0", "fflate": "^0.8.2", "token-types": "^6.0.0" }, "engines": { - "node": ">=16" + "node": ">=18" }, "funding": { "type": "github", "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/@tokenizer/inflate/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/@tokenizer/token": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", @@ -6466,23 +6652,23 @@ } }, "node_modules/crawlee": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.12.0.tgz", - "integrity": "sha512-+3/7XQEiEt2TzhX0uENgyvDIC5CSn8QPtYMeT8Rr/qUbtkhffdmNoypMnPYYruoqlVPORb5y+BNUFAl0IpUFaA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.13.0.tgz", + "integrity": "sha512-bcBI/FW6TqfnIU44hlkFXKgr0rOYiQbT4B1ulsL1OO9zMlUezoL5DfPUu8FVbqNBzHZk/2Kf1vt9nIxfuWNvdA==", "license": "Apache-2.0", "dependencies": { - "@crawlee/basic": "3.12.0", - "@crawlee/browser": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/cheerio": "3.12.0", - "@crawlee/cli": "3.12.0", - "@crawlee/core": "3.12.0", - "@crawlee/http": "3.12.0", - "@crawlee/jsdom": "3.12.0", - "@crawlee/linkedom": "3.12.0", - "@crawlee/playwright": "3.12.0", - "@crawlee/puppeteer": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/basic": "3.13.0", + "@crawlee/browser": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/cheerio": "3.13.0", + "@crawlee/cli": "3.13.0", + "@crawlee/core": "3.13.0", + "@crawlee/http": "3.13.0", + "@crawlee/jsdom": "3.13.0", + "@crawlee/linkedom": "3.13.0", + "@crawlee/playwright": "3.13.0", + "@crawlee/puppeteer": "3.13.0", + "@crawlee/utils": "3.13.0", "import-local": "^3.1.0", "tslib": "^2.4.0" }, @@ -6568,17 +6754,24 @@ "license": "MIT" }, "node_modules/cssstyle": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.1.0.tgz", - "integrity": "sha512-h66W1URKpBS5YMI/V8PyXvTMFT8SupJ1IzoIV8IeBC/ji8WVmrO8dGlTi+2dh6whmdk6BiKJLD/ZBkhWbcg6nA==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.2.1.tgz", + "integrity": "sha512-9+vem03dMXG7gDmZ62uqmRiMRNtinIZ9ZyuF6BdxzfOD+FdN5hretzynkn0ReS2DO2GSw76RWHs0UmJPI2zUjw==", "license": "MIT", "dependencies": { - "rrweb-cssom": "^0.7.1" + "@asamuzakjp/css-color": "^2.8.2", + "rrweb-cssom": "^0.8.0" }, "engines": { "node": ">=18" } }, + "node_modules/cssstyle/node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "license": "MIT" + }, "node_modules/csv-stringify": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.2.tgz", @@ -7074,9 +7267,9 @@ } }, "node_modules/domutils": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", - "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", "license": "BSD-2-Clause", "dependencies": { "dom-serializer": "^2.0.0", @@ -8537,15 +8730,15 @@ } }, "node_modules/file-type": { - "version": "19.6.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-19.6.0.tgz", - "integrity": "sha512-VZR5I7k5wkD0HgFnMsq5hOsSc710MJMu5Nc5QYsbe38NN5iPV/XTObYLc/cpttRTf6lX538+5uO1ZQRhYibiZQ==", + "version": "20.4.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.4.0.tgz", + "integrity": "sha512-+NZeExsi4G6EWaMbSmvBeCoqsj9EqNvOj1o/0uPVPW4O51FSCmxFlNEp/PitsqBMCbax4cGoaYmnUK5FLTuG4g==", "license": "MIT", "dependencies": { - "get-stream": "^9.0.1", - "strtok3": "^9.0.1", + "@tokenizer/inflate": "^0.2.6", + "strtok3": "^10.2.0", "token-types": "^6.0.0", - "uint8array-extras": "^1.3.0" + "uint8array-extras": "^1.4.0" }, "engines": { "node": ">=18" @@ -11395,6 +11588,7 @@ "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.1.tgz", "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==", "license": "MIT", + "peer": true, "dependencies": { "cssstyle": "^4.1.0", "data-urls": "^5.0.0", @@ -11973,18 +12167,49 @@ } }, "node_modules/linkedom": { - "version": "0.18.5", - "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.5.tgz", - "integrity": "sha512-JGLaGGtqtu+eOhYrC1wkWYTBcpVWL4AsnwAtMtgO1Q0gI0PuPJKI0zBBE+a/1BrhOE3Uw8JI/ycByAv5cLrAuQ==", + "version": "0.18.9", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.9.tgz", + "integrity": "sha512-Pfvhwjs46nBrcQdauQjMXDJZqj6VwN7KStT84xQqmIgD9bPH6UVJ/ESW8y4VHVF2h7di0/P+f4Iln4U5emRcmg==", "license": "ISC", "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", - "htmlparser2": "^9.1.0", + "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" } }, + "node_modules/linkedom/node_modules/entities": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.0.tgz", + "integrity": "sha512-aKstq2TDOndCn4diEyp9Uq/Flu2i1GlLkc6XIDQSDMuaFE3OPW5OphLCyQ5SpSJZTb4reN+kTcYru5yIfXoRPw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/linkedom/node_modules/htmlparser2": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", + "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.1", + "entities": "^6.0.0" + } + }, "node_modules/lint-staged": { "version": "15.2.10", "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-15.2.10.tgz", @@ -12682,7 +12907,6 @@ "version": "10.4.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true, "license": "ISC" }, "node_modules/magic-string": { @@ -13589,9 +13813,9 @@ } }, "node_modules/nwsapi": { - "version": "2.2.13", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.13.tgz", - "integrity": "sha512-cTGB9ptp9dY9A5VbMSe7fQBcl/tt22Vcqdq8+eN93rblOuE0aCFu4aZ2vMwct/2t+lFnosm8RkQW1I0Omb1UtQ==", + "version": "2.2.18", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.18.tgz", + "integrity": "sha512-p1TRH/edngVEHVbwqWnxUViEmq5znDvyB+Sik5cmuLpGOIfDf/39zLiq3swPF8Vakqn+gvNiOQAZu8djYlQILA==", "license": "MIT" }, "node_modules/nx": { @@ -14523,12 +14747,12 @@ } }, "node_modules/peek-readable": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.3.1.tgz", - "integrity": "sha512-GVlENSDW6KHaXcd9zkZltB7tCLosKB/4Hg0fqBJkAoBgYG2Tn1xtMgXtSUuMU9AK/gCm/tTdT8mgAeF4YNeeqw==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-6.1.1.tgz", + "integrity": "sha512-7QmvgRKhxM0E2PGV4ocfROItVode+ELI27n4q+lpufZ+tRKBu/pBP8WOmw9HXn2ui/AUizqtvaVQhcJrOkRqYg==", "license": "MIT", "engines": { - "node": ">=14.16" + "node": ">=18" }, "funding": { "type": "github", @@ -15824,7 +16048,8 @@ "version": "0.7.1", "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz", "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/run-async": { "version": "2.4.1", @@ -17039,16 +17264,16 @@ } }, "node_modules/strtok3": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-9.0.1.tgz", - "integrity": "sha512-ERPW+XkvX9W2A+ov07iy+ZFJpVdik04GhDA4eVogiG9hpC97Kem2iucyzhFxbFRvQ5o2UckFtKZdp1hkGvnrEw==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.2.1.tgz", + "integrity": "sha512-Q2dTnW3UXokAvXmXvrvMoUj/me3LyJI76HNHeuGMh2o0As/vzd7eHV3ncLOyvu928vQIDbE7Vf9ldEnC7cwy1w==", "license": "MIT", "dependencies": { "@tokenizer/token": "^0.3.0", - "peek-readable": "^5.3.1" + "peek-readable": "^6.1.1" }, "engines": { - "node": ">=16" + "node": ">=18" }, "funding": { "type": "github", @@ -18818,9 +19043,9 @@ } }, "node_modules/whatwg-url": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz", - "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.1.1.tgz", + "integrity": "sha512-mDGf9diDad/giZ/Sm9Xi2YcyzaFpbdLpJPr+E9fSkyQ7KpQD4SdFcugkRQYzhmfI4KeV4Qpnn2sKPdo+kmsgRQ==", "license": "MIT", "dependencies": { "tr46": "^5.0.0", @@ -19532,7 +19757,7 @@ "@crawlee/core": "^3.12.3-beta.12", "@crawlee/types": "^3.12.3-beta.12", "@crawlee/utils": "^3.12.3-beta.12", - "apify-client": "^2.12.0", + "apify-client": "^2.11.1", "fs-extra": "^11.2.0", "ow": "^0.28.2", "semver": "^7.5.4", @@ -19543,143 +19768,6 @@ "node": ">=16.0.0" } }, - "packages/apify/node_modules/@crawlee/core": { - "version": "3.12.3-beta.12", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.12.3-beta.12.tgz", - "integrity": "sha512-Roz4b1Q/h83r3S8Sx3jCXxXGp+9pCLwpUUpQyAtt6InYr3i8yFBBMJis0PS3hbaL1pn5dbzzIuREHZs4aBEXCg==", - "license": "Apache-2.0", - "dependencies": { - "@apify/consts": "^2.20.0", - "@apify/datastructures": "^2.0.0", - "@apify/log": "^2.4.0", - "@apify/pseudo_url": "^2.0.30", - "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.7.10", - "@crawlee/memory-storage": "3.12.3-beta.12", - "@crawlee/types": "3.12.3-beta.12", - "@crawlee/utils": "3.12.3-beta.12", - "@sapphire/async-queue": "^1.5.1", - "@vladfrangu/async_event_emitter": "^2.2.2", - "csv-stringify": "^6.2.0", - "fs-extra": "^11.0.0", - "got-scraping": "^4.0.0", - "json5": "^2.2.3", - "minimatch": "^9.0.0", - "ow": "^0.28.1", - "stream-json": "^1.8.0", - "tldts": "^6.0.0", - "tough-cookie": "^5.0.0", - "tslib": "^2.4.0", - "type-fest": "^4.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "packages/apify/node_modules/@crawlee/memory-storage": { - "version": "3.12.3-beta.12", - "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.12.3-beta.12.tgz", - "integrity": "sha512-JJ8WxjrV0afPIdiwiofa+xjXItmbcl2ATfRheRFVPJ+xoI97hLlrotkONF9bcsIbff5JfMh6d1RqhTa7b3su4A==", - "license": "Apache-2.0", - "dependencies": { - "@apify/log": "^2.4.0", - "@crawlee/types": "3.12.3-beta.12", - "@sapphire/async-queue": "^1.5.0", - "@sapphire/shapeshift": "^3.0.0", - "content-type": "^1.0.4", - "fs-extra": "^11.0.0", - "json5": "^2.2.3", - "mime-types": "^2.1.35", - "proper-lockfile": "^4.1.2", - "tslib": "^2.4.0" - }, - "engines": { - "node": ">= 16" - } - }, - "packages/apify/node_modules/@crawlee/types": { - "version": "3.12.3-beta.12", - "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.12.3-beta.12.tgz", - "integrity": "sha512-pmN+VKwhd3rv7Tr/RfbIEp42S/1NvLNditWp48M3aL+sUbLCxyERNBV0Dq5hLUDCTqnfFUMqLla8kBs2NGOgtw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.4.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "packages/apify/node_modules/@crawlee/utils": { - "version": "3.12.3-beta.12", - "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.12.3-beta.12.tgz", - "integrity": "sha512-URtKXpcnOHI/QjrzHJtf2hU2iP1viYpfb8a28H3+hF8aWD6Q1ppnvP5RMtm1sBmj8qBrTDK3RnwKJxq+T9zqOA==", - "license": "Apache-2.0", - "dependencies": { - "@apify/log": "^2.4.0", - "@apify/ps-tree": "^1.2.0", - "@crawlee/types": "3.12.3-beta.12", - "@types/sax": "^1.2.7", - "cheerio": "1.0.0-rc.12", - "file-type": "^20.0.0", - "got-scraping": "^4.0.3", - "ow": "^0.28.1", - "robots-parser": "^3.0.1", - "sax": "^1.4.1", - "tslib": "^2.4.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "packages/apify/node_modules/file-type": { - "version": "20.1.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.1.0.tgz", - "integrity": "sha512-XoxU+lETfCf+bYK3SXkxFusAvmtYQl1u/ZC4zw1DBLEsHUvh339uwYucgQnnSMz1mRCWYJrCzsbJJ95hsQbZ8A==", - "license": "MIT", - "dependencies": { - "@tokenizer/inflate": "^0.2.6", - "strtok3": "^10.2.0", - "token-types": "^6.0.0", - "uint8array-extras": "^1.4.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/file-type?sponsor=1" - } - }, - "packages/apify/node_modules/peek-readable": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-6.1.1.tgz", - "integrity": "sha512-7QmvgRKhxM0E2PGV4ocfROItVode+ELI27n4q+lpufZ+tRKBu/pBP8WOmw9HXn2ui/AUizqtvaVQhcJrOkRqYg==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, - "packages/apify/node_modules/strtok3": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.2.1.tgz", - "integrity": "sha512-Q2dTnW3UXokAvXmXvrvMoUj/me3LyJI76HNHeuGMh2o0As/vzd7eHV3ncLOyvu928vQIDbE7Vf9ldEnC7cwy1w==", - "license": "MIT", - "dependencies": { - "@tokenizer/token": "^0.3.0", - "peek-readable": "^6.1.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "packages/scraper-tools": { "name": "@apify/scraper-tools", "version": "1.1.4", diff --git a/package.json b/package.json index 37e153a5fb..6e34a48ae9 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.11.5", + "crawlee": "^3.13", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", diff --git a/test/e2e/sdk/publicUrl/src/main.mjs b/test/e2e/sdk/publicUrl/src/main.mjs new file mode 100644 index 0000000000..2bdef6052c --- /dev/null +++ b/test/e2e/sdk/publicUrl/src/main.mjs @@ -0,0 +1,16 @@ +import { Actor, log } from 'apify'; + +const PUBLIC_RECORD_KEY = 'public-record-key'; + +await Actor.init(); + +await Actor.setValue(PUBLIC_RECORD_KEY, JSON.stringify({ exposedData: 'test' }), { contentType: `application/json` }); + +const defaultKeyValueStore = await Actor.openKeyValueStore(); +const publicUrl = defaultKeyValueStore.getPublicUrl(PUBLIC_RECORD_KEY); + +log.info('Generated public url', { publicUrl }); + +await Actor.pushData({ publicUrl }); + +await Actor.exit(); diff --git a/test/e2e/sdk/publicUrl/test.mjs b/test/e2e/sdk/publicUrl/test.mjs new file mode 100644 index 0000000000..366ce82703 --- /dev/null +++ b/test/e2e/sdk/publicUrl/test.mjs @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; + +import { createHmacSignature } from '@apify/utilities'; +import { ApifyClient, Dataset } from 'apify'; + +// Also needs to be changed in main.mjs +const PUBLIC_RECORD_KEY = 'public-record-key'; + +const client = new ApifyClient({ + token: process.env.APIFY_TOKEN, +}); + +const actor = client.actor(process.argv[2]); + +const run = await actor.call({}, { waitSecs: 15 }); +assert.equal(run.exitCode, 0); + +const keyValueStore = await client.keyValueStore(run.defaultKeyValueStoreId).get(); + +const dataset = await Dataset.open(run.defaultDatasetId, { storageClient: client }); +const data = await dataset.getData(); +assert.deepEqual(data.items, [{ publicUrl: `https://api.apify.com/v2/key-value-stores/${keyValueStore.id}/records/${PUBLIC_RECORD_KEY}?signature=${createHmacSignature(keyValueStore.urlSigningSecretKey, PUBLIC_RECORD_KEY)}` }]); From a404dbf10bb551fdf265995701eb1299316e7c9f Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 14 Mar 2025 14:21:09 +0100 Subject: [PATCH 05/10] fix: use apiPublicBaseUrl to generate public url --- packages/apify/src/configuration.ts | 3 +++ packages/apify/src/key_value_store.ts | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index d22ae446d2..482bc358b2 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -16,6 +16,7 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { actorRunId?: string; actorTaskId?: string; apiBaseUrl?: string; + apiPublicBaseUrl?: string; containerPort?: number; containerUrl?: string; proxyHostname?: string; @@ -139,6 +140,7 @@ export class Configuration extends CoreConfiguration { APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl', APIFY_ACTOR_ID: 'actorId', APIFY_API_BASE_URL: 'apiBaseUrl', + APIFY_API_PUBLIC_BASE_URL: 'apiPublicBaseUrl', APIFY_IS_AT_HOME: 'isAtHome', APIFY_ACTOR_RUN_ID: 'actorRunId', APIFY_ACTOR_TASK_ID: 'actorTaskId', @@ -183,6 +185,7 @@ export class Configuration extends CoreConfiguration { defaultRequestQueueId: LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID], inputKey: 'INPUT', apiBaseUrl: 'https://api.apify.com', + apiPublicBaseUrl: 'https://api.apify.com', proxyStatusUrl: 'http://proxy.apify.com', proxyHostname: LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME], proxyPort: +LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT], diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index 89f7500e5c..e5b673b1a3 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -16,11 +16,12 @@ export class KeyValueStore extends CoreKeyValueStore { * access the value in the remote key-value store. */ override getPublicUrl(key: string): string { - if (!(this.config as Configuration).get('isAtHome') && getPublicUrl) { + const config = this.config as Configuration; + if (!config.get('isAtHome') && getPublicUrl) { return getPublicUrl.call(this, key); } - const publicUrl = new URL(`https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`); + const publicUrl = new URL(`${config.get('apiPublicBaseUrl', 'https://api.apify.com')}/v2/key-value-stores/${this.id}/records/${key}`); if (this.storageObject?.urlSigningSecretKey) { publicUrl.searchParams.append('signature', createHmacSignature(this.storageObject.urlSigningSecretKey as string, key)); From 23505f1afe9828b3cce1f6d71148fa59a4c2119f Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 14 Mar 2025 14:21:37 +0100 Subject: [PATCH 06/10] fix: improve test --- test/e2e/sdk/publicUrl/src/main.mjs | 7 ++++++- test/e2e/sdk/publicUrl/test.mjs | 12 +++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/test/e2e/sdk/publicUrl/src/main.mjs b/test/e2e/sdk/publicUrl/src/main.mjs index 2bdef6052c..2c8a5964ab 100644 --- a/test/e2e/sdk/publicUrl/src/main.mjs +++ b/test/e2e/sdk/publicUrl/src/main.mjs @@ -1,14 +1,19 @@ import { Actor, log } from 'apify'; +// Also needs to be changed in test.mjs const PUBLIC_RECORD_KEY = 'public-record-key'; +const PUBLIC_DATA = { exposedData: 'test' }; await Actor.init(); -await Actor.setValue(PUBLIC_RECORD_KEY, JSON.stringify({ exposedData: 'test' }), { contentType: `application/json` }); +await Actor.setValue(PUBLIC_RECORD_KEY, JSON.stringify(PUBLIC_DATA), { contentType: `application/json` }); const defaultKeyValueStore = await Actor.openKeyValueStore(); const publicUrl = defaultKeyValueStore.getPublicUrl(PUBLIC_RECORD_KEY); +// Here we store the url itself +await Actor.setValue('urlToPublicData', publicUrl); + log.info('Generated public url', { publicUrl }); await Actor.pushData({ publicUrl }); diff --git a/test/e2e/sdk/publicUrl/test.mjs b/test/e2e/sdk/publicUrl/test.mjs index 366ce82703..32273d9b03 100644 --- a/test/e2e/sdk/publicUrl/test.mjs +++ b/test/e2e/sdk/publicUrl/test.mjs @@ -1,10 +1,9 @@ import assert from 'node:assert/strict'; -import { createHmacSignature } from '@apify/utilities'; -import { ApifyClient, Dataset } from 'apify'; +import { ApifyClient } from 'apify'; // Also needs to be changed in main.mjs -const PUBLIC_RECORD_KEY = 'public-record-key'; +const PUBLIC_DATA = { exposedData: 'test' }; const client = new ApifyClient({ token: process.env.APIFY_TOKEN, @@ -15,8 +14,7 @@ const actor = client.actor(process.argv[2]); const run = await actor.call({}, { waitSecs: 15 }); assert.equal(run.exitCode, 0); -const keyValueStore = await client.keyValueStore(run.defaultKeyValueStoreId).get(); +const publicUrl = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('urlToPublicData'); +const data = await fetch(publicUrl.value).then((res) => res.json()); -const dataset = await Dataset.open(run.defaultDatasetId, { storageClient: client }); -const data = await dataset.getData(); -assert.deepEqual(data.items, [{ publicUrl: `https://api.apify.com/v2/key-value-stores/${keyValueStore.id}/records/${PUBLIC_RECORD_KEY}?signature=${createHmacSignature(keyValueStore.urlSigningSecretKey, PUBLIC_RECORD_KEY)}` }]); +assert.deepEqual(data, PUBLIC_DATA); From af0f70b49e72090b126d8ecc4c979d3ae8d80ed7 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:36:20 +0100 Subject: [PATCH 07/10] refactor: clean up --- package-lock.json | 16 ++++++++-------- package.json | 2 +- packages/apify/package.json | 8 ++++---- packages/apify/src/key_value_store.ts | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 43ffc4d2ce..bdfb55063c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.13", + "crawlee": "^3.13.0", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", @@ -4261,9 +4261,9 @@ "link": true }, "node_modules/apify-client": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/apify-client/-/apify-client-2.12.0.tgz", - "integrity": "sha512-h04rPVft8tNjnwZswqF2k46bdHZWsDsfOE8PkmklZ9+/s/mb/Q/dMOXCx0u2+RTc8QoAkYS9LYs97wZyUWpoag==", + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/apify-client/-/apify-client-2.12.1.tgz", + "integrity": "sha512-5K9VfSwjb2veD0ts1mScI5mVVuw/eTqF3tjyQj/jGyi7o3WIP5NI+3cXWz/aZGIQtPMLSjJ1VAVGu/of+wcmpw==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.25.0", @@ -19754,10 +19754,10 @@ "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.13.0", - "@crawlee/core": "^3.12.3-beta.12", - "@crawlee/types": "^3.12.3-beta.12", - "@crawlee/utils": "^3.12.3-beta.12", - "apify-client": "^2.11.1", + "@crawlee/core": "^3.13.0", + "@crawlee/types": "^3.13.0", + "@crawlee/utils": "^3.13.0", + "apify-client": "^2.12.1", "fs-extra": "^11.2.0", "ow": "^0.28.2", "semver": "^7.5.4", diff --git a/package.json b/package.json index 6e34a48ae9..537134cd57 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.13", + "crawlee": "^3.13.0", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", diff --git a/packages/apify/package.json b/packages/apify/package.json index 1e22298af6..04e982c9c2 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -59,10 +59,10 @@ "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.13.0", - "@crawlee/core": "^3.12.3-beta.12", - "@crawlee/types": "^3.12.3-beta.12", - "@crawlee/utils": "^3.12.3-beta.12", - "apify-client": "^2.11.1", + "@crawlee/core": "^3.13.0", + "@crawlee/types": "^3.13.0", + "@crawlee/utils": "^3.13.0", + "apify-client": "^2.12.1", "fs-extra": "^11.2.0", "ow": "^0.28.2", "semver": "^7.5.4", diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index e5b673b1a3..6efa2c9146 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -21,7 +21,7 @@ export class KeyValueStore extends CoreKeyValueStore { return getPublicUrl.call(this, key); } - const publicUrl = new URL(`${config.get('apiPublicBaseUrl', 'https://api.apify.com')}/v2/key-value-stores/${this.id}/records/${key}`); + const publicUrl = new URL(`${config.get('apiPublicBaseUrl')}/v2/key-value-stores/${this.id}/records/${key}`); if (this.storageObject?.urlSigningSecretKey) { publicUrl.searchParams.append('signature', createHmacSignature(this.storageObject.urlSigningSecretKey as string, key)); From d2a11f31ff795152c365c3991c6daeb9dc57eec4 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:41:16 +0100 Subject: [PATCH 08/10] refactor: improved test --- test/e2e/sdk/publicUrl/src/main.mjs | 10 ++++------ test/e2e/sdk/publicUrl/test.mjs | 3 +-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/test/e2e/sdk/publicUrl/src/main.mjs b/test/e2e/sdk/publicUrl/src/main.mjs index 2c8a5964ab..9e99a5e719 100644 --- a/test/e2e/sdk/publicUrl/src/main.mjs +++ b/test/e2e/sdk/publicUrl/src/main.mjs @@ -1,15 +1,13 @@ import { Actor, log } from 'apify'; -// Also needs to be changed in test.mjs -const PUBLIC_RECORD_KEY = 'public-record-key'; -const PUBLIC_DATA = { exposedData: 'test' }; - await Actor.init(); -await Actor.setValue(PUBLIC_RECORD_KEY, JSON.stringify(PUBLIC_DATA), { contentType: `application/json` }); +const { data, recordKey } = await Actor.getInput(); + +await Actor.setValue(recordKey, JSON.stringify(data), { contentType: `application/json` }); const defaultKeyValueStore = await Actor.openKeyValueStore(); -const publicUrl = defaultKeyValueStore.getPublicUrl(PUBLIC_RECORD_KEY); +const publicUrl = defaultKeyValueStore.getPublicUrl(recordKey); // Here we store the url itself await Actor.setValue('urlToPublicData', publicUrl); diff --git a/test/e2e/sdk/publicUrl/test.mjs b/test/e2e/sdk/publicUrl/test.mjs index 32273d9b03..0d8fb250ed 100644 --- a/test/e2e/sdk/publicUrl/test.mjs +++ b/test/e2e/sdk/publicUrl/test.mjs @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import { ApifyClient } from 'apify'; -// Also needs to be changed in main.mjs const PUBLIC_DATA = { exposedData: 'test' }; const client = new ApifyClient({ @@ -11,7 +10,7 @@ const client = new ApifyClient({ const actor = client.actor(process.argv[2]); -const run = await actor.call({}, { waitSecs: 15 }); +const run = await actor.call({ data: PUBLIC_DATA, recordKey: 'public-record-key' }, { waitSecs: 15 }); assert.equal(run.exitCode, 0); const publicUrl = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('urlToPublicData'); From d830ed1ea4eb0a7e79708c92107b2197df76df86 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 17 Mar 2025 16:06:36 +0100 Subject: [PATCH 09/10] refactor: clean up --- packages/apify/src/configuration.ts | 2 ++ test/e2e/sdk/publicUrl/src/main.mjs | 6 +++--- test/e2e/sdk/publicUrl/test.mjs | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index 482bc358b2..8619283fc1 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -16,6 +16,8 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { actorRunId?: string; actorTaskId?: string; apiBaseUrl?: string; + // apiBaseUrl is the internal API URL, accessible only within the platform(private network), + // while apiPublicBaseUrl is the public API URL, available externally(through internet). apiPublicBaseUrl?: string; containerPort?: number; containerUrl?: string; diff --git a/test/e2e/sdk/publicUrl/src/main.mjs b/test/e2e/sdk/publicUrl/src/main.mjs index 9e99a5e719..dbe5b5a8e6 100644 --- a/test/e2e/sdk/publicUrl/src/main.mjs +++ b/test/e2e/sdk/publicUrl/src/main.mjs @@ -2,12 +2,12 @@ import { Actor, log } from 'apify'; await Actor.init(); -const { data, recordKey } = await Actor.getInput(); +const { data } = await Actor.getInput(); -await Actor.setValue(recordKey, JSON.stringify(data), { contentType: `application/json` }); +await Actor.setValue('public-record-key', JSON.stringify(data), { contentType: `application/json` }); const defaultKeyValueStore = await Actor.openKeyValueStore(); -const publicUrl = defaultKeyValueStore.getPublicUrl(recordKey); +const publicUrl = defaultKeyValueStore.getPublicUrl('public-record-key'); // Here we store the url itself await Actor.setValue('urlToPublicData', publicUrl); diff --git a/test/e2e/sdk/publicUrl/test.mjs b/test/e2e/sdk/publicUrl/test.mjs index 0d8fb250ed..b7ee380814 100644 --- a/test/e2e/sdk/publicUrl/test.mjs +++ b/test/e2e/sdk/publicUrl/test.mjs @@ -10,7 +10,7 @@ const client = new ApifyClient({ const actor = client.actor(process.argv[2]); -const run = await actor.call({ data: PUBLIC_DATA, recordKey: 'public-record-key' }, { waitSecs: 15 }); +const run = await actor.call({ data: PUBLIC_DATA }, { waitSecs: 15 }); assert.equal(run.exitCode, 0); const publicUrl = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('urlToPublicData'); From cf7144e3981cb3dedc5597ccb8acb232aa445ed1 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 31 Mar 2025 14:20:34 +0200 Subject: [PATCH 10/10] fix: CI --- .../puppeteer-scraper/src/internals/crawler_setup.ts | 2 +- .../actor-scraper/web-scraper/src/internals/crawler_setup.ts | 2 +- packages/apify/src/proxy_configuration.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/actor-scraper/puppeteer-scraper/src/internals/crawler_setup.ts b/packages/actor-scraper/puppeteer-scraper/src/internals/crawler_setup.ts index 757eb7fd90..aed8c04899 100644 --- a/packages/actor-scraper/puppeteer-scraper/src/internals/crawler_setup.ts +++ b/packages/actor-scraper/puppeteer-scraper/src/internals/crawler_setup.ts @@ -185,7 +185,7 @@ export class CrawlerSetup implements CrawlerSetupOptions { launchContext: { useChrome: this.input.useChrome, launchOptions: { - ignoreHTTPSErrors: this.input.ignoreSslErrors, + acceptInsecureCerts: this.input.ignoreSslErrors, defaultViewport: DEFAULT_VIEWPORT, devtools: this.devtools, args, diff --git a/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts b/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts index b1e44362c4..99813e9540 100644 --- a/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts +++ b/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts @@ -228,7 +228,7 @@ export class CrawlerSetup implements CrawlerSetupOptions { launchContext: { useChrome: this.input.useChrome, launchOptions: { - ignoreHTTPSErrors: this.input.ignoreSslErrors, + acceptInsecureCerts: this.input.ignoreSslErrors, defaultViewport: DEFAULT_VIEWPORT, args, headless: this.input.headless, diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 3292a53421..ebb063d37e 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -305,7 +305,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { return this._handleTieredUrl( sessionId ?? cryptoRandomObjectId(6), options, - ).proxyUrl; + ).proxyUrl ?? undefined; } return this.composeDefaultUrl(sessionId);