diff --git a/package-lock.json b/package-lock.json index 8bb22204d6..2147fe4bd8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.11.5", + "crawlee": "^3.13.0", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", @@ -44,9 +44,9 @@ } }, "node_modules/@apify/consts": { - "version": "2.33.0", - "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.33.0.tgz", - "integrity": "sha512-Du4unQM2w1Hdt9ZT5AOMetP9jrTeYJelFRaIBDKNSF6ynZ+sbwBswHWAsxQfuy0gCHRftocexz5xNLzfV7tg4g==", + "version": "2.37.0", + "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.37.0.tgz", + "integrity": "sha512-GCK7wZcUPnHMUpoxd1tgbDgojdgcdxpnaF8UX5h5/hq5ZPdlik7EP2CkU6MpRfYxYl8T5LOsANUJvdB6bGthKA==", "license": "Apache-2.0" }, "node_modules/@apify/datastructures": { @@ -174,12 +174,12 @@ } }, "node_modules/@apify/log": { - "version": "2.5.9", - "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.9.tgz", - "integrity": "sha512-HVmRHPMacUiWA7HYqgIq9sJywFEsyexZBUazfwCvmrOSHIt8kRCpvMNAgm+aMDNNnNm6e+pyA2pBQST7xXCGWQ==", + "version": "2.5.13", + "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.13.tgz", + "integrity": "sha512-gxvWyD9JWBkLoTX7UfQ4s0F32/UfF4T8TG4jAl2CE0vNKr0COOJFkLmmyWnTNSVDL+yGC+LZp3mtbPnB+l6Sog==", "license": "Apache-2.0", "dependencies": { - "@apify/consts": "^2.33.0", + "@apify/consts": "^2.37.0", "ansi-colors": "^4.1.1" } }, @@ -225,13 +225,26 @@ "license": "Apache-2.0" }, "node_modules/@apify/utilities": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.10.10.tgz", - "integrity": "sha512-oJT2oRoTLJAhiTD/uQAQvPetLEiwkNKkhJSNkKi3TUsnxdRmHwxJz6S+evouOAt6Snipe9sb9PBune6DUAXGoA==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.13.1.tgz", + "integrity": "sha512-czQz2oANACVuwE7KrJKB8LrbhvMZyB1bvwO9jTL2SCmKQHxzY34XLQnSfblq/qsLWrwKynKxWXSvAVRZH2oIlA==", "license": "Apache-2.0", "dependencies": { - "@apify/consts": "^2.33.0", - "@apify/log": "^2.5.9" + "@apify/consts": "^2.37.0", + "@apify/log": "^2.5.13" + } + }, + "node_modules/@asamuzakjp/css-color": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-2.8.3.tgz", + "integrity": "sha512-GIc76d9UI1hCvOATjZPyHFmE5qhRccp3/zGfMPapK3jBi+yocEzp6BBB0UnfRYP9NP4FANqUZYb0hnfs3TM3hw==", + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.1", + "@csstools/css-color-parser": "^3.0.7", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" } }, "node_modules/@babel/code-frame": { @@ -515,17 +528,17 @@ } }, "node_modules/@crawlee/basic": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.12.0.tgz", - "integrity": "sha512-ltA1Z8zrmsgcCbiwa5PwqG4Pw4MdfaXqaXjdM6WYKdOHa7P0meDSrTaVtTrBBvNOpqODKFef94x5zo7/7Ya1Fg==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.13.0.tgz", + "integrity": "sha512-OJu+o/sK0L/St1qeK2w8e6ypZv/dVRqCuCYOPNQ4xl6VC8F9DlPZRg/iJyDFImj+SXZIWDOJWJEHPmPPjD+eKA==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/core": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/core": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "csv-stringify": "^6.2.0", "fs-extra": "^11.0.0", "got-scraping": "^4.0.0", @@ -539,16 +552,16 @@ } }, "node_modules/@crawlee/browser": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.12.0.tgz", - "integrity": "sha512-VL/Dfs0Pdr/duNJKNEg9b9ndYskRvG/Bwc/VHOf2j2K/De4pk+eRZhlt8FnQItIljhAb55EGoya90ZYck4o8cQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.13.0.tgz", + "integrity": "sha512-UnAawFfzzOWzX2g9WL5+Iapz3stVL3iG8kf0UQi65JAjZj48coOyt0KQb8WgrRMj3Xmz+P/IxR/55qeiCS9/dQ==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", - "@crawlee/basic": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/basic": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "ow": "^0.28.1", "tslib": "^2.4.0", "type-fest": "^4.0.0" @@ -570,15 +583,15 @@ } }, "node_modules/@crawlee/browser-pool": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.12.0.tgz", - "integrity": "sha512-URModIQb033TC8cqfgJLmwVpueR4dAdyRBOPhyzOiahZSXd3PC2ZLvZ8ALyAUhW5Qjqlxr6vpEuZpEo/MBrReg==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.13.0.tgz", + "integrity": "sha512-e3LbgsAmb2au+KVgcDMCxkRs4XWBFJjQSFcQuSPIy0ObYeVWhdeY+9g21gJoj/oHL+VFEX5GYhuIvIr3FY7cNA==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.0", - "@crawlee/core": "3.12.0", - "@crawlee/types": "3.12.0", + "@crawlee/core": "3.13.0", + "@crawlee/types": "3.13.0", "fingerprint-generator": "^2.0.6", "fingerprint-injector": "^2.0.5", "lodash.merge": "^4.6.2", @@ -607,14 +620,14 @@ } }, "node_modules/@crawlee/cheerio": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.12.0.tgz", - "integrity": "sha512-OHLUjyoSaW1TYpquigU440UHuOmEPiyYImxPPykaY5j3OPOBDFKR1BsHA+dVT9A7ZOKcpy2abnnv4lsjz60L7A==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.13.0.tgz", + "integrity": "sha512-VlK6BX5cGZzTOGDoAF/Z0DVav1KjiYpD2GXP3kpuJiGFFN97GNIIVoMPnnoTT0XnYQ7S1b1DyyJwNn4vkA0QdA==", "license": "Apache-2.0", "dependencies": { - "@crawlee/http": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/http": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "cheerio": "1.0.0-rc.12", "htmlparser2": "^9.0.0", "tslib": "^2.4.0" @@ -624,12 +637,12 @@ } }, "node_modules/@crawlee/cli": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.12.0.tgz", - "integrity": "sha512-YGgL605IgXGJfJRtjN52S7GOWBzY8kuHMsKkAZKqRPsCnvoOtqEdJlIoXrMOKJbQfFb1u34UqMvDlozrh3udEA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.13.0.tgz", + "integrity": "sha512-AzXlmJKRQ9EQGjDr+Vc4pYe0VrE1DTHaFt0Cf9x16HGIKMWrkKN9mX1vwvuL0Exv4Za/QXF5i8xnV0ralmtlNQ==", "license": "Apache-2.0", "dependencies": { - "@crawlee/templates": "3.12.0", + "@crawlee/templates": "3.13.0", "ansi-colors": "^4.1.3", "fs-extra": "^11.0.0", "inquirer": "^8.2.4", @@ -645,9 +658,9 @@ } }, "node_modules/@crawlee/core": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.12.0.tgz", - "integrity": "sha512-64TM+/gBh2r15SjLhPMTmTQrgdtfiQT96c0XKtgH5oATFytn83NkUzjpGiJRyKzKmfcTby3xUhOhSDeRo2qhtQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.13.0.tgz", + "integrity": "sha512-oDP1tumieNPcKORQfTtolVP6fkOQovTKT4LISAJ/P02iu4bBBfWMxno4uJncskgw6eMUW5xUAhAdPJi52qa6uQ==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.20.0", @@ -656,9 +669,9 @@ "@apify/pseudo_url": "^2.0.30", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/memory-storage": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/memory-storage": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "@sapphire/async-queue": "^1.5.1", "@vladfrangu/async_event_emitter": "^2.2.2", "csv-stringify": "^6.2.0", @@ -678,16 +691,16 @@ } }, "node_modules/@crawlee/http": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.12.0.tgz", - "integrity": "sha512-kaSz5FmnVChfZkGJNblX2l2vUgcHHHpntagBr3dAKW+fkTTPqH4/G3f6dQcyBotW33gR0nq/e3AhC/T+mYjn+Q==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.13.0.tgz", + "integrity": "sha512-IJAMXXBOhjqKPYRAH4rBeVLeX/juCbbKkVmsrxmbmctl9cQc4WbTRPjJkYDjx82Lb6BbN1OZwFWHEuJNm44TAw==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/basic": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/basic": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "@types/content-type": "^1.1.5", "cheerio": "1.0.0-rc.12", "content-type": "^1.0.4", @@ -703,19 +716,19 @@ } }, "node_modules/@crawlee/jsdom": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.12.0.tgz", - "integrity": "sha512-y2+wYuKKD8q6tmJTUujQw4jz7EixxYGKYc0pNNm57Gu/5VAYO7w5OvLMEqdYIB6HmQknAZ8n46KxBzqbcHlokA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.13.0.tgz", + "integrity": "sha512-SkAltVDH0cvQKdS/Y7xBM0oJsQX0CNe9dXbTfgX+Tm092ZceR1gtncUJvXXoD+Dx8twhga9s5mrOi4KeowPqVg==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/http": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "@types/jsdom": "^21.0.0", "cheerio": "1.0.0-rc.12", - "jsdom": "^25.0.0", + "jsdom": "^26.0.0", "ow": "^0.28.2", "tslib": "^2.4.0" }, @@ -723,16 +736,62 @@ "node": ">=16.0.0" } }, + "node_modules/@crawlee/jsdom/node_modules/jsdom": { + "version": "26.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.0.0.tgz", + "integrity": "sha512-BZYDGVAIriBWTpIxYzrXjv3E/4u8+/pSG5bQdIYCbNCGOvsPkDQfTVLAIXAf9ETdCpduCVTkDe2NNZ8NIwUVzw==", + "license": "MIT", + "dependencies": { + "cssstyle": "^4.2.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.4.3", + "form-data": "^4.0.1", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.16", + "parse5": "^7.2.1", + "rrweb-cssom": "^0.8.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.0.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.1.0", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/@crawlee/jsdom/node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "license": "MIT" + }, "node_modules/@crawlee/linkedom": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.12.0.tgz", - "integrity": "sha512-1Y+tDLT+802O2NGi3rhzluW7nbI5wFCvCEiYjHBt/fKM/rcaSakhNwfycG8QZn217PEw/SCy/V9Z5oymtpqoBQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.13.0.tgz", + "integrity": "sha512-ZCsw7ydIhVPdc4SWtkBh/uhPMDiP1xzah/ECKNxCn1AYgR+E8mcEHl/J9XxnPl0jV87g/qXIVwHKPWMLxkkc4A==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.12.0", - "@crawlee/types": "3.12.0", + "@crawlee/http": "3.13.0", + "@crawlee/types": "3.13.0", "linkedom": "^0.18.0", "ow": "^0.28.2", "tslib": "^2.4.0" @@ -742,13 +801,13 @@ } }, "node_modules/@crawlee/memory-storage": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.12.0.tgz", - "integrity": "sha512-PrIWH/tAw1+8kxH9+Y1eKwDuq2T+JKRiv+ltOIog0n/3tu7HTGU2Uom/1GZHHXag6jWDyBQsrHFMGSneTfhJ/w==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.13.0.tgz", + "integrity": "sha512-WC+zG2yucSL7lstrijwzm/ZkX0f/iN42x79wDkTG11ZOEGJyCFR3dmPGuNiyhK2fdGIZapLJiUolJnGOAsNtxg==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", - "@crawlee/types": "3.12.0", + "@crawlee/types": "3.13.0", "@sapphire/async-queue": "^1.5.0", "@sapphire/shapeshift": "^3.0.0", "content-type": "^1.0.4", @@ -763,19 +822,19 @@ } }, "node_modules/@crawlee/playwright": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.12.0.tgz", - "integrity": "sha512-FQYUH3oy6BNkQfB9b88ATS67e2qZcITXQSQxI/sezx6KPakRfIGUmhVuGuDrnZAe6atGBWiJDR/NRINF7hEnhA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.13.0.tgz", + "integrity": "sha512-C2mdkOp/hZQyq1/EUdovXLFmDRrWa4zGRTVaYZ178FmxwdZnVe95hpWd2ZXL5CagHCifVqhbrMDp7M4lKz28nA==", "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.0", "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.1", - "@crawlee/browser": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/core": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/browser": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/core": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "cheerio": "1.0.0-rc.12", "idcac-playwright": "^0.1.2", "jquery": "^3.6.0", @@ -799,17 +858,17 @@ } }, "node_modules/@crawlee/puppeteer": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.12.0.tgz", - "integrity": "sha512-YZTAnvLJ1HRjw02TICesWI0ZNbmu4XPdIS/Wuv+82MUTjqJ1sqiRa+pBfx3KNgiOOcOj1/P9bNmINqOmPqA+aA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.13.0.tgz", + "integrity": "sha512-UaMsmHGRtKGT4ru0gupemzIMgLt36Vb0369lRp3zRLeZrLux4cDyh7AzlHwv1cudW0gQaClujaQUP1YK2x3pXw==", "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.0", "@apify/log": "^2.4.0", - "@crawlee/browser": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/types": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/browser": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/types": "3.13.0", + "@crawlee/utils": "3.13.0", "cheerio": "1.0.0-rc.12", "devtools-protocol": "*", "idcac-playwright": "^0.1.2", @@ -830,9 +889,9 @@ } }, "node_modules/@crawlee/templates": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.12.0.tgz", - "integrity": "sha512-pr8qOmfAB7oYLYBXvxLThRV9S/Kpj74qG3vPGdJc2ax6RskooQtaa7Jqgr8vJFlTbCnbgquiko0QhvLamC0aRg==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.13.0.tgz", + "integrity": "sha512-5VM2ukcM/G7FvklnlmSMd/Th4F/Cr+lUVWSQsFJI2GKK0DnovQ1T4OhcSPFmGAjKqaIL/MT2L7KjIJNV+dA21w==", "license": "Apache-2.0", "dependencies": { "ansi-colors": "^4.1.3", @@ -896,9 +955,9 @@ } }, "node_modules/@crawlee/types": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.12.0.tgz", - "integrity": "sha512-W52s8H3jZ9H7pjxygC7mhqwdNIV29mHxJr5yujnHYJyH1iWLU6p22h0M55l11KZmUBwaRq5LqmPX6LbHNRqyoQ==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.13.0.tgz", + "integrity": "sha512-CdUecVRN2TkqwmVqJ+DiiKBj/jBBlUzsIaCZYDaRwAYEZmzVmWQ7GkcQju6YYbqtRTS+3HLyxW4bgTZdRQW98g==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.4.0" @@ -908,17 +967,17 @@ } }, "node_modules/@crawlee/utils": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.12.0.tgz", - "integrity": "sha512-8BILVHixIzAvOhHjychVnfch/ptt7dcJJ1pRLWdYvhF6Y+LmXg2KKPWltRajx0St9pXOkJaNQ11HzSK1n29wfw==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.13.0.tgz", + "integrity": "sha512-hiL+VrJ7jrExwuMHVm28EvfloonrZshhzIaV3MnocURM9B/ByILdEjuYr2bb+lRlsuk3czuhkprYbE+AsEXT9A==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/ps-tree": "^1.2.0", - "@crawlee/types": "3.12.0", + "@crawlee/types": "3.13.0", "@types/sax": "^1.2.7", "cheerio": "1.0.0-rc.12", - "file-type": "^19.0.0", + "file-type": "^20.0.0", "got-scraping": "^4.0.3", "ow": "^0.28.1", "robots-parser": "^3.0.1", @@ -930,6 +989,116 @@ "node": ">=16.0.0" } }, + "node_modules/@csstools/color-helpers": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.0.2.tgz", + "integrity": "sha512-JqWH1vsgdGcw2RR6VliXXdA0/59LttzlU8UlRT/iUUsEeWfYq8I+K0yhihEUTTHLRm1EXvpsCx3083EU15ecsA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.2.tgz", + "integrity": "sha512-TklMyb3uBB28b5uQdxjReG4L80NxAqgrECqLZFQbyLekwwlcDDS8r3f07DKqeo8C4926Br0gf/ZDe17Zv4wIuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.0.8.tgz", + "integrity": "sha512-pdwotQjCCnRPuNi06jFuP68cykU1f3ZWExLe/8MQ1LOs8Xq+fTkYgd+2V8mWUWMrOn9iS2HftPVaMZDaXzGbhQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.0.2", + "@csstools/css-calc": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.4.tgz", + "integrity": "sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.3.tgz", + "integrity": "sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@emnapi/core": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.3.1.tgz", @@ -1560,9 +1729,9 @@ } }, "node_modules/@inquirer/figures": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.8.tgz", - "integrity": "sha512-tKd+jsmhq21AP1LhexC0pPwsCxEhGgAkg28byjJAd+xhmIs8LUX8JbUc3vBf3PhLxWiB5EvyBE5X7JSPAqMAqg==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.10.tgz", + "integrity": "sha512-Ey6176gZmeqZuY/W/nZiUyvmb1/qInjcpiZjXWi6nON+nxJpD1bxtSoBxNliGISae32n6OwbY+TSXPZ1CfS4bw==", "license": "MIT", "engines": { "node": ">=18" @@ -3276,6 +3445,41 @@ "node": ">=14.16" } }, + "node_modules/@tokenizer/inflate": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", + "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "fflate": "^0.8.2", + "token-types": "^6.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/inflate/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/@tokenizer/token": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", @@ -4057,9 +4261,9 @@ "link": true }, "node_modules/apify-client": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/apify-client/-/apify-client-2.12.0.tgz", - "integrity": "sha512-h04rPVft8tNjnwZswqF2k46bdHZWsDsfOE8PkmklZ9+/s/mb/Q/dMOXCx0u2+RTc8QoAkYS9LYs97wZyUWpoag==", + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/apify-client/-/apify-client-2.12.1.tgz", + "integrity": "sha512-5K9VfSwjb2veD0ts1mScI5mVVuw/eTqF3tjyQj/jGyi7o3WIP5NI+3cXWz/aZGIQtPMLSjJ1VAVGu/of+wcmpw==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.25.0", @@ -6448,23 +6652,23 @@ } }, "node_modules/crawlee": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.12.0.tgz", - "integrity": "sha512-+3/7XQEiEt2TzhX0uENgyvDIC5CSn8QPtYMeT8Rr/qUbtkhffdmNoypMnPYYruoqlVPORb5y+BNUFAl0IpUFaA==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.13.0.tgz", + "integrity": "sha512-bcBI/FW6TqfnIU44hlkFXKgr0rOYiQbT4B1ulsL1OO9zMlUezoL5DfPUu8FVbqNBzHZk/2Kf1vt9nIxfuWNvdA==", "license": "Apache-2.0", "dependencies": { - "@crawlee/basic": "3.12.0", - "@crawlee/browser": "3.12.0", - "@crawlee/browser-pool": "3.12.0", - "@crawlee/cheerio": "3.12.0", - "@crawlee/cli": "3.12.0", - "@crawlee/core": "3.12.0", - "@crawlee/http": "3.12.0", - "@crawlee/jsdom": "3.12.0", - "@crawlee/linkedom": "3.12.0", - "@crawlee/playwright": "3.12.0", - "@crawlee/puppeteer": "3.12.0", - "@crawlee/utils": "3.12.0", + "@crawlee/basic": "3.13.0", + "@crawlee/browser": "3.13.0", + "@crawlee/browser-pool": "3.13.0", + "@crawlee/cheerio": "3.13.0", + "@crawlee/cli": "3.13.0", + "@crawlee/core": "3.13.0", + "@crawlee/http": "3.13.0", + "@crawlee/jsdom": "3.13.0", + "@crawlee/linkedom": "3.13.0", + "@crawlee/playwright": "3.13.0", + "@crawlee/puppeteer": "3.13.0", + "@crawlee/utils": "3.13.0", "import-local": "^3.1.0", "tslib": "^2.4.0" }, @@ -6550,17 +6754,24 @@ "license": "MIT" }, "node_modules/cssstyle": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.1.0.tgz", - "integrity": "sha512-h66W1URKpBS5YMI/V8PyXvTMFT8SupJ1IzoIV8IeBC/ji8WVmrO8dGlTi+2dh6whmdk6BiKJLD/ZBkhWbcg6nA==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.2.1.tgz", + "integrity": "sha512-9+vem03dMXG7gDmZ62uqmRiMRNtinIZ9ZyuF6BdxzfOD+FdN5hretzynkn0ReS2DO2GSw76RWHs0UmJPI2zUjw==", "license": "MIT", "dependencies": { - "rrweb-cssom": "^0.7.1" + "@asamuzakjp/css-color": "^2.8.2", + "rrweb-cssom": "^0.8.0" }, "engines": { "node": ">=18" } }, + "node_modules/cssstyle/node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "license": "MIT" + }, "node_modules/csv-stringify": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.2.tgz", @@ -7056,9 +7267,9 @@ } }, "node_modules/domutils": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", - "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", "license": "BSD-2-Clause", "dependencies": { "dom-serializer": "^2.0.0", @@ -8463,6 +8674,12 @@ "pend": "~1.2.0" } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, "node_modules/figlet": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/figlet/-/figlet-1.8.0.tgz", @@ -8513,15 +8730,15 @@ } }, "node_modules/file-type": { - "version": "19.6.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-19.6.0.tgz", - "integrity": "sha512-VZR5I7k5wkD0HgFnMsq5hOsSc710MJMu5Nc5QYsbe38NN5iPV/XTObYLc/cpttRTf6lX538+5uO1ZQRhYibiZQ==", + "version": "20.4.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.4.0.tgz", + "integrity": "sha512-+NZeExsi4G6EWaMbSmvBeCoqsj9EqNvOj1o/0uPVPW4O51FSCmxFlNEp/PitsqBMCbax4cGoaYmnUK5FLTuG4g==", "license": "MIT", "dependencies": { - "get-stream": "^9.0.1", - "strtok3": "^9.0.1", + "@tokenizer/inflate": "^0.2.6", + "strtok3": "^10.2.0", "token-types": "^6.0.0", - "uint8array-extras": "^1.3.0" + "uint8array-extras": "^1.4.0" }, "engines": { "node": ">=18" @@ -11371,6 +11588,7 @@ "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.1.tgz", "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==", "license": "MIT", + "peer": true, "dependencies": { "cssstyle": "^4.1.0", "data-urls": "^5.0.0", @@ -11949,18 +12167,49 @@ } }, "node_modules/linkedom": { - "version": "0.18.5", - "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.5.tgz", - "integrity": "sha512-JGLaGGtqtu+eOhYrC1wkWYTBcpVWL4AsnwAtMtgO1Q0gI0PuPJKI0zBBE+a/1BrhOE3Uw8JI/ycByAv5cLrAuQ==", + "version": "0.18.9", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.9.tgz", + "integrity": "sha512-Pfvhwjs46nBrcQdauQjMXDJZqj6VwN7KStT84xQqmIgD9bPH6UVJ/ESW8y4VHVF2h7di0/P+f4Iln4U5emRcmg==", "license": "ISC", "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", - "htmlparser2": "^9.1.0", + "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" } }, + "node_modules/linkedom/node_modules/entities": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.0.tgz", + "integrity": "sha512-aKstq2TDOndCn4diEyp9Uq/Flu2i1GlLkc6XIDQSDMuaFE3OPW5OphLCyQ5SpSJZTb4reN+kTcYru5yIfXoRPw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/linkedom/node_modules/htmlparser2": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", + "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.1", + "entities": "^6.0.0" + } + }, "node_modules/lint-staged": { "version": "15.2.10", "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-15.2.10.tgz", @@ -12658,7 +12907,6 @@ "version": "10.4.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true, "license": "ISC" }, "node_modules/magic-string": { @@ -13565,9 +13813,9 @@ } }, "node_modules/nwsapi": { - "version": "2.2.13", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.13.tgz", - "integrity": "sha512-cTGB9ptp9dY9A5VbMSe7fQBcl/tt22Vcqdq8+eN93rblOuE0aCFu4aZ2vMwct/2t+lFnosm8RkQW1I0Omb1UtQ==", + "version": "2.2.18", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.18.tgz", + "integrity": "sha512-p1TRH/edngVEHVbwqWnxUViEmq5znDvyB+Sik5cmuLpGOIfDf/39zLiq3swPF8Vakqn+gvNiOQAZu8djYlQILA==", "license": "MIT" }, "node_modules/nx": { @@ -14499,12 +14747,12 @@ } }, "node_modules/peek-readable": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.3.1.tgz", - "integrity": "sha512-GVlENSDW6KHaXcd9zkZltB7tCLosKB/4Hg0fqBJkAoBgYG2Tn1xtMgXtSUuMU9AK/gCm/tTdT8mgAeF4YNeeqw==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-6.1.1.tgz", + "integrity": "sha512-7QmvgRKhxM0E2PGV4ocfROItVode+ELI27n4q+lpufZ+tRKBu/pBP8WOmw9HXn2ui/AUizqtvaVQhcJrOkRqYg==", "license": "MIT", "engines": { - "node": ">=14.16" + "node": ">=18" }, "funding": { "type": "github", @@ -15800,7 +16048,8 @@ "version": "0.7.1", "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz", "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/run-async": { "version": "2.4.1", @@ -17015,16 +17264,16 @@ } }, "node_modules/strtok3": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-9.0.1.tgz", - "integrity": "sha512-ERPW+XkvX9W2A+ov07iy+ZFJpVdik04GhDA4eVogiG9hpC97Kem2iucyzhFxbFRvQ5o2UckFtKZdp1hkGvnrEw==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.2.1.tgz", + "integrity": "sha512-Q2dTnW3UXokAvXmXvrvMoUj/me3LyJI76HNHeuGMh2o0As/vzd7eHV3ncLOyvu928vQIDbE7Vf9ldEnC7cwy1w==", "license": "MIT", "dependencies": { "@tokenizer/token": "^0.3.0", - "peek-readable": "^5.3.1" + "peek-readable": "^6.1.1" }, "engines": { - "node": ">=16" + "node": ">=18" }, "funding": { "type": "github", @@ -18794,9 +19043,9 @@ } }, "node_modules/whatwg-url": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz", - "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.1.1.tgz", + "integrity": "sha512-mDGf9diDad/giZ/Sm9Xi2YcyzaFpbdLpJPr+E9fSkyQ7KpQD4SdFcugkRQYzhmfI4KeV4Qpnn2sKPdo+kmsgRQ==", "license": "MIT", "dependencies": { "tr46": "^5.0.0", @@ -19504,11 +19753,11 @@ "@apify/input_secrets": "^1.1.40", "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.9.3", - "@crawlee/core": "^3.9.0", - "@crawlee/types": "^3.9.0", - "@crawlee/utils": "^3.9.0", - "apify-client": "^2.12.0", + "@apify/utilities": "^2.13.0", + "@crawlee/core": "^3.13.0", + "@crawlee/types": "^3.13.0", + "@crawlee/utils": "^3.13.0", + "apify-client": "^2.12.1", "fs-extra": "^11.2.0", "ow": "^0.28.2", "semver": "^7.5.4", diff --git a/package.json b/package.json index 37e153a5fb..537134cd57 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.11.5", + "crawlee": "^3.13.0", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", diff --git a/packages/apify/package.json b/packages/apify/package.json index 8cc37f5193..04e982c9c2 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -58,15 +58,15 @@ "@apify/input_secrets": "^1.1.40", "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.9.3", - "@crawlee/core": "^3.9.0", - "@crawlee/types": "^3.9.0", - "@crawlee/utils": "^3.9.0", - "apify-client": "^2.12.0", + "@apify/utilities": "^2.13.0", + "@crawlee/core": "^3.13.0", + "@crawlee/types": "^3.13.0", + "@crawlee/utils": "^3.13.0", + "apify-client": "^2.12.1", "fs-extra": "^11.2.0", "ow": "^0.28.2", "semver": "^7.5.4", "tslib": "^2.6.2", "ws": "^8.18.0" } -} \ No newline at end of file +} diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index d22ae446d2..8619283fc1 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -16,6 +16,9 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { actorRunId?: string; actorTaskId?: string; apiBaseUrl?: string; + // apiBaseUrl is the internal API URL, accessible only within the platform(private network), + // while apiPublicBaseUrl is the public API URL, available externally(through internet). + apiPublicBaseUrl?: string; containerPort?: number; containerUrl?: string; proxyHostname?: string; @@ -139,6 +142,7 @@ export class Configuration extends CoreConfiguration { APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl', APIFY_ACTOR_ID: 'actorId', APIFY_API_BASE_URL: 'apiBaseUrl', + APIFY_API_PUBLIC_BASE_URL: 'apiPublicBaseUrl', APIFY_IS_AT_HOME: 'isAtHome', APIFY_ACTOR_RUN_ID: 'actorRunId', APIFY_ACTOR_TASK_ID: 'actorTaskId', @@ -183,6 +187,7 @@ export class Configuration extends CoreConfiguration { defaultRequestQueueId: LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID], inputKey: 'INPUT', apiBaseUrl: 'https://api.apify.com', + apiPublicBaseUrl: 'https://api.apify.com', proxyStatusUrl: 'http://proxy.apify.com', proxyHostname: LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME], proxyPort: +LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT], diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index e19b713d89..6efa2c9146 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -1,3 +1,4 @@ +import { createHmacSignature } from '@apify/utilities'; import type { StorageManagerOptions } from '@crawlee/core'; import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core'; @@ -15,11 +16,18 @@ export class KeyValueStore extends CoreKeyValueStore { * access the value in the remote key-value store. */ override getPublicUrl(key: string): string { - if (!(this.config as Configuration).get('isAtHome') && getPublicUrl) { + const config = this.config as Configuration; + if (!config.get('isAtHome') && getPublicUrl) { return getPublicUrl.call(this, key); } - return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`; + const publicUrl = new URL(`${config.get('apiPublicBaseUrl')}/v2/key-value-stores/${this.id}/records/${key}`); + + if (this.storageObject?.urlSigningSecretKey) { + publicUrl.searchParams.append('signature', createHmacSignature(this.storageObject.urlSigningSecretKey as string, key)); + } + + return publicUrl.toString(); } /** diff --git a/test/e2e/sdk/publicUrl/src/main.mjs b/test/e2e/sdk/publicUrl/src/main.mjs new file mode 100644 index 0000000000..dbe5b5a8e6 --- /dev/null +++ b/test/e2e/sdk/publicUrl/src/main.mjs @@ -0,0 +1,19 @@ +import { Actor, log } from 'apify'; + +await Actor.init(); + +const { data } = await Actor.getInput(); + +await Actor.setValue('public-record-key', JSON.stringify(data), { contentType: `application/json` }); + +const defaultKeyValueStore = await Actor.openKeyValueStore(); +const publicUrl = defaultKeyValueStore.getPublicUrl('public-record-key'); + +// Here we store the url itself +await Actor.setValue('urlToPublicData', publicUrl); + +log.info('Generated public url', { publicUrl }); + +await Actor.pushData({ publicUrl }); + +await Actor.exit(); diff --git a/test/e2e/sdk/publicUrl/test.mjs b/test/e2e/sdk/publicUrl/test.mjs new file mode 100644 index 0000000000..b7ee380814 --- /dev/null +++ b/test/e2e/sdk/publicUrl/test.mjs @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; + +import { ApifyClient } from 'apify'; + +const PUBLIC_DATA = { exposedData: 'test' }; + +const client = new ApifyClient({ + token: process.env.APIFY_TOKEN, +}); + +const actor = client.actor(process.argv[2]); + +const run = await actor.call({ data: PUBLIC_DATA }, { waitSecs: 15 }); +assert.equal(run.exitCode, 0); + +const publicUrl = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('urlToPublicData'); +const data = await fetch(publicUrl.value).then((res) => res.json()); + +assert.deepEqual(data, PUBLIC_DATA);