fix: Add storages name validation (#1457)

Mantisus · web-flow · commit 84de11a3a603 · 2025-10-07T19:54:31.000+02:00
### Description - Added name validation for storages in accordance with Apify platform restrictions ### Issues - Closes: #1434 - Relates: #1354 ### Testing - Added new tests to verify correct validation.
diff --git a/docs/upgrading/upgrading_to_v1.md b/docs/upgrading/upgrading_to_v1.md
@@ -333,3 +333,7 @@ async def main() -> None:
 
     await crawler.run(['https://crawlee.dev/'])
 ```
+
+### New storage naming restrictions
+
+We've introduced naming restrictions for storages to ensure compatibility with Apify Platform requirements and prevent potential conflicts. Storage names may include only letters (a–z, A–Z), digits (0–9), and hyphens (-), with hyphens allowed only in the middle of the name (for example, my-storage-1).
diff --git a/src/crawlee/storages/_base.py b/src/crawlee/storages/_base.py
@@ -44,7 +44,9 @@ async def open(
 
         Args:
             id: The storage ID.
-            name: The storage name (global scope, persists across runs).
+            name: The storage name (global scope, persists across runs). Name can only contain letters "a" through "z",
+                the digits "0" through "9", and the hyphen ("-") but only in the middle of the string
+                (e.g. "my-value-1").
             alias: The storage alias (run scope, creates unnamed storage).
             configuration: Configuration object used during the storage creation or restoration process.
             storage_client: Underlying storage client to use. If not provided, the default global storage client
diff --git a/src/crawlee/storages/_dataset.py b/src/crawlee/storages/_dataset.py
@@ -12,6 +12,7 @@
 
 from ._base import Storage
 from ._key_value_store import KeyValueStore
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -75,6 +76,8 @@ def __init__(self, client: DatasetClient, id: str, name: str | None) -> None:
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name
diff --git a/src/crawlee/storages/_key_value_store.py b/src/crawlee/storages/_key_value_store.py
@@ -15,6 +15,7 @@
 from crawlee.storage_clients.models import KeyValueStoreMetadata
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -84,6 +85,8 @@ def __init__(self, client: KeyValueStoreClient, id: str, name: str | None) -> No
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name
diff --git a/src/crawlee/storages/_request_queue.py b/src/crawlee/storages/_request_queue.py
@@ -13,6 +13,7 @@
 from crawlee.request_loaders import RequestManager
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -80,6 +81,8 @@ def __init__(self, client: RequestQueueClient, id: str, name: str | None) -> Non
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name
diff --git a/src/crawlee/storages/_storage_instance_manager.py b/src/crawlee/storages/_storage_instance_manager.py
@@ -8,6 +8,8 @@
 from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, RequestQueueClient
 
+from ._utils import validate_storage_name
+
 if TYPE_CHECKING:
     from ._base import Storage
 
@@ -90,7 +92,9 @@ async def open_storage_instance(
         Args:
             cls: The storage class to instantiate.
             id: Storage ID.
-            name: Storage name. (global scope, persists across runs).
+            name: Storage name. (global scope, persists across runs). Name can only contain letters "a" through "z",
+                the digits "0" through "9", and the hyphen ("-") but only in the middle of the string
+                (e.g. "my-value-1").
             alias: Storage alias (run scope, creates unnamed storage).
             client_opener_coro: Coroutine to open the storage client when storage instance not found in cache.
             storage_client_cache_key: Additional optional key from storage client to differentiate cache entries.
@@ -146,6 +150,10 @@ async def open_storage_instance(
                     f'Use a different name or drop the existing alias storage first.'
                 )
 
+            # Validate storage name
+            if name is not None:
+                validate_storage_name(name)
+
             # Create new instance
             client: KeyValueStoreClient | DatasetClient | RequestQueueClient
             client = await client_opener_coro
diff --git a/src/crawlee/storages/_utils.py b/src/crawlee/storages/_utils.py
@@ -0,0 +1,11 @@
+import re
+
+NAME_REGEX = re.compile(r'^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])$')
+
+
+def validate_storage_name(name: str | None) -> None:
+    if name and not NAME_REGEX.match(name):
+        raise ValueError(
+            f'Invalid storage name "{name}". Name can only contain letters "a" through "z", the digits "0" through'
+            '"9", and the hyphen ("-") but only in the middle of the string (e.g. "my-value-1")'
+        )
diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
@@ -473,7 +473,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
 async def test_adaptive_playwright_crawler_statistics_in_init() -> None:
     """Tests that adaptive crawler uses created AdaptivePlaywrightCrawlerStatistics from inputted Statistics."""
     persistence_enabled = True
-    persist_state_kvs_name = 'some name'
+    persist_state_kvs_name = 'some-name'
     persist_state_key = 'come key'
     log_message = 'some message'
     periodic_message_logger = logging.getLogger('some logger')
diff --git a/tests/unit/otel/test_crawler_instrumentor.py b/tests/unit/otel/test_crawler_instrumentor.py
@@ -38,7 +38,7 @@ async def test_crawler_instrumentor_capability(server_url: URL) -> None:
 
     # Generate first telemetry data from `Dataset` public methods.
     # `Dataset` is in `instrument_classes` argument, and thus it's public methods are instrumented.
-    dataset = await Dataset.open(name='test_dataset')
+    dataset = await Dataset.open(name='test-dataset')
     await dataset.drop()
 
     # Other traces will be from crawler run.
diff --git a/tests/unit/sessions/test_session_pool.py b/tests/unit/sessions/test_session_pool.py
@@ -17,7 +17,7 @@
     from collections.abc import AsyncGenerator
 
 MAX_POOL_SIZE = 3
-KVS_NAME = 'test_session_pool'
+KVS_NAME = 'test-session-pool'
 PERSIST_STATE_KEY = 'crawlee_session_pool_state'
 
 
diff --git a/tests/unit/storage_clients/_file_system/test_fs_dataset_client.py b/tests/unit/storage_clients/_file_system/test_fs_dataset_client.py
@@ -27,14 +27,14 @@ def configuration(tmp_path: Path) -> Configuration:
 @pytest.fixture
 async def dataset_client(configuration: Configuration) -> AsyncGenerator[FileSystemDatasetClient, None]:
     """A fixture for a file system dataset client."""
-    client = await FileSystemStorageClient().create_dataset_client(name='test_dataset', configuration=configuration)
+    client = await FileSystemStorageClient().create_dataset_client(name='test-dataset', configuration=configuration)
     yield client
     await client.drop()
 
 
 async def test_file_and_directory_creation(configuration: Configuration) -> None:
     """Test that file system dataset creates proper files and directories."""
-    client = await FileSystemStorageClient().create_dataset_client(name='new_dataset', configuration=configuration)
+    client = await FileSystemStorageClient().create_dataset_client(name='new-dataset', configuration=configuration)
 
     # Verify files were created
     assert client.path_to_dataset.exists()
@@ -45,7 +45,7 @@ async def test_file_and_directory_creation(configuration: Configuration) -> None
         metadata = json.load(f)
         client_metadata = await client.get_metadata()
         assert metadata['id'] == client_metadata.id
-        assert metadata['name'] == 'new_dataset'
+        assert metadata['name'] == 'new-dataset'
         assert metadata['item_count'] == 0
 
     await client.drop()
diff --git a/tests/unit/storage_clients/_file_system/test_fs_kvs_client.py b/tests/unit/storage_clients/_file_system/test_fs_kvs_client.py
@@ -27,14 +27,14 @@ def configuration(tmp_path: Path) -> Configuration:
 @pytest.fixture
 async def kvs_client(configuration: Configuration) -> AsyncGenerator[FileSystemKeyValueStoreClient, None]:
     """A fixture for a file system key-value store client."""
-    client = await FileSystemStorageClient().create_kvs_client(name='test_kvs', configuration=configuration)
+    client = await FileSystemStorageClient().create_kvs_client(name='test-kvs', configuration=configuration)
     yield client
     await client.drop()
 
 
 async def test_file_and_directory_creation(configuration: Configuration) -> None:
     """Test that file system KVS creates proper files and directories."""
-    client = await FileSystemStorageClient().create_kvs_client(name='new_kvs', configuration=configuration)
+    client = await FileSystemStorageClient().create_kvs_client(name='new-kvs', configuration=configuration)
 
     # Verify files were created
     assert client.path_to_kvs.exists()
@@ -44,7 +44,7 @@ async def test_file_and_directory_creation(configuration: Configuration) -> None
     with client.path_to_metadata.open() as f:
         metadata = json.load(f)
         assert metadata['id'] == (await client.get_metadata()).id
-        assert metadata['name'] == 'new_kvs'
+        assert metadata['name'] == 'new-kvs'
 
     await client.drop()
 
diff --git a/tests/unit/storage_clients/_file_system/test_fs_rq_client.py b/tests/unit/storage_clients/_file_system/test_fs_rq_client.py
@@ -28,15 +28,15 @@ def configuration(tmp_path: Path) -> Configuration:
 async def rq_client() -> AsyncGenerator[FileSystemRequestQueueClient, None]:
     """A fixture for a file system request queue client."""
     client = await FileSystemStorageClient().create_rq_client(
-        name='test_request_queue',
+        name='test-request-queue',
     )
     yield client
     await client.drop()
 
 
 async def test_file_and_directory_creation() -> None:
     """Test that file system RQ creates proper files and directories."""
-    client = await FileSystemStorageClient().create_rq_client(name='new_request_queue')
+    client = await FileSystemStorageClient().create_rq_client(name='new-request-queue')
 
     # Verify files were created
     assert client.path_to_rq.exists()
@@ -46,7 +46,7 @@ async def test_file_and_directory_creation() -> None:
     with client.path_to_metadata.open() as f:
         metadata = json.load(f)
         assert metadata['id'] == (await client.get_metadata()).id
-        assert metadata['name'] == 'new_request_queue'
+        assert metadata['name'] == 'new-request-queue'
 
     await client.drop()
 
diff --git a/tests/unit/storage_clients/_memory/test_memory_dataset_client.py b/tests/unit/storage_clients/_memory/test_memory_dataset_client.py
@@ -16,7 +16,7 @@
 @pytest.fixture
 async def dataset_client() -> AsyncGenerator[MemoryDatasetClient, None]:
     """Fixture that provides a fresh memory dataset client for each test."""
-    client = await MemoryStorageClient().create_dataset_client(name='test_dataset')
+    client = await MemoryStorageClient().create_dataset_client(name='test-dataset')
     yield client
     await client.drop()
 
@@ -25,7 +25,7 @@ async def test_memory_specific_purge_behavior() -> None:
     """Test memory-specific purge behavior and in-memory storage characteristics."""
     # Create dataset and add data
     dataset_client1 = await MemoryStorageClient().create_dataset_client(
-        name='test_purge_dataset',
+        name='test-purge-dataset',
     )
     await dataset_client1.push_data({'item': 'initial data'})
 
@@ -35,7 +35,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Reopen with same storage client instance
     dataset_client2 = await MemoryStorageClient().create_dataset_client(
-        name='test_purge_dataset',
+        name='test-purge-dataset',
     )
 
     # Verify data was purged (memory storage specific behavior)
diff --git a/tests/unit/storage_clients/_memory/test_memory_kvs_client.py b/tests/unit/storage_clients/_memory/test_memory_kvs_client.py
@@ -16,7 +16,7 @@
 @pytest.fixture
 async def kvs_client() -> AsyncGenerator[MemoryKeyValueStoreClient, None]:
     """Fixture that provides a fresh memory key-value store client for each test."""
-    client = await MemoryStorageClient().create_kvs_client(name='test_kvs')
+    client = await MemoryStorageClient().create_kvs_client(name='test-kvs')
     yield client
     await client.drop()
 
@@ -26,7 +26,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Create KVS and add data
     kvs_client1 = await MemoryStorageClient().create_kvs_client(
-        name='test_purge_kvs',
+        name='test-purge-kvs',
     )
     await kvs_client1.set_value(key='test-key', value='initial value')
 
@@ -37,7 +37,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Reopen with same storage client instance
     kvs_client2 = await MemoryStorageClient().create_kvs_client(
-        name='test_purge_kvs',
+        name='test-purge-kvs',
     )
 
     # Verify value was purged (memory storage specific behavior)
diff --git a/tests/unit/storage_clients/_memory/test_memory_rq_client.py b/tests/unit/storage_clients/_memory/test_memory_rq_client.py
@@ -17,7 +17,7 @@
 @pytest.fixture
 async def rq_client() -> AsyncGenerator[MemoryRequestQueueClient, None]:
     """Fixture that provides a fresh memory request queue client for each test."""
-    client = await MemoryStorageClient().create_rq_client(name='test_rq')
+    client = await MemoryStorageClient().create_rq_client(name='test-rq')
     yield client
     await client.drop()
 
@@ -26,7 +26,7 @@ async def test_memory_specific_purge_behavior() -> None:
     """Test memory-specific purge behavior and in-memory storage characteristics."""
     # Create RQ and add data
     rq_client1 = await MemoryStorageClient().create_rq_client(
-        name='test_purge_rq',
+        name='test-purge-rq',
     )
     request = Request.from_url(url='https://example.com/initial')
     await rq_client1.add_batch_of_requests([request])
@@ -36,7 +36,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Reopen with same storage client instance
     rq_client2 = await MemoryStorageClient().create_rq_client(
-        name='test_purge_rq',
+        name='test-purge-rq',
     )
 
     # Verify queue was purged (memory storage specific behavior)
diff --git a/tests/unit/storage_clients/_sql/test_sql_dataset_client.py b/tests/unit/storage_clients/_sql/test_sql_dataset_client.py
@@ -44,7 +44,7 @@ async def dataset_client(
     async with SqlStorageClient() as storage_client:
         monkeypatch.setattr(storage_client, '_accessed_modified_update_interval', timedelta(seconds=0))
         client = await storage_client.create_dataset_client(
-            name='test_dataset',
+            name='test-dataset',
             configuration=configuration,
         )
         yield client
@@ -57,7 +57,7 @@ async def test_create_tables_with_connection_string(configuration: Configuration
 
     async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client:
         await storage_client.create_dataset_client(
-            name='new_dataset',
+            name='new-dataset',
             configuration=configuration,
         )
 
@@ -75,7 +75,7 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
 
     async with SqlStorageClient(engine=engine) as storage_client:
         await storage_client.create_dataset_client(
-            name='new_dataset',
+            name='new-dataset',
             configuration=configuration,
         )
 
@@ -89,7 +89,7 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL dataset creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
         client = await storage_client.create_dataset_client(
-            name='new_dataset',
+            name='new-dataset',
             configuration=configuration,
         )
 
@@ -101,12 +101,12 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
             assert 'datasets' in tables
 
         async with client.get_session() as session:
-            stmt = select(DatasetMetadataDb).where(DatasetMetadataDb.name == 'new_dataset')
+            stmt = select(DatasetMetadataDb).where(DatasetMetadataDb.name == 'new-dataset')
             result = await session.execute(stmt)
             orm_metadata = result.scalar_one_or_none()
             assert orm_metadata is not None
             assert orm_metadata.id == client_metadata.id
-            assert orm_metadata.name == 'new_dataset'
+            assert orm_metadata.name == 'new-dataset'
             assert orm_metadata.item_count == 0
 
         await client.drop()
diff --git a/tests/unit/storage_clients/_sql/test_sql_kvs_client.py b/tests/unit/storage_clients/_sql/test_sql_kvs_client.py
diff --git a/tests/unit/storage_clients/_sql/test_sql_rq_client.py b/tests/unit/storage_clients/_sql/test_sql_rq_client.py
diff --git a/tests/unit/storages/test_dataset.py b/tests/unit/storages/test_dataset.py
diff --git a/tests/unit/storages/test_key_value_store.py b/tests/unit/storages/test_key_value_store.py
diff --git a/tests/unit/storages/test_request_queue.py b/tests/unit/storages/test_request_queue.py
