fix(downloader): create directories for identically named files (#178)

When a dataset contains files with the same file name in different
directories (e.g. 0002/AO2D.root, 0003/AO2D.root), the downloader
now preserves the directory structure instead of overwriting files.
The common directory prefix is stripped to keep paths minimal.

Closes #177

Author: tiborsimko

cernopendata_client/cli.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # This file is part of cernopendata-client.
 #
-# Copyright (C) 2019, 2020, 2021, 2023, 2025 CERN.
+# Copyright (C) 2019, 2020, 2021, 2023, 2025, 2026 CERN.
 #
 # cernopendata-client is free software; you can redistribute it and/or modify
 # it under the terms of the GPLv3 license; see LICENSE file for more details.
@@ -29,6 +29,8 @@
     get_download_files_by_name,
     get_download_files_by_range,
     get_download_files_by_regexp,
+    get_download_path,
+    get_file_subdirectories,
 )
 from .validator import (
     validate_range,
@@ -355,21 +357,23 @@ def download_files(
         sys.exit(0)
 
     total_files = len(download_file_locations)
-    path = record_recid
-    if not os.path.isdir(path):
+    base_path = record_recid
+    if not os.path.isdir(base_path):
         try:
-            os.mkdir(path)
+            os.mkdir(base_path)
         except OSError:
             display_message(
                 msg_type="error",
-                msg="Creation of the directory {} failed".format(path),
+                msg="Creation of the directory {} failed".format(base_path),
             )
+    file_subdirs = get_file_subdirectories(download_file_locations)
     if not download_engine:
         if protocol.startswith("http"):
             download_engine = "requests"
         elif protocol == "xrootd":
             download_engine = "xrootd"
     for file_location in download_file_locations:
+        path = get_download_path(base_path, file_location, file_subdirs)
         display_message(
             msg_type="info",
             msg="Downloading file {} of {}".format(

cernopendata_client/config.py

@@ -8,7 +8,6 @@
 
 """cernopendata-client configuration."""
 
-
 SERVER_HTTP_URI = "http://opendata.cern.ch"
 """Default CERN Open Data server to query over HTTP protocol."""
 

cernopendata_client/downloader.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # This file is part of cernopendata-client.
 #
-# Copyright (C) 2020, 2021 CERN.
+# Copyright (C) 2020, 2021, 2026 CERN.
 #
 # cernopendata-client is free software; you can redistribute it and/or modify
 # it under the terms of the GPLv3 license; see LICENSE file for more details.
@@ -360,6 +360,69 @@ def download_single_file(
     return
 
 
+def get_file_subdirectories(file_locations):
+    """Return a mapping of file locations to subdirectory paths for disambiguation.
+
+    When multiple files share the same file name, compute subdirectory paths
+    by stripping the longest common directory prefix from the URL paths.
+    This preserves the original directory structure relative to the common root.
+
+    :param file_locations: List of remote file locations (URLs)
+    :type file_locations: list
+
+    :return: Dictionary mapping each file location to its subdirectory (empty
+        string if no subdirectory is needed)
+    :rtype: dict
+    """
+    from collections import Counter
+
+    file_names = [loc.split("/")[-1] for loc in file_locations]
+    file_name_counts = Counter(file_names)
+    has_duplicates = any(count > 1 for count in file_name_counts.values())
+
+    if not has_duplicates:
+        return {loc: "" for loc in file_locations}
+
+    # Split each URL into directory components (excluding the filename)
+    dir_parts_list = [loc.split("/")[:-1] for loc in file_locations]
+    # Find the longest common prefix of all directory paths
+    common_prefix_len = 0
+    if dir_parts_list:
+        min_len = min(len(parts) for parts in dir_parts_list)
+        for i in range(min_len):
+            if len(set(parts[i] for parts in dir_parts_list)) == 1:
+                common_prefix_len = i + 1
+            else:
+                break
+    # Build subdirectory for each file by stripping the common prefix
+    result = {}
+    for loc, dir_parts in zip(file_locations, dir_parts_list):
+        subdir = "/".join(dir_parts[common_prefix_len:])
+        result[loc] = subdir
+    return result
+
+
+def get_download_path(base_path, file_location, file_subdirs):
+    """Return download path for a file, creating subdirectories if needed.
+
+    :param base_path: Base directory for downloads (e.g. record ID)
+    :param file_location: Remote file location URL
+    :param file_subdirs: Mapping from file locations to subdirectory paths
+    :type base_path: str
+    :type file_location: str
+    :type file_subdirs: dict
+
+    :return: Local directory path where the file should be saved
+    :rtype: str
+    """
+    subdir = file_subdirs[file_location]
+    if subdir:
+        path = os.path.join(base_path, subdir)
+        os.makedirs(path, exist_ok=True)
+        return path
+    return base_path
+
+
 def get_download_files_by_name(names=None, file_locations=None):
     """Return the files filtered by file names.
 

tests/test_downloader.py

@@ -2,7 +2,7 @@
 #
 # This file is part of cernopendata-client.
 #
-# Copyright (C) 2025 CERN.
+# Copyright (C) 2025, 2026 CERN.
 #
 # cernopendata-client is free software; you can redistribute it and/or modify
 # it under the terms of the GPLv3 license; see LICENSE file for more details.
@@ -15,6 +15,7 @@
     get_download_files_by_name,
     get_download_files_by_regexp,
     get_download_files_by_range,
+    get_file_subdirectories,
 )
 
 
@@ -180,3 +181,80 @@ def test_get_download_files_by_range_with_filtered_files():
         ranges=["1-2"], file_locations=file_locations, filtered_files=filtered_files
     )
     assert result == ["http://example.com/file1.txt", "http://example.com/file3.txt"]
+
+
+@pytest.mark.local
+def test_get_file_subdirectories_unique_names():
+    """Test that unique file names produce no subdirectories."""
+    file_locations = [
+        "http://example.com/dir1/a.txt",
+        "http://example.com/dir2/b.txt",
+        "http://example.com/dir3/c.txt",
+    ]
+    result = get_file_subdirectories(file_locations)
+    assert all(subdir == "" for subdir in result.values())
+
+
+@pytest.mark.local
+def test_get_file_subdirectories_duplicate_names_one_level():
+    """Test that duplicate names with different parent dirs use one level."""
+    file_locations = [
+        "http://opendata.cern.ch/eos/opendata/alice/2015/LHC15o/000245349/0002/AO2D.root",
+        "http://opendata.cern.ch/eos/opendata/alice/2015/LHC15o/000245349/0003/AO2D.root",
+        "http://opendata.cern.ch/eos/opendata/alice/2015/LHC15o/000245349/0004/AO2D.root",
+    ]
+    result = get_file_subdirectories(file_locations)
+    assert result[file_locations[0]] == "0002"
+    assert result[file_locations[1]] == "0003"
+    assert result[file_locations[2]] == "0004"
+
+
+@pytest.mark.local
+def test_get_file_subdirectories_duplicate_names_two_levels():
+    """Test that deeper disambiguation uses multiple directory levels."""
+    file_locations = [
+        "http://opendata.cern.ch/eos/opendata/alice/000245349/0002/AO2D.root",
+        "http://opendata.cern.ch/eos/opendata/alice/000245350/0002/AO2D.root",
+    ]
+    result = get_file_subdirectories(file_locations)
+    assert result[file_locations[0]] == "000245349/0002"
+    assert result[file_locations[1]] == "000245350/0002"
+
+
+@pytest.mark.local
+def test_get_file_subdirectories_mixed_duplicates():
+    """Test mixed unique and duplicate file names."""
+    file_locations = [
+        "http://example.com/dir1/0001/data.root",
+        "http://example.com/dir1/0002/data.root",
+        "http://example.com/dir1/unique.txt",
+    ]
+    result = get_file_subdirectories(file_locations)
+    # Common prefix is "http://example.com/dir1", so subdirs are relative to that
+    assert result[file_locations[0]] == "0001"
+    assert result[file_locations[1]] == "0002"
+    assert result[file_locations[2]] == ""
+
+
+@pytest.mark.local
+def test_get_file_subdirectories_varying_depths():
+    """Test files at varying directory depths with common prefix stripping."""
+    file_locations = [
+        "http://opendata.cern.ch/eos/opendata/alice/mydata/foo/bar/data.root",
+        "http://opendata.cern.ch/eos/opendata/alice/mydata/data.root",
+        "http://opendata.cern.ch/eos/opendata/alice/mydata/foo/baz/data.root",
+        "http://opendata.cern.ch/eos/opendata/alice/mydata/foo/data.root",
+    ]
+    result = get_file_subdirectories(file_locations)
+    # Common prefix is "http://opendata.cern.ch/eos/opendata/alice/mydata"
+    assert result[file_locations[0]] == "foo/bar"
+    assert result[file_locations[1]] == ""
+    assert result[file_locations[2]] == "foo/baz"
+    assert result[file_locations[3]] == "foo"
+
+
+@pytest.mark.local
+def test_get_file_subdirectories_empty_list():
+    """Test with an empty file list."""
+    result = get_file_subdirectories([])
+    assert result == {}