refactor (DataManager): getReplicasFromDirectory and __cleanDirectory use getDirectoryDump; remove __getCatalogDirectoryContents

Author: chaen

src/DIRAC/DataManagementSystem/Client/DataManager.py

@@ -160,24 +160,22 @@ def __cleanDirectory(self, folder):
             errStr = "Write access not permitted for this credential."
             log.debug(errStr, folder)
             return S_ERROR(errStr)
-        res = self.__getCatalogDirectoryContents([folder], includeDirectories=True)
+
+        res = returnSingleResult(self.fileCatalog.getDirectoryDump([folder]))
+
         if not res["OK"]:
             return res
 
-        if not res["Value"]:
+        if not (res["Value"]["Files"] or res["Value"]["SubDirs"]):
             # folder is empty, just remove it and return
             res = returnSingleResult(self.fileCatalog.removeDirectory(folder, recursive=True))
             return res
 
         # create a list of folders so that empty folders are also deleted
-        areDirs = self.fileCatalog.isDirectory(res["Value"])
-        if not areDirs["OK"]:
-            return areDirs
-        listOfFolders = [aDir for aDir in areDirs["Value"]["Successful"] if areDirs["Value"]["Successful"][aDir]]
-        for lfn in listOfFolders:
-            res["Value"].pop(lfn)
-
-        res = self.removeFile(res["Value"])
+        listOfFolders = res["Value"]["SubDirs"]
+        listOfFiles = res["Value"]["Files"]
+
+        res = self.removeFile(listOfFiles)
         if not res["OK"]:
             return res
         for lfn, reason in res["Value"]["Failed"].items():  # can be an iterator
@@ -238,34 +236,6 @@ def __removeStorageDirectory(self, directory, storageElement):
         )
         return S_OK()
 
-    def __getCatalogDirectoryContents(self, directories, includeDirectories=False):
-        """ls recursively all files in directories
-
-        :param self: self reference
-        :param list directories: folder names
-        :param bool includeDirectories: if True includes directories in the return dictionary
-        :return: S_OK with dict of LFNs and their attribute dictionary
-        """
-        log = self.log.getSubLogger("__getCatalogDirectoryContents")
-        log.debug("Obtaining the catalog contents for %d directories:" % len(directories))
-        activeDirs = directories
-        allFiles = {}
-        while len(activeDirs) > 0:
-            currentDir = activeDirs[0]
-            res = returnSingleResult(self.fileCatalog.listDirectory(currentDir, verbose=True))
-            activeDirs.remove(currentDir)
-
-            if not res["OK"]:
-                log.debug("Problem getting the %s directory content" % currentDir, res["Message"])
-            else:
-                dirContents = res["Value"]
-                activeDirs.extend(dirContents["SubDirs"])
-                allFiles.update(dirContents["Files"])
-                if includeDirectories:
-                    allFiles.update(dirContents["SubDirs"])
-        log.debug("Found %d files" % len(allFiles))
-        return S_OK(allFiles)
-
     def getReplicasFromDirectory(self, directory):
         """get all replicas from a given directory
 
@@ -276,11 +246,16 @@ def getReplicasFromDirectory(self, directory):
             directories = [directory]
         else:
             directories = directory
-        res = self.__getCatalogDirectoryContents(directories)
+        res = returnSingleResult(self.fileCatalog.getDirectoryDump(directories))
         if not res["OK"]:
             return res
-        allReplicas = {lfn: metadata["Replicas"] for lfn, metadata in res["Value"].items()}  # can be an iterator
-        return S_OK(allReplicas)
+
+        lfns = res["Value"]["Files"]
+        res = self.fileCatalog.getReplicas(lfns, allStatus=True)
+        if not res["OK"]:
+            return res
+        res["Value"] = res["Value"]["Successful"]
+        return res
 
     def getFilesFromDirectory(self, directory, days=0, wildcard="*"):
         """get all files from :directory: older than :days: days matching to :wildcard:

src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py

@@ -100,83 +100,6 @@ def test__getFileTypesCount(self):
     #   res = self.ci.catalogDirectoryToSE(lfnDir)
     #   self.assertTrue(res['OK'])
 
-    def test__getCatalogDirectoryContents(self):
-        lfnDirs = ["/this/is/dir1/", "/this/is/dir2/"]
-
-        res = self.ci._getCatalogDirectoryContents(lfnDirs)
-        self.assertTrue(res["OK"])
-
-        resExpected = {
-            "Metadata": {
-                "/this/is/dir1/file1.txt": {
-                    "MetaData": {
-                        "Checksum": "7149ed85",
-                        "ChecksumType": "Adler32",
-                        "CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
-                        "FileID": 156301805,
-                        "GID": 2695,
-                        "GUID": "6A5C6C86-AD7B-E411-9EDB",
-                        "Mode": 436,
-                        "ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
-                        "Owner": "phicharp",
-                        "OwnerGroup": "lhcb_prod",
-                        "Size": 206380531,
-                        "Status": "AprioriGood",
-                        "Type": "File",
-                        "UID": 19503,
-                    }
-                },
-                "/this/is/dir1/file2.foo.bar": {
-                    "MetaData": {
-                        "Checksum": "7149ed86",
-                        "ChecksumType": "Adler32",
-                        "CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
-                        "FileID": 156301805,
-                        "GID": 2695,
-                        "GUID": "6A5C6C86-AD7B-E411-9EDB",
-                        "Mode": 436,
-                        "ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
-                        "Owner": "phicharp",
-                        "OwnerGroup": "lhcb_prod",
-                        "Size": 206380532,
-                        "Status": "AprioriGood",
-                        "Type": "File",
-                        "UID": 19503,
-                    }
-                },
-                "/this/is/dir2/subdir1/file3.pippo": {
-                    "MetaData": {
-                        "Checksum": "7149ed86",
-                        "ChecksumType": "Adler32",
-                        "CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
-                        "FileID": 156301805,
-                        "GID": 2695,
-                        "GUID": "6A5C6C86-AD7B-E411-9EDB",
-                        "Mode": 436,
-                        "ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
-                        "Owner": "phicharp",
-                        "OwnerGroup": "lhcb_prod",
-                        "Size": 206380532,
-                        "Status": "AprioriGood",
-                        "Type": "File",
-                        "UID": 19503,
-                    }
-                },
-            },
-            "Replicas": {
-                "/this/is/dir1/file1.txt": {
-                    "SE1": "smr://srm.SE1.ch:8443/srm/v2/server?SFN=/this/is/dir1/file1.txt",
-                    "SE2": "smr://srm.SE2.fr:8443/srm/v2/server?SFN=/this/is/dir1/file1.txt",
-                },
-                "/this/is/dir1/file2.foo.bar": {
-                    "SE1": "smr://srm.SE1.ch:8443/srm/v2/server?SFN=/this/is/dir1/file2.foo.bar",
-                    "SE3": "smr://srm.SE3.es:8443/srm/v2/server?SFN=/this/is/dir1/file2.foo.bar",
-                },
-            },
-        }
-
-        self.assertEqual(res["Value"], resExpected)
-
 
 if __name__ == "__main__":
     suite = unittest.defaultTestLoader.loadTestsFromTestCase(UtilitiesTestCase)