Skip to content

Commit 7b49368

Browse files
blizzzblizzz
authored
Merge pull request #20232 from nextcloud/storage-getdirectorycontent
Add method to storage backends to get directory content with metadata
2 parents b1a90da + 9735b5d commit 7b49368

14 files changed

Lines changed: 177 additions & 69 deletions

File tree

‎apps/files_external/lib/Lib/Storage/SMB.php‎

Lines changed: 57 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
use OC\Files\Filesystem;
5656
use OC\Files\Storage\Common;
5757
use OCA\Files_External\Lib\Notify\SMBNotifyHandler;
58+
use OCP\Constants;
5859
use OCP\Files\Notify\IChange;
5960
use OCP\Files\Notify\IRenameChange;
6061
use OCP\Files\Storage\INotifyStorage;
@@ -97,7 +98,7 @@ public function __construct($params) {
9798
if (isset($params['auth'])) {
9899
$auth = $params['auth'];
99100
} elseif (isset($params['user']) && isset($params['password']) && isset($params['share'])) {
100-
list($workgroup, $user) = $this->splitUser($params['user']);
101+
[$workgroup, $user] = $this->splitUser($params['user']);
101102
$auth = new BasicAuth($user, $workgroup, $params['password']);
102103
} else {
103104
throw new \Exception('Invalid configuration, no credentials provided');
@@ -206,30 +207,31 @@ protected function throwUnavailable(\Exception $e) {
206207
* @return \Icewind\SMB\IFileInfo[]
207208
* @throws StorageNotAvailableException
208209
*/
209-
protected function getFolderContents($path) {
210+
protected function getFolderContents($path): iterable {
210211
try {
211212
$path = ltrim($this->buildPath($path), '/');
212213
$files = $this->share->dir($path);
213214
foreach ($files as $file) {
214215
$this->statCache[$path . '/' . $file->getName()] = $file;
215216
}
216-
return array_filter($files, function (IFileInfo $file) {
217+
218+
foreach ($files as $file) {
217219
try {
218220
// the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch
219221
// so we trigger the below exceptions where applicable
220222
$hide = $file->isHidden() && !$this->showHidden;
221223
if ($hide) {
222224
$this->logger->debug('hiding hidden file ' . $file->getName());
223225
}
224-
return !$hide;
226+
if (!$hide) {
227+
yield $file;
228+
}
225229
} catch (ForbiddenException $e) {
226230
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]);
227-
return false;
228231
} catch (NotFoundException $e) {
229232
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]);
230-
return false;
231233
}
232-
});
234+
}
233235
} catch (ConnectException $e) {
234236
$this->logger->logException($e, ['message' => 'Error while getting folder content']);
235237
throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e);
@@ -508,6 +510,46 @@ public function touch($path, $time = null) {
508510
}
509511
}
510512

513+
public function getMetaData($path) {
514+
$fileInfo = $this->getFileInfo($path);
515+
if (!$fileInfo) {
516+
return null;
517+
}
518+
519+
return $this->getMetaDataFromFileInfo($fileInfo);
520+
}
521+
522+
private function getMetaDataFromFileInfo(IFileInfo $fileInfo) {
523+
$permissions = Constants::PERMISSION_READ + Constants::PERMISSION_SHARE;
524+
525+
if (!$fileInfo->isReadOnly()) {
526+
$permissions += Constants::PERMISSION_DELETE;
527+
$permissions += Constants::PERMISSION_UPDATE;
528+
if ($fileInfo->isDirectory()) {
529+
$permissions += Constants::PERMISSION_CREATE;
530+
}
531+
}
532+
533+
$data = [];
534+
if ($fileInfo->isDirectory()) {
535+
$data['mimetype'] = 'httpd/unix-directory';
536+
} else {
537+
$data['mimetype'] = \OC::$server->getMimeTypeDetector()->detectPath($fileInfo->getPath());
538+
}
539+
$data['mtime'] = $fileInfo->getMTime();
540+
if ($fileInfo->isDirectory()) {
541+
$data['size'] = -1; //unknown
542+
} else {
543+
$data['size'] = $fileInfo->getSize();
544+
}
545+
$data['etag'] = $this->getETag($fileInfo->getPath());
546+
$data['storage_mtime'] = $data['mtime'];
547+
$data['permissions'] = $permissions;
548+
$data['name'] = $fileInfo->getName();
549+
550+
return $data;
551+
}
552+
511553
public function opendir($path) {
512554
try {
513555
$files = $this->getFolderContents($path);
@@ -519,10 +561,17 @@ public function opendir($path) {
519561
$names = array_map(function ($info) {
520562
/** @var \Icewind\SMB\IFileInfo $info */
521563
return $info->getName();
522-
}, $files);
564+
}, iterator_to_array($files));
523565
return IteratorDirectory::wrap($names);
524566
}
525567

568+
public function getDirectoryContent($directory): \Traversable {
569+
$files = $this->getFolderContents($directory);
570+
foreach ($files as $file) {
571+
yield $this->getMetaDataFromFileInfo($file);
572+
}
573+
}
574+
526575
public function filetype($path) {
527576
try {
528577
return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file';

‎apps/files_sharing/lib/External/Scanner.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc
5757
* @param bool $lock set to false to disable getting an additional read lock during scanning
5858
* @return array an array of metadata of the scanned file
5959
*/
60-
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
60+
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
6161
try {
6262
return parent::scanFile($file, $reuseExisting);
6363
} catch (ForbiddenException $e) {

‎apps/files_sharing/lib/Scanner.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ private function getSourceScanner() {
7171
}
7272
}
7373

74-
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
74+
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
7575
$sourceScanner = $this->getSourceScanner();
7676
if ($sourceScanner instanceof NoopScanner) {
7777
return [];

‎lib/private/Files/Cache/Scanner.php‎

Lines changed: 10 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,11 @@ protected function getData($path) {
126126
* @param int $parentId
127127
* @param array|null|false $cacheData existing data in the cache for the file to be scanned
128128
* @param bool $lock set to false to disable getting an additional read lock during scanning
129+
* @param null $data the metadata for the file, as returned by the storage
129130
* @return array an array of metadata of the scanned file
130-
* @throws \OC\ServerNotAvailableException
131131
* @throws \OCP\Lock\LockedException
132132
*/
133-
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
133+
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
134134
if ($file !== '') {
135135
try {
136136
$this->storage->verifyPath(dirname($file), basename($file));
@@ -149,7 +149,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData =
149149
}
150150

151151
try {
152-
$data = $this->getData($file);
152+
$data = $data ?? $this->getData($file);
153153
} catch (ForbiddenException $e) {
154154
if ($lock) {
155155
if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
@@ -366,26 +366,6 @@ protected function getExistingChildren($folderId) {
366366
return $existingChildren;
367367
}
368368

369-
/**
370-
* Get the children from the storage
371-
*
372-
* @param string $folder
373-
* @return string[]
374-
*/
375-
protected function getNewChildren($folder) {
376-
$children = [];
377-
if ($dh = $this->storage->opendir($folder)) {
378-
if (is_resource($dh)) {
379-
while (($file = readdir($dh)) !== false) {
380-
if (!Filesystem::isIgnoredDir($file)) {
381-
$children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/');
382-
}
383-
}
384-
}
385-
}
386-
return $children;
387-
}
388-
389369
/**
390370
* scan all the files and folders in a folder
391371
*
@@ -425,19 +405,22 @@ protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse
425405
private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
426406
// we put this in it's own function so it cleans up the memory before we start recursing
427407
$existingChildren = $this->getExistingChildren($folderId);
428-
$newChildren = $this->getNewChildren($path);
408+
$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
429409

430410
if ($this->useTransactions) {
431411
\OC::$server->getDatabaseConnection()->beginTransaction();
432412
}
433413

434414
$exceptionOccurred = false;
435415
$childQueue = [];
436-
foreach ($newChildren as $file) {
416+
$newChildNames = [];
417+
foreach ($newChildren as $fileMeta) {
418+
$file = $fileMeta['name'];
419+
$newChildNames[] = $file;
437420
$child = $path ? $path . '/' . $file : $file;
438421
try {
439422
$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
440-
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock);
423+
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
441424
if ($data) {
442425
if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
443426
$childQueue[$child] = $data['fileid'];
@@ -471,7 +454,7 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s
471454
throw $e;
472455
}
473456
}
474-
$removedChildren = \array_diff(array_keys($existingChildren), $newChildren);
457+
$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
475458
foreach ($removedChildren as $childName) {
476459
$child = $path ? $path . '/' . $childName : $childName;
477460
$this->removeFromCache($child);

‎lib/private/Files/ObjectStore/NoopScanner.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public function __construct(Storage $storage) {
4444
* @param array|null $cacheData existing data in the cache for the file to be scanned
4545
* @return array an array of metadata of the scanned file
4646
*/
47-
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
47+
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
4848
return [];
4949
}
5050

‎lib/private/Files/Storage/Common.php‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ public function copy($path1, $path2) {
234234
} else {
235235
$source = $this->fopen($path1, 'r');
236236
$target = $this->fopen($path2, 'w');
237-
list(, $result) = \OC_Helper::streamCopy($source, $target);
237+
[, $result] = \OC_Helper::streamCopy($source, $target);
238238
if (!$result) {
239239
\OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2");
240240
}
@@ -626,7 +626,7 @@ public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
626626
// are not the same as the original one.Once this is fixed we also
627627
// need to adjust the encryption wrapper.
628628
$target = $this->fopen($targetInternalPath, 'w');
629-
list(, $result) = \OC_Helper::streamCopy($source, $target);
629+
[, $result] = \OC_Helper::streamCopy($source, $target);
630630
if ($result and $preserveMtime) {
631631
$this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath));
632632
}
@@ -719,6 +719,7 @@ public function getMetaData($path) {
719719
$data['etag'] = $this->getETag($path);
720720
$data['storage_mtime'] = $data['mtime'];
721721
$data['permissions'] = $permissions;
722+
$data['name'] = basename($path);
722723

723724
return $data;
724725
}
@@ -867,4 +868,17 @@ public function writeStream(string $path, $stream, int $size = null): int {
867868
}
868869
return $count;
869870
}
871+
872+
public function getDirectoryContent($directory): \Traversable {
873+
$dh = $this->opendir($directory);
874+
if (is_resource($dh)) {
875+
$basePath = rtrim($directory, '/');
876+
while (($file = readdir($dh)) !== false) {
877+
if (!Filesystem::isIgnoredDir($file)) {
878+
$childPath = $basePath . '/' . trim($file, '/');
879+
yield $this->getMetaData($childPath);
880+
}
881+
}
882+
}
883+
}
870884
}

‎lib/private/Files/Storage/Local.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ public function getMetaData($path) {
196196
$data['etag'] = $this->calculateEtag($path, $stat);
197197
$data['storage_mtime'] = $data['mtime'];
198198
$data['permissions'] = $permissions;
199+
$data['name'] = basename($path);
199200

200201
return $data;
201202
}

‎lib/private/Files/Storage/Storage.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,4 +119,22 @@ public function releaseLock($path, $type, ILockingProvider $provider);
119119
* @throws \OCP\Lock\LockedException
120120
*/
121121
public function changeLock($path, $type, ILockingProvider $provider);
122+
123+
/**
124+
* Get the contents of a directory with metadata
125+
*
126+
* @param string $directory
127+
* @return \Traversable an iterator, containing file metadata
128+
*
129+
* The metadata array will contain the following fields
130+
*
131+
* - name
132+
* - mimetype
133+
* - mtime
134+
* - size
135+
* - etag
136+
* - storage_mtime
137+
* - permissions
138+
*/
139+
public function getDirectoryContent($directory): \Traversable;
122140
}

‎lib/private/Files/Storage/Wrapper/Availability.php‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,4 +461,15 @@ protected function setUnavailable(StorageNotAvailableException $e) {
461461
$this->getStorageCache()->setAvailability(false, $delay);
462462
throw $e;
463463
}
464+
465+
466+
467+
public function getDirectoryContent($directory): \Traversable {
468+
$this->checkAvailability();
469+
try {
470+
return parent::getDirectoryContent($directory);
471+
} catch (StorageNotAvailableException $e) {
472+
$this->setUnavailable($e);
473+
}
474+
}
464475
}

‎lib/private/Files/Storage/Wrapper/Encoding.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,4 +534,8 @@ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
534534
public function getMetaData($path) {
535535
return $this->storage->getMetaData($this->findPathToUse($path));
536536
}
537+
538+
public function getDirectoryContent($directory): \Traversable {
539+
return $this->storage->getDirectoryContent($this->findPathToUse($directory));
540+
}
537541
}

0 commit comments

Comments
 (0)