Skip to content

Commit d101999

Browse files
icewind1991icewind1991
committed
repair -1 folder sizes for object store background scan
Signed-off-by: Robin Appelman <robin@icewind.nl>
1 parent 74f31ba commit d101999

7 files changed

Lines changed: 72 additions & 22 deletions

File tree

‎apps/files_sharing/lib/Scanner.php‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
namespace OCA\Files_Sharing;
2727

28-
use OC\Files\ObjectStore\NoopScanner;
28+
use OC\Files\ObjectStore\ObjectStoreScanner;
2929

3030
/**
3131
* Scanner for SharedStorage
@@ -72,7 +72,8 @@ private function getSourceScanner() {
7272

7373
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
7474
$sourceScanner = $this->getSourceScanner();
75-
if ($sourceScanner instanceof NoopScanner) {
75+
if ($sourceScanner instanceof ObjectStoreScanner) {
76+
// ObjectStoreScanner doesn't scan
7677
return [];
7778
} else {
7879
return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock);

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1253,7 +1253,7 @@
12531253
'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
12541254
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
12551255
'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',
1256-
'OC\\Files\\ObjectStore\\NoopScanner' => $baseDir . '/lib/private/Files/ObjectStore/NoopScanner.php',
1256+
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
12571257
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',
12581258
'OC\\Files\\ObjectStore\\S3' => $baseDir . '/lib/private/Files/ObjectStore/S3.php',
12591259
'OC\\Files\\ObjectStore\\S3ConnectionTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1286,7 +1286,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
12861286
'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
12871287
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
12881288
'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',
1289-
'OC\\Files\\ObjectStore\\NoopScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/NoopScanner.php',
1289+
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
12901290
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',
12911291
'OC\\Files\\ObjectStore\\S3' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3.php',
12921292
'OC\\Files\\ObjectStore\\S3ConnectionTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php',

‎lib/private/Files/Cache/Scanner.php‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@
4141
use OCP\Files\ForbiddenException;
4242
use OCP\Files\NotFoundException;
4343
use OCP\Files\Storage\IReliableEtagStorage;
44+
use OCP\IDBConnection;
4445
use OCP\Lock\ILockingProvider;
45-
use OC\Files\Storage\Wrapper\Encoding;
4646
use OC\Files\Storage\Wrapper\Jail;
4747
use OC\Hooks\BasicEmitter;
4848
use Psr\Log\LoggerInterface;
@@ -89,12 +89,15 @@ class Scanner extends BasicEmitter implements IScanner {
8989
*/
9090
protected $lockingProvider;
9191

92+
protected IDBConnection $connection;
93+
9294
public function __construct(\OC\Files\Storage\Storage $storage) {
9395
$this->storage = $storage;
9496
$this->storageId = $this->storage->getId();
9597
$this->cache = $storage->getCache();
9698
$this->cacheActive = !\OC::$server->getConfig()->getSystemValueBool('filesystem_cache_readonly', false);
9799
$this->lockingProvider = \OC::$server->getLockingProvider();
100+
$this->connection = \OC::$server->get(IDBConnection::class);
98101
}
99102

100103
/**
@@ -430,7 +433,7 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s
430433
}
431434

432435
if ($this->useTransactions) {
433-
\OC::$server->getDatabaseConnection()->beginTransaction();
436+
$this->connection->beginTransaction();
434437
}
435438

436439
$exceptionOccurred = false;
@@ -473,8 +476,8 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s
473476
// process is running in parallel
474477
// log and ignore
475478
if ($this->useTransactions) {
476-
\OC::$server->getDatabaseConnection()->rollback();
477-
\OC::$server->getDatabaseConnection()->beginTransaction();
479+
$this->connection->rollback();
480+
$this->connection->beginTransaction();
478481
}
479482
\OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [
480483
'app' => 'core',
@@ -483,7 +486,7 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s
483486
$exceptionOccurred = true;
484487
} catch (\OCP\Lock\LockedException $e) {
485488
if ($this->useTransactions) {
486-
\OC::$server->getDatabaseConnection()->rollback();
489+
$this->connection->rollback();
487490
}
488491
throw $e;
489492
}
@@ -494,7 +497,7 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s
494497
$this->removeFromCache($child);
495498
}
496499
if ($this->useTransactions) {
497-
\OC::$server->getDatabaseConnection()->commit();
500+
$this->connection->commit();
498501
}
499502
if ($exceptionOccurred) {
500503
// It might happen that the parallel scan process has already
@@ -558,7 +561,7 @@ public function backgroundScan() {
558561
}
559562
}
560563

561-
private function runBackgroundScanJob(callable $callback, $path) {
564+
protected function runBackgroundScanJob(callable $callback, $path) {
562565
try {
563566
$callback();
564567
\OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]);

lib/private/Files/ObjectStore/NoopScanner.php renamed to lib/private/Files/ObjectStore/ObjectStoreScanner.php

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,10 @@
2727
namespace OC\Files\ObjectStore;
2828

2929
use OC\Files\Cache\Scanner;
30-
use OC\Files\Storage\Storage;
31-
32-
class NoopScanner extends Scanner {
33-
public function __construct(Storage $storage) {
34-
// we don't need the storage, so do nothing here
35-
}
30+
use OCP\DB\QueryBuilder\IQueryBuilder;
31+
use OCP\Files\FileInfo;
3632

33+
class ObjectStoreScanner extends Scanner {
3734
/**
3835
* scan a single file and store it in the cache
3936
*
@@ -76,6 +73,55 @@ protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse
7673
* walk over any folders that are not fully scanned yet and scan them
7774
*/
7875
public function backgroundScan() {
79-
//noop
76+
$lastPath = null;
77+
// find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
78+
// we sort by path DESC to ensure that contents of a folder are handled before the parent folder
79+
while (($path = $this->getIncomplete()) !== false && $path !== $lastPath) {
80+
$this->runBackgroundScanJob(function () use ($path) {
81+
$item = $this->cache->get($path);
82+
if ($item && $item->getMimeType() !== FileInfo::MIMETYPE_FOLDER) {
83+
$fh = $this->storage->fopen($path, 'r');
84+
if ($fh) {
85+
$stat = fstat($fh);
86+
if ($stat['size']) {
87+
$this->cache->update($item->getId(), ['size' => $stat['size']]);
88+
}
89+
}
90+
}
91+
}, $path);
92+
// FIXME: this won't proceed with the next item, needs revamping of getIncomplete()
93+
// to make this possible
94+
$lastPath = $path;
95+
}
96+
}
97+
98+
/**
99+
* Unlike the default Cache::getIncomplete this one sorts by path.
100+
*
101+
* This is needed since self::backgroundScan doesn't fix child entries when running on a parent folder.
102+
* By sorting by path we ensure that we encounter the child entries first.
103+
*
104+
* @return false|string
105+
* @throws \OCP\DB\Exception
106+
*/
107+
private function getIncomplete() {
108+
$query = $this->connection->getQueryBuilder();
109+
$query->select('path')
110+
->from('filecache')
111+
->where($query->expr()->eq('storage', $query->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT)))
112+
->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
113+
->orderBy('path', 'DESC')
114+
->setMaxResults(1);
115+
116+
$result = $query->executeQuery();
117+
$path = $result->fetchOne();
118+
$result->closeCursor();
119+
120+
if ($path === false) {
121+
return false;
122+
}
123+
124+
// Make sure Oracle does not continue with null for empty strings
125+
return (string)$path;
80126
}
81127
}

‎lib/private/Files/ObjectStore/ObjectStoreStorage.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,14 +163,14 @@ private function normalizePath($path) {
163163
*
164164
* @param string $path
165165
* @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
166-
* @return \OC\Files\ObjectStore\NoopScanner
166+
* @return \OC\Files\ObjectStore\ObjectStoreScanner
167167
*/
168168
public function getScanner($path = '', $storage = null) {
169169
if (!$storage) {
170170
$storage = $this;
171171
}
172172
if (!isset($this->scanner)) {
173-
$this->scanner = new NoopScanner($storage);
173+
$this->scanner = new ObjectStoreScanner($storage);
174174
}
175175
return $this->scanner;
176176
}

‎tests/lib/Files/ObjectStore/NoopScannerTest.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@ class NoopScannerTest extends \Test\TestCase {
1616
/** @var \OC\Files\Storage\Storage $storage */
1717
private $storage;
1818

19-
/** @var \OC\Files\ObjectStore\NoopScanner $scanner */
19+
/** @var \OC\Files\ObjectStore\ObjectStoreScanner $scanner */
2020
private $scanner;
2121

2222
protected function setUp(): void {
2323
parent::setUp();
2424

2525
$this->storage = new \OC\Files\Storage\Temporary([]);
26-
$this->scanner = new \OC\Files\ObjectStore\NoopScanner($this->storage);
26+
$this->scanner = new \OC\Files\ObjectStore\ObjectStoreScanner($this->storage);
2727
}
2828

2929
public function testFile() {

0 commit comments

Comments
 (0)