Skip to content

Commit 0e467c9

Browse files
MorrisJobkeMorrisJobke
committed
Add occ preview:migrate to migrate previews from the old flat structure to a subfolder structure
* `php occ preview:repair` - a preview migration tool that moves existing previews into the new location introduced with #19214 * moves `appdata_INSTANCEID/previews/FILEID` to `appdata_INSTANCEID/previews/0/5/8/4/c/e/5/FILEID` * migration tool can be stopped during migration via `CTRL+C` - it then finishes the current folder (with the previews of one file) and stops gracefully * if a PHP memory limit is set in the `php.ini` then it will stop automatically once it has less than 25 MiB memory left (this is to avoid hard crashes in the middle of a migration) * the tool can be used during operation - possible drawbacks: * there is the chance of a race condition that a new preview is generated in the moment the folder is already migrated away - so the old folder with the newly cached preview is deleted and one cached preview needs to be re-generated * there is the chance of a race condition during access of a preview while it is migrated to the other folder - then no preview can be shown and results in a 404 (as of now this is an accepted risk) Signed-off-by: Morris Jobke <hey@morrisjobke.de>
1 parent c24f3d1 commit 0e467c9

3 files changed

Lines changed: 284 additions & 3 deletions

File tree

‎core/Command/Preview/Repair.php‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
6+
*
7+
* @author Morris Jobke <hey@morrisjobke.de>
8+
*
9+
* @license GNU AGPL version 3 or any later version
10+
*
11+
* This program is free software: you can redistribute it and/or modify
12+
* it under the terms of the GNU Affero General Public License as
13+
* published by the Free Software Foundation, either version 3 of the
14+
* License, or (at your option) any later version.
15+
*
16+
* This program is distributed in the hope that it will be useful,
17+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
18+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+
* GNU Affero General Public License for more details.
20+
*
21+
* You should have received a copy of the GNU Affero General Public License
22+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
23+
*
24+
*/
25+
26+
namespace OC\Core\Command\Preview;
27+
28+
use bantu\IniGetWrapper\IniGetWrapper;
29+
use OC\Preview\Storage\Root;
30+
use OCP\Files\Folder;
31+
use OCP\Files\IRootFolder;
32+
use OCP\Files\NotFoundException;
33+
use OCP\IConfig;
34+
use OCP\ILogger;
35+
use Symfony\Component\Console\Command\Command;
36+
use Symfony\Component\Console\Helper\ProgressBar;
37+
use Symfony\Component\Console\Input\InputInterface;
38+
use Symfony\Component\Console\Input\InputOption;
39+
use Symfony\Component\Console\Output\OutputInterface;
40+
use Symfony\Component\Console\Question\ConfirmationQuestion;
41+
42+
class Repair extends Command {
43+
/** @var IConfig */
44+
protected $config;
45+
/** @var IRootFolder */
46+
private $rootFolder;
47+
/** @var ILogger */
48+
private $logger;
49+
50+
/** @var bool */
51+
private $stopSignalReceived = false;
52+
/** @var int */
53+
private $memoryLimit;
54+
/** @var int */
55+
private $memoryTreshold;
56+
57+
public function __construct(IConfig $config, IRootFolder $rootFolder, ILogger $logger, IniGetWrapper $phpIni) {
58+
$this->config = $config;
59+
$this->rootFolder = $rootFolder;
60+
$this->logger = $logger;
61+
62+
$this->memoryLimit = $phpIni->getBytes('memory_limit');
63+
$this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024;
64+
65+
parent::__construct();
66+
}
67+
68+
protected function configure() {
69+
$this
70+
->setName('preview:repair')
71+
->setDescription('distributes the existing previews into subfolders')
72+
->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.')
73+
->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.');
74+
}
75+
76+
protected function execute(InputInterface $input, OutputInterface $output): int {
77+
if ($this->memoryLimit !== -1) {
78+
$limitInMiB = round($this->memoryLimit / 1024 /1024, 1);
79+
$thresholdInMiB = round($this->memoryTreshold / 1024 /1024, 1);
80+
$output->writeln("Memory limit is $limitInMiB MiB");
81+
$output->writeln("Memory threshold is $thresholdInMiB MiB");
82+
$output->writeln("");
83+
$memoryCheckEnabled = true;
84+
} else {
85+
$output->writeln("No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.");
86+
$output->writeln("");
87+
$memoryCheckEnabled = false;
88+
}
89+
90+
$dryMode = $input->getOption('dry');
91+
92+
if ($dryMode) {
93+
$output->writeln("INFO: The migration is run in dry mode and will not modify anything.");
94+
$output->writeln("");
95+
}
96+
97+
$verbose = $output->isVerbose();
98+
99+
$instanceId = $this->config->getSystemValueString('instanceid');
100+
101+
$output->writeln("This will migrate all previews from the old preview location to the new one.");
102+
$output->writeln('');
103+
104+
$output->writeln('Fetching previews that need to be migrated …');
105+
/** @var \OCP\Files\Folder $currentPreviewFolder */
106+
$currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview");
107+
108+
$directoryListing = $currentPreviewFolder->getDirectoryListing();
109+
110+
$total = count($directoryListing);
111+
/**
112+
* by default there could be 0-9 a-f and the old-multibucket folder which are all fine
113+
*/
114+
if ($total < 18) {
115+
foreach ($directoryListing as $index => $dir) {
116+
if ($dir->getName() === 'old-multibucket') {
117+
unset($directoryListing[$index]);
118+
}
119+
// a-f can't be a file ID -> removing from migration
120+
if (preg_match('!^[a-f]$!', $dir->getName())) {
121+
unset($directoryListing[$index]);
122+
}
123+
if (preg_match('!^[0-9]$!', $dir->getName())) {
124+
// ignore folders that only has folders in them
125+
if ($dir instanceof Folder) {
126+
$hasFile = false;
127+
foreach ($dir->getDirectoryListing() as $entry) {
128+
if (!$entry instanceof Folder) {
129+
$hasFile = true;
130+
break;
131+
}
132+
}
133+
if (!$hasFile) {
134+
unset($directoryListing[$index]);
135+
}
136+
}
137+
}
138+
}
139+
$total = count($directoryListing);
140+
}
141+
142+
if ($total === 0) {
143+
$output->writeln("All previews are already migrated.");
144+
return 0;
145+
}
146+
147+
$output->writeln("A total of $total preview files need to be migrated.");
148+
$output->writeln("");
149+
$output->writeln("The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This fill finish the current batch and then stop the migration. This migration can then just be started and it will continue.");
150+
151+
if ($input->getOption('batch')) {
152+
$output->writeln('Batch mode active: migration is started right away.');
153+
} else {
154+
$helper = $this->getHelper('question');
155+
$question = new ConfirmationQuestion('<info>Should the migration be started? (y/[n]) </info>', false);
156+
157+
if (!$helper->ask($input, $output, $question)) {
158+
return 0;
159+
}
160+
}
161+
162+
// register the SIGINT listener late in here to be able to exit in the early process of this command
163+
pcntl_signal(SIGINT, [$this, 'sigIntHandler']);
164+
165+
$output->writeln("");
166+
$output->writeln("");
167+
$progressBar = new ProgressBar($output, $total);
168+
$progressBar->setFormat("%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s% \n %message%\n");
169+
$time = (new \DateTime())->format('H:i:s');
170+
$progressBar->setMessage("$time Starting …");
171+
$progressBar->maxSecondsBetweenRedraws(0.2);
172+
$progressBar->setOverwrite(false);
173+
$progressBar->start();
174+
175+
foreach ($directoryListing as $oldPreviewFolder) {
176+
pcntl_signal_dispatch();
177+
$name = $oldPreviewFolder->getName();
178+
$time = (new \DateTime())->format('H:i:s');
179+
$progressBar->setMessage("$time Migrating previews of file with fileId $name …");
180+
$progressBar->display();
181+
182+
if ($this->stopSignalReceived) {
183+
$progressBar->setMessage("$time Stopping migration …");
184+
return 0;
185+
}
186+
if (!$oldPreviewFolder instanceof Folder) {
187+
$progressBar->setMessage("Skipping non-folder $name …");
188+
$progressBar->advance();
189+
continue;
190+
}
191+
if ($name === 'old-multibucket') {
192+
$progressBar->setMessage("Skipping fallback mount point $name …");
193+
$progressBar->advance();
194+
continue;
195+
}
196+
if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) {
197+
$progressBar->setMessage("Skipping hex-digit folder $name …");
198+
$progressBar->advance();
199+
continue;
200+
}
201+
if (!preg_match('!^\d+$!', $name)) {
202+
$progressBar->setMessage("Skipping non-numeric folder $name …");
203+
$progressBar->advance();
204+
continue;
205+
}
206+
207+
$newFoldername = Root::getInternalFolder($name);
208+
209+
$memoryUsage = memory_get_usage();
210+
if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) {
211+
$output->writeln("");
212+
$output->writeln("");
213+
$output->writeln("");
214+
$output->writeln("Stopped process 25 MB before reaching the memory limit to avoid a hard crash.");
215+
$time = (new \DateTime())->format('H:i:s');
216+
$progressBar->setMessage("$time Reached memory limit and stopped to avoid hard crash.");
217+
return 1;
218+
}
219+
220+
$previews = $oldPreviewFolder->getDirectoryListing();
221+
if ($previews !== []) {
222+
try {
223+
$this->rootFolder->get("appdata_$instanceId/preview/$newFoldername");
224+
} catch (NotFoundException $e) {
225+
if ($verbose) {
226+
$output->writeln("Create folder preview/$newFoldername");
227+
}
228+
if (!$dryMode) {
229+
$this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername");
230+
}
231+
}
232+
233+
foreach ($previews as $preview) {
234+
pcntl_signal_dispatch();
235+
$previewName = $preview->getName();
236+
237+
if ($preview instanceof Folder) {
238+
$progressBar->setMessage("Skipping folder $name/$previewName …");
239+
$progressBar->advance();
240+
continue;
241+
}
242+
if ($verbose) {
243+
$output->writeln("Move preview/$name/$previewName to preview/$newFoldername");
244+
}
245+
if (!$dryMode) {
246+
try {
247+
$preview->move("appdata_$instanceId/preview/$newFoldername/$previewName");
248+
} catch(\Exception $e) {
249+
$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to move preview from preview/$name/$previewName to preview/$newFoldername"]);
250+
}
251+
}
252+
}
253+
}
254+
if ($oldPreviewFolder->getDirectoryListing() === []) {
255+
if ($verbose) {
256+
$output->writeln("Delete empty folder preview/$name");
257+
}
258+
if (!$dryMode) {
259+
try {
260+
$oldPreviewFolder->delete();
261+
} catch(\Exception $e) {
262+
$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to delete empty folder preview/$name"]);
263+
}
264+
}
265+
}
266+
$progressBar->setMessage("Finished migrating previews of file with fileId $name …");
267+
$progressBar->advance();
268+
}
269+
270+
$progressBar->finish();
271+
$output->writeln("");
272+
return 0;
273+
}
274+
275+
protected function sigIntHandler() {
276+
echo "\n\nSignal received - will finish the step and then stop the migration.\n\n";
277+
$this->stopSignalReceived = true;
278+
}
279+
}

‎core/register_command.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@
155155
\OC::$server->getAppManager()
156156
));
157157

158+
$application->add(\OC::$server->query(\OC\Core\Command\Preview\Repair::class));
159+
158160
$application->add(new OC\Core\Command\User\Add(\OC::$server->getUserManager(), \OC::$server->getGroupManager()));
159161
$application->add(new OC\Core\Command\User\Delete(\OC::$server->getUserManager()));
160162
$application->add(new OC\Core\Command\User\Disable(\OC::$server->getUserManager()));

‎lib/private/Preview/Storage/Root.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig)
3939

4040

4141
public function getFolder(string $name): ISimpleFolder {
42-
$internalFolder = $this->getInternalFolder($name);
42+
$internalFolder = self::getInternalFolder($name);
4343

4444
try {
4545
return parent::getFolder($internalFolder);
@@ -54,7 +54,7 @@ public function getFolder(string $name): ISimpleFolder {
5454
}
5555

5656
public function newFolder(string $name): ISimpleFolder {
57-
$internalFolder = $this->getInternalFolder($name);
57+
$internalFolder = self::getInternalFolder($name);
5858
return parent::newFolder($internalFolder);
5959
}
6060

@@ -66,7 +66,7 @@ public function getDirectoryListing(): array {
6666
return [];
6767
}
6868

69-
private function getInternalFolder(string $name): string {
69+
public static function getInternalFolder(string $name): string {
7070
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
7171
}
7272
}

0 commit comments

Comments
 (0)