From b18a098f47ea3e09a8fe8b5863557a0d6dae2ace Mon Sep 17 00:00:00 2001 From: Varun Patil Date: Tue, 17 Jan 2023 20:52:26 -0800 Subject: [PATCH] index: allow cleaning up orphans (fix #326) --- lib/Command/Index.php | 69 ++++++++++++++----- lib/Db/TimelineWrite.php | 43 ++++++++++++ .../Version401000Date20230118043813.php | 69 +++++++++++++++++++ 3 files changed, 165 insertions(+), 16 deletions(-) create mode 100644 lib/Migration/Version401000Date20230118043813.php diff --git a/lib/Command/Index.php b/lib/Command/Index.php index 69520e31..244b14ed 100644 --- a/lib/Command/Index.php +++ b/lib/Command/Index.php @@ -41,6 +41,20 @@ use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\ConsoleSectionOutput; use Symfony\Component\Console\Output\OutputInterface; +class IndexOpts +{ + public bool $refresh = false; + public bool $clear = false; + public bool $cleanup = false; + + public function __construct(InputInterface $input) + { + $this->refresh = (bool) $input->getOption('refresh'); + $this->clear = (bool) $input->getOption('clear'); + $this->cleanup = (bool) $input->getOption('cleanup'); + } +} + class Index extends Command { /** @var int[][] */ @@ -99,7 +113,13 @@ class Index extends Command 'clear', null, InputOption::VALUE_NONE, - 'Clear existing index before creating a new one (SLOW)' + 'Clear existing index before creating a new one (slow)' + ) + ->addOption( + 'cleanup', + null, + InputOption::VALUE_NONE, + 'Remove orphaned entries from index (e.g. from .nomedia files)' ) ; } @@ -131,11 +151,10 @@ class Index extends Command } // Get options and arguments - $refresh = $input->getOption('refresh') ? true : false; - $clear = $input->getOption('clear') ? true : false; + $opts = new IndexOpts($input); // Clear index if asked for this - if ($clear && $input->isInteractive()) { + if ($opts->clear && $input->isInteractive()) { $output->write('Are you sure you want to clear the existing index? (y/N): '); $answer = trim(fgets(STDIN)); if ('y' !== $answer) { @@ -144,16 +163,23 @@ class Index extends Command return 1; } } - if ($clear) { + if ($opts->clear) { $this->timelineWrite->clear(); $output->writeln('Cleared existing index'); } + // Orphan all entries so we can delete them later + if ($opts->cleanup) { + $output->write('Marking all entries for cleanup ... '); + $count = $this->timelineWrite->orphanAll(); + $output->writeln("{$count} marked"); + } + // Run with the static process try { \OCA\Memories\Exif::ensureStaticExiftoolProc(); - return $this->executeWithOpts($output, $refresh); + return $this->executeWithOpts($output, $opts); } catch (\Exception $e) { error_log('FATAL: '.$e->getMessage()); @@ -163,7 +189,7 @@ class Index extends Command } } - protected function executeWithOpts(OutputInterface $output, bool &$refresh): int + protected function executeWithOpts(OutputInterface $output, IndexOpts &$opts): int { // Refuse to run without exiftool if (!$this->testExif()) { @@ -184,10 +210,17 @@ class Index extends Command } $this->output = $output; - $this->userManager->callForSeenUsers(function (IUser &$user) use (&$refresh) { - $this->generateUserEntries($user, $refresh); + $this->userManager->callForSeenUsers(function (IUser &$user) use (&$opts) { + $this->generateUserEntries($user, $opts); }); + // Clear orphans if asked for this + if ($opts->cleanup) { + $output->write('Deleting orphaned entries ... '); + $count = $this->timelineWrite->removeOrphans(); + $output->writeln("{$count} deleted"); + } + // Show some stats $endTime = microtime(true); $execTime = (int) (($endTime - $startTime) * 1000) / 1000; @@ -239,7 +272,7 @@ class Index extends Command return true; } - private function generateUserEntries(IUser &$user, bool &$refresh): void + private function generateUserEntries(IUser &$user, IndexOpts &$opts): void { \OC_Util::tearDownFS(); \OC_Util::setupFS($user->getUID()); @@ -247,12 +280,12 @@ class Index extends Command $uid = $user->getUID(); $userFolder = $this->rootFolder->getUserFolder($uid); $this->outputSection = $this->output->section(); - $this->parseFolder($userFolder, $refresh, (float) $this->nUser, (float) $this->userManager->countSeenUsers()); + $this->parseFolder($userFolder, $opts, (float) $this->nUser, (float) $this->userManager->countSeenUsers()); $this->outputSection->overwrite('Scanned '.$userFolder->getPath()); ++$this->nUser; } - private function parseFolder(Folder &$folder, bool $refresh, float $progress_i, float $progress_n): void + private function parseFolder(Folder &$folder, IndexOpts &$opts, float $progress_i, float $progress_n): void { try { // Respect the '.nomedia' file. If present don't traverse the folder @@ -268,11 +301,11 @@ class Index extends Command if ($node instanceof Folder) { $new_progress_i = (float) ($progress_i * \count($nodes) + $i); $new_progress_n = (float) ($progress_n * \count($nodes)); - $this->parseFolder($node, $refresh, $new_progress_i, $new_progress_n); + $this->parseFolder($node, $opts, $new_progress_i, $new_progress_n); } elseif ($node instanceof File) { $progress = (float) (($progress_i / $progress_n) * 100); $this->outputSection->overwrite(sprintf('%.2f%%', $progress).' scanning '.$node->getPath()); - $this->parseFile($node, $refresh); + $this->parseFile($node, $opts); } } } catch (\Exception $e) { @@ -284,13 +317,17 @@ class Index extends Command } } - private function parseFile(File &$file, bool &$refresh): void + private function parseFile(File &$file, IndexOpts &$opts): void { // Process the file $res = 1; try { - $res = $this->timelineWrite->processFile($file, $refresh); + $res = $this->timelineWrite->processFile($file, $opts->refresh); + + if ($opts->cleanup) { + $this->timelineWrite->unorphan($file); + } } catch (\Error $e) { $this->output->writeln(sprintf( 'Could not process file %s: %s', diff --git a/lib/Db/TimelineWrite.php b/lib/Db/TimelineWrite.php index db456e3a..9514d543 100644 --- a/lib/Db/TimelineWrite.php +++ b/lib/Db/TimelineWrite.php @@ -222,4 +222,47 @@ class TimelineWrite $t2 = $p->getTruncateTableSQL('`*PREFIX*memories_livephoto`', false); $this->connection->executeStatement("{$t1}; {$t2}"); } + + /** + * Mark a file as not orphaned. + */ + public function unorphan(File &$file) + { + $query = $this->connection->getQueryBuilder(); + $query->update('memories') + ->set('orphan', $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL)) + ->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT))) + ; + $query->executeStatement(); + } + + /** + * Mark all files in the table as orphaned. + * + * @return int Number of rows affected + */ + public function orphanAll(): int + { + $query = $this->connection->getQueryBuilder(); + $query->update('memories') + ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)) + ; + + return $query->executeStatement(); + } + + /** + * Remove all entries that are orphans. + * + * @return int Number of rows affected + */ + public function removeOrphans(): int + { + $query = $this->connection->getQueryBuilder(); + $query->delete('memories') + ->where($query->expr()->eq('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))) + ; + + return $query->executeStatement(); + } } diff --git a/lib/Migration/Version401000Date20230118043813.php b/lib/Migration/Version401000Date20230118043813.php new file mode 100644 index 00000000..7eec413c --- /dev/null +++ b/lib/Migration/Version401000Date20230118043813.php @@ -0,0 +1,69 @@ + + * @author Your name + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCA\Memories\Migration; + +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +/** + * Auto-generated migration step: Please modify to your needs! + */ +class Version401000Date20230118043813 extends SimpleMigrationStep +{ + /** + * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + */ + public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void + { + } + + /** + * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + */ + public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper + { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + $table = $schema->getTable('memories'); + + if (!$table->hasColumn('orphan')) { + $table->addColumn('orphan', Types::BOOLEAN, [ + 'notnull' => false, + 'default' => false, + ]); + } + + return $schema; + } + + /** + * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + */ + public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void + { + } +}