index: allow cleaning up orphans (fix #326)

lib/Command/Index.php

@@ -41,6 +41,20 @@ use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\ConsoleSectionOutput;
 use Symfony\Component\Console\Output\OutputInterface;
 
+class IndexOpts
+{
+    public bool $refresh = false;
+    public bool $clear = false;
+    public bool $cleanup = false;
+
+    public function __construct(InputInterface $input)
+    {
+        $this->refresh = (bool) $input->getOption('refresh');
+        $this->clear = (bool) $input->getOption('clear');
+        $this->cleanup = (bool) $input->getOption('cleanup');
+    }
+}
+
 class Index extends Command
 {
     /** @var int[][] */
@@ -99,7 +113,13 @@ class Index extends Command
                 'clear',
                 null,
                 InputOption::VALUE_NONE,
-                'Clear existing index before creating a new one (SLOW)'
+                'Clear existing index before creating a new one (slow)'
+            )
+            ->addOption(
+                'cleanup',
+                null,
+                InputOption::VALUE_NONE,
+                'Remove orphaned entries from index (e.g. from .nomedia files)'
             )
         ;
     }
@@ -131,11 +151,10 @@ class Index extends Command
         }
 
         // Get options and arguments
-        $refresh = $input->getOption('refresh') ? true : false;
+        $opts = new IndexOpts($input);
-        $clear = $input->getOption('clear') ? true : false;
 
         // Clear index if asked for this
-        if ($clear && $input->isInteractive()) {
+        if ($opts->clear && $input->isInteractive()) {
             $output->write('Are you sure you want to clear the existing index? (y/N): ');
             $answer = trim(fgets(STDIN));
             if ('y' !== $answer) {
@@ -144,16 +163,23 @@ class Index extends Command
                 return 1;
             }
         }
-        if ($clear) {
+        if ($opts->clear) {
             $this->timelineWrite->clear();
             $output->writeln('Cleared existing index');
         }
 
+        // Orphan all entries so we can delete them later
+        if ($opts->cleanup) {
+            $output->write('Marking all entries for cleanup ... ');
+            $count = $this->timelineWrite->orphanAll();
+            $output->writeln("{$count} marked");
+        }
+
         // Run with the static process
         try {
             \OCA\Memories\Exif::ensureStaticExiftoolProc();
 
-            return $this->executeWithOpts($output, $refresh);
+            return $this->executeWithOpts($output, $opts);
         } catch (\Exception $e) {
             error_log('FATAL: '.$e->getMessage());
 
@@ -163,7 +189,7 @@ class Index extends Command
         }
     }
 
-    protected function executeWithOpts(OutputInterface $output, bool &$refresh): int
+    protected function executeWithOpts(OutputInterface $output, IndexOpts &$opts): int
     {
         // Refuse to run without exiftool
         if (!$this->testExif()) {
@@ -184,10 +210,17 @@ class Index extends Command
         }
         $this->output = $output;
 
-        $this->userManager->callForSeenUsers(function (IUser &$user) use (&$refresh) {
+        $this->userManager->callForSeenUsers(function (IUser &$user) use (&$opts) {
-            $this->generateUserEntries($user, $refresh);
+            $this->generateUserEntries($user, $opts);
         });
 
+        // Clear orphans if asked for this
+        if ($opts->cleanup) {
+            $output->write('Deleting orphaned entries ... ');
+            $count = $this->timelineWrite->removeOrphans();
+            $output->writeln("{$count} deleted");
+        }
+
         // Show some stats
         $endTime = microtime(true);
         $execTime = (int) (($endTime - $startTime) * 1000) / 1000;
@@ -239,7 +272,7 @@ class Index extends Command
         return true;
     }
 
-    private function generateUserEntries(IUser &$user, bool &$refresh): void
+    private function generateUserEntries(IUser &$user, IndexOpts &$opts): void
     {
         \OC_Util::tearDownFS();
         \OC_Util::setupFS($user->getUID());
@@ -247,12 +280,12 @@ class Index extends Command
         $uid = $user->getUID();
         $userFolder = $this->rootFolder->getUserFolder($uid);
         $this->outputSection = $this->output->section();
-        $this->parseFolder($userFolder, $refresh, (float) $this->nUser, (float) $this->userManager->countSeenUsers());
+        $this->parseFolder($userFolder, $opts, (float) $this->nUser, (float) $this->userManager->countSeenUsers());
         $this->outputSection->overwrite('Scanned '.$userFolder->getPath());
         ++$this->nUser;
     }
 
-    private function parseFolder(Folder &$folder, bool $refresh, float $progress_i, float $progress_n): void
+    private function parseFolder(Folder &$folder, IndexOpts &$opts, float $progress_i, float $progress_n): void
     {
         try {
             // Respect the '.nomedia' file. If present don't traverse the folder
@@ -268,11 +301,11 @@ class Index extends Command
                 if ($node instanceof Folder) {
                     $new_progress_i = (float) ($progress_i * \count($nodes) + $i);
                     $new_progress_n = (float) ($progress_n * \count($nodes));
-                    $this->parseFolder($node, $refresh, $new_progress_i, $new_progress_n);
+                    $this->parseFolder($node, $opts, $new_progress_i, $new_progress_n);
                 } elseif ($node instanceof File) {
                     $progress = (float) (($progress_i / $progress_n) * 100);
                     $this->outputSection->overwrite(sprintf('%.2f%%', $progress).' scanning '.$node->getPath());
-                    $this->parseFile($node, $refresh);
+                    $this->parseFile($node, $opts);
                 }
             }
         } catch (\Exception $e) {
@@ -284,13 +317,17 @@ class Index extends Command
         }
     }
 
-    private function parseFile(File &$file, bool &$refresh): void
+    private function parseFile(File &$file, IndexOpts &$opts): void
     {
         // Process the file
         $res = 1;
 
         try {
-            $res = $this->timelineWrite->processFile($file, $refresh);
+            $res = $this->timelineWrite->processFile($file, $opts->refresh);
+
+            if ($opts->cleanup) {
+                $this->timelineWrite->unorphan($file);
+            }
         } catch (\Error $e) {
             $this->output->writeln(sprintf(
                 '<error>Could not process file %s: %s</error>',

lib/Db/TimelineWrite.php

@@ -222,4 +222,47 @@ class TimelineWrite
         $t2 = $p->getTruncateTableSQL('`*PREFIX*memories_livephoto`', false);
         $this->connection->executeStatement("{$t1}; {$t2}");
     }
+
+    /**
+     * Mark a file as not orphaned.
+     */
+    public function unorphan(File &$file)
+    {
+        $query = $this->connection->getQueryBuilder();
+        $query->update('memories')
+            ->set('orphan', $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL))
+            ->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT)))
+        ;
+        $query->executeStatement();
+    }
+
+    /**
+     * Mark all files in the table as orphaned.
+     *
+     * @return int Number of rows affected
+     */
+    public function orphanAll(): int
+    {
+        $query = $this->connection->getQueryBuilder();
+        $query->update('memories')
+            ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
+        ;
+
+        return $query->executeStatement();
+    }
+
+    /**
+     * Remove all entries that are orphans.
+     *
+     * @return int Number of rows affected
+     */
+    public function removeOrphans(): int
+    {
+        $query = $this->connection->getQueryBuilder();
+        $query->delete('memories')
+            ->where($query->expr()->eq('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)))
+        ;
+
+        return $query->executeStatement();
+    }
 }

lib/Migration/Version401000Date20230118043813.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Your name <your@email.com>
+ * @author Your name <your@email.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Migration;
+
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version401000Date20230118043813 extends SimpleMigrationStep
+{
+    /**
+     * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+     */
+    public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+    }
+
+    /**
+     * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+     */
+    public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
+    {
+        /** @var ISchemaWrapper $schema */
+        $schema = $schemaClosure();
+
+        $table = $schema->getTable('memories');
+
+        if (!$table->hasColumn('orphan')) {
+            $table->addColumn('orphan', Types::BOOLEAN, [
+                'notnull' => false,
+                'default' => false,
+            ]);
+        }
+
+        return $schema;
+    }
+
+    /**
+     * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+     */
+    public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+    }
+}