Restore refresh command

Signed-off-by: Varun Patil <varunpatil@ucla.edu>

lib/Command/Index.php

@@ -105,6 +105,7 @@ class Index extends Command
         $this->input = $input;
         $this->output = $output;
         $this->opts = new IndexOpts($input);
+        $this->indexer->output = $output;
 
         try {
             // Use static exiftool process
@@ -135,19 +136,21 @@ class Index extends Command
      */
     protected function checkClear(): void
     {
-        if ($this->opts->clear) {
+        if (!$this->opts->clear) {
-            if ($this->input->isInteractive()) {
+            return;
-                $this->output->write('Are you sure you want to clear the existing index? (y/N): ');
-                if ('y' !== trim(fgets(STDIN))) {
-                    $this->output->writeln('Aborting');
-
-                    exit;
-                }
-            }
-
-            $this->timelineWrite->clear();
-            $this->output->writeln('Cleared existing index');
         }
+
+        if ($this->input->isInteractive()) {
+            $this->output->write('Are you sure you want to clear the existing index? (y/N): ');
+            if ('y' !== trim(fgets(STDIN))) {
+                $this->output->writeln('Aborting');
+
+                exit;
+            }
+        }
+
+        $this->timelineWrite->clear();
+        $this->output->writeln('Cleared existing index');
     }
 
     /**
@@ -155,11 +158,13 @@ class Index extends Command
      */
     protected function checkForce(): void
     {
-        if ($this->opts->force) {
+        if (!$this->opts->force) {
-            $this->output->writeln('Forcing refresh of existing index entries');
+            return;
-
-            // TODO
         }
+
+        $this->output->writeln('Forcing refresh of existing index entries');
+
+        $this->timelineWrite->orphanAll();
     }
 
     /**
@@ -167,23 +172,32 @@ class Index extends Command
      */
     protected function runIndex(): void
     {
-        // Call indexing for specified or each user
+        $this->runForUsers(function (IUser $user) {
+            try {
+                $uid = $user->getUID();
+                $this->output->writeln("Indexing user {$uid}");
+                $this->indexer->indexUser($uid, $this->opts->folder);
+            } catch (\Exception $e) {
+                $this->output->writeln("<error>{$e->getMessage()}</error>");
+            }
+        });
+    }
+
+    /**
+     * Run function for all users (or selected user if set).
+     *
+     * @param mixed $closure
+     */
+    private function runForUsers($closure)
+    {
         if ($uid = $this->opts->user) {
             if ($user = $this->userManager->get($uid)) {
-                $this->indexer->indexUser($user->getUID(), $this->opts->folder);
+                $closure($user);
             } else {
-                throw new \Exception("User {$uid} not found");
+                $this->output->writeln("<error>User {$uid} not found</error>");
             }
         } else {
-            $this->userManager->callForSeenUsers(function (IUser $user) {
+            $this->userManager->callForSeenUsers(fn (IUser $user) => $closure($user));
-                try {
-                    $uid = $user->getUID();
-                    $this->output->writeln("Indexing user {$uid}");
-                    $this->indexer->indexUser($uid, $this->opts->folder);
-                } catch (\Exception $e) {
-                    $this->output->writeln("<error>{$e->getMessage()}</error>");
-                }
-            });
         }
     }
 }

lib/Command/MigrateGoogleTakeout.php

@@ -63,7 +63,8 @@ class MigrateGoogleTakeout extends Command
         IUserManager $userManager,
         IConfig $config,
         IDBConnection $connection,
-        ITempManager $tempManager
+        ITempManager $tempManager,
+        TimelineWrite $timelineWrite
     ) {
         parent::__construct();
 
@@ -72,7 +73,7 @@ class MigrateGoogleTakeout extends Command
         $this->config = $config;
         $this->connection = $connection;
         $this->tempManager = $tempManager;
-        $this->timelineWrite = new TimelineWrite($connection);
+        $this->timelineWrite = $timelineWrite;
     }
 
     protected function configure(): void

lib/Db/LivePhoto.php

@@ -103,13 +103,13 @@ class LivePhoto
     /**
      * Process video part of Live Photo.
      */
-    public function processVideoPart(File $file, array $exif)
+    public function processVideoPart(File $file, array $exif): bool
     {
         $fileId = $file->getId();
         $mtime = $file->getMTime();
         $liveid = $exif['ContentIdentifier'];
         if (empty($liveid)) {
-            return;
+            return false;
         }
 
         $query = $this->connection->getQueryBuilder();
@@ -117,32 +117,30 @@ class LivePhoto
             ->from('memories_livephoto')
             ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
         ;
-        $cursor = $query->executeQuery();
+        $prevRow = $query->executeQuery()->fetch();
-        $prevRow = $cursor->fetch();
-        $cursor->closeCursor();
 
-        if ($prevRow) {
+        $params = [
-            // Update existing row
+            'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
-            $query->update('memories_livephoto')
+            'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
-                ->set('liveid', $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR))
+            'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
-                ->set('mtime', $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT))
+            'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL),
-                ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
+        ];
-            ;
+
-            $query->executeStatement();
+        try {
-        } else {
+            if ($prevRow) {
-            // Try to create new row
+                $query->update('memories_livephoto')
-            try {
+                    ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
-                $query->insert('memories_livephoto')
-                    ->values([
-                        'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
-                        'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
-                        'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
-                    ])
                 ;
-                $query->executeStatement();
+                foreach ($params as $key => $value) {
-            } catch (\Exception $ex) {
+                    $query->set($key, $value);
-                error_log('Failed to create memories_livephoto record: '.$ex->getMessage());
+                }
+            } else {
+                $query->insert('memories_livephoto')->values($params);
             }
+
+            return $query->executeStatement() > 0;
+        } catch (\Exception $ex) {
+            throw new \Exception('Failed to create livephoto record: '.$ex->getMessage());
         }
     }
 

lib/Db/TimelineWrite.php

@@ -9,7 +9,6 @@ use OCA\Memories\Service\Index;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\File;
 use OCP\IDBConnection;
-use OCP\IPreview;
 
 require_once __DIR__.'/../ExifFields.php';
 
@@ -22,14 +21,14 @@ class TimelineWrite
     use TimelineWriteOrphans;
     use TimelineWritePlaces;
     protected IDBConnection $connection;
-    protected IPreview $preview;
     protected LivePhoto $livePhoto;
 
-    public function __construct(IDBConnection $connection)
+    public function __construct(
-    {
+        IDBConnection $connection,
+        LivePhoto $livePhoto
+    ) {
         $this->connection = $connection;
-        $this->preview = \OC::$server->get(IPreview::class);
+        $this->livePhoto = $livePhoto;
-        $this->livePhoto = new LivePhoto($connection);
     }
 
     /**
@@ -53,8 +52,12 @@ class TimelineWrite
         // Get previous row
         $prevRow = $this->getCurrentRow($fileId);
 
-        // Skip if not forced and file has not changed
+        // Skip if all of the following:
-        if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime)) {
+        // - not forced
+        // - the record exists
+        // - the file has not changed
+        // - the record is not an orphan
+        if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime) && (!(bool) $prevRow['orphan'])) {
             return false;
         }
 
@@ -127,6 +130,7 @@ class TimelineWrite
             'lat' => $query->createNamedParameter($lat, IQueryBuilder::PARAM_STR),
             'lon' => $query->createNamedParameter($lon, IQueryBuilder::PARAM_STR),
             'mapcluster' => $query->createNamedParameter($mapCluster, IQueryBuilder::PARAM_INT),
+            'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL),
         ];
 
         // There is no easy way to UPSERT in standard SQL
@@ -145,9 +149,7 @@ class TimelineWrite
 
             return $query->executeStatement() > 0;
         } catch (\Exception $ex) {
-            error_log('Failed to create memories record: '.$ex->getMessage());
+            throw new \Exception('Failed to create memories record: '.$ex->getMessage());
-
-            return false;
         }
     }
 

lib/Db/TimelineWriteOrphans.php

@@ -32,12 +32,16 @@ trait TimelineWriteOrphans
      */
     public function orphanAll(): int
     {
-        $query = $this->connection->getQueryBuilder();
+        $do = function (string $table) {
-        $query->update('memories')
+            $query = $this->connection->getQueryBuilder();
-            ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
+            $query->update($table)
-        ;
+                ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
+            ;
 
-        return $query->executeStatement();
+            return $query->executeStatement();
+        };
+
+        return $do('memories') + $do('memories_livephoto');
     }
 
     /**

lib/Exif.php

@@ -71,6 +71,7 @@ class Exif
     {
         $config = \OC::$server->get(IConfig::class);
         $paths = $config->getUserValue($uid, Application::APPNAME, 'timelinePath', null) ?? 'Photos/';
+
         return array_map(fn ($p) => self::sanitizePath(trim($p)), explode(';', $paths));
     }
 

lib/Listeners/PostDeleteListener.php

@@ -26,15 +26,14 @@ use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
 use OCP\Files\Events\Node\NodeDeletedEvent;
 use OCP\Files\Folder;
-use OCP\IDBConnection;
 
 class PostDeleteListener implements IEventListener
 {
     private TimelineWrite $util;
 
-    public function __construct(IDBConnection $connection)
+    public function __construct(TimelineWrite $util)
     {
-        $this->util = new TimelineWrite($connection);
+        $this->util = $util;
     }
 
     public function handle(Event $event): void

lib/Listeners/PostWriteListener.php

@@ -22,20 +22,19 @@ declare(strict_types=1);
 namespace OCA\Memories\Listeners;
 
 use OCA\Memories\Db\TimelineWrite;
+use OCA\Memories\Service\Index;
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
 use OCP\Files\Events\Node\NodeTouchedEvent;
 use OCP\Files\Events\Node\NodeWrittenEvent;
-use OCP\Files\Folder;
-use OCP\IDBConnection;
 
 class PostWriteListener implements IEventListener
 {
     private TimelineWrite $timelineWrite;
 
-    public function __construct(IDBConnection $connection)
+    public function __construct(TimelineWrite $timelineWrite)
     {
-        $this->timelineWrite = new TimelineWrite($connection);
+        $this->timelineWrite = $timelineWrite;
     }
 
     public function handle(Event $event): void
@@ -46,12 +45,9 @@ class PostWriteListener implements IEventListener
         }
 
         $node = $event->getNode();
-        if ($node instanceof Folder) {
-            return;
-        }
 
         // Check the mime type first
-        if (!$this->timelineWrite->getFileType($node)) {
+        if (!Index::isSupported($node)) {
             return;
         }
 

lib/Migration/Version500000Date20230414042534.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Your name <your@email.com>
+ * @author Your name <your@email.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Migration;
+
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version500000Date20230414042534 extends SimpleMigrationStep
+{
+    /**
+     * @param \Closure(): ISchemaWrapper $schemaClosure
+     */
+    public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+    }
+
+    /**
+     * @param \Closure(): ISchemaWrapper $schemaClosure
+     */
+    public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
+    {
+        /** @var ISchemaWrapper $schema */
+        $schema = $schemaClosure();
+
+        $table = $schema->getTable('memories_livephoto');
+
+        if (!$table->hasColumn('orphan')) {
+            $table->addColumn('orphan', Types::BOOLEAN, [
+                'notnull' => false,
+                'default' => false,
+            ]);
+        }
+
+        return $schema;
+    }
+
+    /**
+     * @param \Closure(): ISchemaWrapper $schemaClosure
+     */
+    public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+    }
+}

lib/Service/Index.php

@@ -30,13 +30,17 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\File;
 use OCP\Files\Folder;
 use OCP\Files\IRootFolder;
+use OCP\Files\Node;
 use OCP\IDBConnection;
 use OCP\IPreview;
 use OCP\ITempManager;
 use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Output\OutputInterface;
 
 class Index
 {
+    public ?OutputInterface $output;
+
     protected IRootFolder $rootFolder;
     protected TimelineWrite $timelineWrite;
     protected IDBConnection $db;
@@ -74,12 +78,14 @@ class Index
         $mode = Util::getSystemConfig('memories.index.mode');
         if (null !== $folder) {
             $paths = [$folder];
-        }  elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
+        } elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
             $paths = ['/'];
         } elseif ('2' === $mode) { // timeline
             $paths = \OCA\Memories\Exif::getTimelinePaths($uid);
         } elseif ('3' === $mode) { // custom
             $paths = [Util::getSystemConfig('memories.index.path')];
+        } else {
+            throw new \Exception('Invalid index mode');
         }
 
         // If a folder is specified, traverse only that folder
@@ -90,11 +96,8 @@ class Index
                     throw new \Exception('Not a folder');
                 }
             } catch (\Exception $e) {
-                if (\OC::$CLI && null !== $folder) { // admin asked for this explicitly
+                $this->error("The specified folder {$path} does not exist for {$uid}");
-                    throw new \Exception("The specified folder {$path} does not exist for ${uid}");
-                }
 
-                $this->logger->warning("The specified folder {$path} does not exist for ${uid}");
                 continue;
             }
 
@@ -138,24 +141,18 @@ class Index
                 ->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
             ;
 
-            // TODO: check if forcing a refresh is needed
+            // Filter out files that are already indexed
-            // Check in memories table
+            $addFilter = function (string $table, string $alias) use (&$query) {
-            $query->leftJoin('f', 'memories', 'm', $query->expr()->andX(
+                $query->leftJoin('f', $table, $alias, $query->expr()->andX(
-                $query->expr()->eq('f.fileid', 'm.fileid'),
+                    $query->expr()->eq('f.fileid', "$alias.fileid"),
-                $query->expr()->eq('f.mtime', 'm.mtime')
+                    $query->expr()->eq('f.mtime', "$alias.mtime"),
-            ));
+                    $query->expr()->eq("$alias.orphan", $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL))
+                ));
 
-            // Check in livephoto table
+                $query->andWhere($query->expr()->isNull("$alias.fileid"));
-            $query->leftJoin('f', 'memories_livephoto', 'mlp', $query->expr()->andX(
+            };
-                $query->expr()->eq('f.fileid', 'mlp.fileid'),
+            $addFilter('memories', 'm');
-                $query->expr()->eq('f.mtime', 'mlp.mtime')
+            $addFilter('memories_livephoto', 'lp');
-            ));
-
-            // Exclude files that are already indexed
-            $query->andWhere($query->expr()->andX(
-                $query->expr()->isNull('m.mtime'),
-                $query->expr()->isNull('mlp.mtime'),
-            ));
 
             // Get file IDs to actually index
             $fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
@@ -188,10 +185,7 @@ class Index
         try {
             $this->timelineWrite->processFile($file);
         } catch (\Exception $e) {
-            $this->logger->error('Failed to index file {file}: {error}', [
+            $this->error("Failed to index file {$file->getPath()}: {$e->getMessage()}");
-                'file' => $file->getPath(),
-                'error' => $e->getMessage(),
-            ]);
         }
 
         $this->tempManager->clean();
@@ -245,7 +239,7 @@ class Index
     /**
      * Check if a file is supported.
      */
-    public static function isSupported(File $file): bool
+    public static function isSupported(Node $file): bool
     {
         return \in_array($file->getMimeType(), self::getMimeList(), true);
     }
@@ -257,4 +251,22 @@ class Index
     {
         return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
     }
+
+    /** Log to console if CLI or logger */
+    private function error(string $message)
+    {
+        $this->logger->error($message);
+
+        if ($this->output) {
+            $this->output->writeln("<error>{$message}</error>");
+        }
+    }
+
+    /** Log to console if CLI */
+    private function log(string $message)
+    {
+        if ($this->output) {
+            $this->output->writeln($message);
+        }
+    }
 }