index: move to service (partial)

Signed-off-by: Varun Patil <varunpatil@ucla.edu>

lib/BinExt.php

@@ -55,12 +55,33 @@ class BinExt
             throw new \Exception("failed to run exiftool: {$cmd}");
         }
 
+        // Check version
         $version = trim($out);
         $target = self::EXIFTOOL_VER;
         if (!version_compare($version, $target, '=')) {
             throw new \Exception("version does not match {$version} <==> {$target}");
         }
 
+        // Test with actual file
+        $file = realpath(__DIR__.'/../exiftest.jpg');
+        if (!$file) {
+            throw new \Exception('Could not find EXIF test file');
+        }
+
+        try {
+            $exif = \OCA\Memories\Exif::getExifFromLocalPath($file);
+        } catch (\Exception $e) {
+            throw new \Exception("Couldn't read Exif data from test file: ".$e->getMessage());
+        }
+
+        if (!$exif) {
+            throw new \Exception('Got no Exif data from test file');
+        }
+
+        if (($exp = '2004:08:31 19:52:58') !== ($got = $exif['DateTimeOriginal'])) {
+            throw new \Exception("Got wrong Exif data from test file {$exp} <==> {$got}");
+        }
+
         return true;
     }
 

lib/Command/Index.php

@@ -23,17 +23,11 @@ declare(strict_types=1);
 
 namespace OCA\Memories\Command;
 
-use OC\DB\Connection;
+use OCA\Memories\BinExt;
-use OC\DB\SchemaWrapper;
-use OCA\Memories\AppInfo\Application;
 use OCA\Memories\Db\TimelineWrite;
-use OCP\Files\File;
+use OCA\Memories\Service;
-use OCP\Files\Folder;
 use OCP\Files\IRootFolder;
 use OCP\IConfig;
-use OCP\IDBConnection;
-use OCP\IPreview;
-use OCP\ITempManager;
 use OCP\IUser;
 use OCP\IUserManager;
 use Symfony\Component\Console\Command\Command;
@@ -44,17 +38,15 @@ use Symfony\Component\Console\Output\OutputInterface;
 
 class IndexOpts
 {
-    public bool $refresh = false;
+    public bool $force = false;
     public bool $clear = false;
-    public bool $cleanup = false;
     public ?string $user = null;
     public ?string $folder = null;
 
     public function __construct(InputInterface $input)
     {
-        $this->refresh = (bool) $input->getOption('refresh');
+        $this->force = (bool) $input->getOption('force');
         $this->clear = (bool) $input->getOption('clear');
-        $this->cleanup = (bool) $input->getOption('cleanup');
         $this->user = $input->getOption('user');
         $this->folder = $input->getOption('folder');
     }
@@ -67,22 +59,13 @@ class Index extends Command
 
     protected IUserManager $userManager;
     protected IRootFolder $rootFolder;
-    protected IPreview $preview;
     protected IConfig $config;
-    protected OutputInterface $output;
+    protected Service\Index $indexer;
-    protected IDBConnection $connection;
-    protected Connection $connectionForSchema;
     protected TimelineWrite $timelineWrite;
-    protected ITempManager $tempManager;
 
-    // Stats
+    // IO
-    private int $nUser = 0;
+    private InputInterface $input;
-    private int $nProcessed = 0;
+    private OutputInterface $output;
-    private int $nSkipped = 0;
-    private int $nInvalid = 0;
-    private int $nNoMedia = 0;
-
-    // Helper for the progress bar
     private ConsoleSectionOutput $outputSection;
 
     // Command options
@@ -91,22 +74,17 @@ class Index extends Command
     public function __construct(
         IRootFolder $rootFolder,
         IUserManager $userManager,
-        IPreview $preview,
         IConfig $config,
-        IDBConnection $connection,
+        Service\Index $indexer,
-        Connection $connectionForSchema,
+        TimelineWrite $timelineWrite
-        ITempManager $tempManager
     ) {
         parent::__construct();
 
         $this->userManager = $userManager;
         $this->rootFolder = $rootFolder;
-        $this->preview = $preview;
         $this->config = $config;
-        $this->connection = $connection;
+        $this->indexer = $indexer;
-        $this->connectionForSchema = $connectionForSchema;
+        $this->timelineWrite = $timelineWrite;
-        $this->tempManager = $tempManager;
-        $this->timelineWrite = new TimelineWrite($connection);
     }
 
     protected function configure(): void
@@ -115,77 +93,34 @@ class Index extends Command
             ->setName('memories:index')
             ->setDescription('Generate photo entries')
             ->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Index only the specified user')
-            ->addOption('folder', null, InputOption::VALUE_REQUIRED, 'Index only the specified folder')
+            ->addOption('folder', null, InputOption::VALUE_REQUIRED, 'Index only the specified folder (relative to the user\'s root)')
-            ->addOption('refresh', 'f', InputOption::VALUE_NONE, 'Refresh existing entries')
+            ->addOption('force', 'f', InputOption::VALUE_NONE, 'Force refresh of existing index entries')
-            ->addOption('clear', null, InputOption::VALUE_NONE, 'Clear existing index before creating a new one (slow)')
+            ->addOption('clear', null, InputOption::VALUE_NONE, 'Clear all existing index entries')
-            ->addOption('cleanup', null, InputOption::VALUE_NONE, 'Remove orphaned entries from index (e.g. from .nomedia files)')
         ;
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
-        // Add missing indices
+        // Store input/output/opts for later use
-        $output->writeln('Checking database indices');
+        $this->input = $input;
-        \OCA\Memories\Db\AddMissingIndices::run(new SchemaWrapper($this->connectionForSchema), $this->connectionForSchema);
+        $this->output = $output;
-
-        // Print mime type support information
-        $output->writeln("\nMIME Type support:");
-        $mimes = array_merge(Application::IMAGE_MIMES, Application::VIDEO_MIMES);
-        $someUnsupported = false;
-        foreach ($mimes as $mimeType) {
-            if ($this->preview->isMimeSupported($mimeType)) {
-                $output->writeln("  {$mimeType}: supported");
-            } else {
-                $output->writeln("  {$mimeType}: <error>not supported</error>");
-                $someUnsupported = true;
-            }
-        }
-
-        // Print file type support info
-        if ($someUnsupported) {
-            $output->writeln("\nSome file types are not supported by your preview provider.\nPlease see https://github.com/pulsejet/memories/wiki/File-Type-Support\n");
-        } else {
-            $output->writeln("\nAll file types are supported by your preview provider.\n");
-        }
-
-        // Get options and arguments
         $this->opts = new IndexOpts($input);
 
-        // Clear index if asked for this
-        if ($this->opts->clear && $input->isInteractive()) {
-            $output->write('Are you sure you want to clear the existing index? (y/N): ');
-            $answer = trim(fgets(STDIN));
-            if ('y' !== $answer) {
-                $output->writeln('Aborting');
-
-                return 1;
-            }
-        }
-        if ($this->opts->clear) {
-            $this->timelineWrite->clear();
-            $output->writeln('Cleared existing index');
-        }
-
-        // Detect incompatible options
-        if ($this->opts->cleanup && ($this->opts->user || $this->opts->folder)) {
-            $output->writeln('<error>Cannot use --cleanup with --user or --folder</error>');
-
-            return 1;
-        }
-
-        // Orphan all entries so we can delete them later
-        // Refresh works similarly, with a different flag on the process call
-        if ($this->opts->cleanup || $this->opts->refresh) {
-            $output->write('Marking all entries for refresh / cleanup ... ');
-            $count = $this->timelineWrite->orphanAll();
-            $output->writeln("{$count} marked");
-        }
-
-        // Run with the static process
         try {
+            // Use static exiftool process
             \OCA\Memories\Exif::ensureStaticExiftoolProc();
+            if (!BinExt::testExiftool()) { // throws
+                throw new \Exception('exiftool could not be executed or test failed');
+            }
 
-            return $this->executeNow($output);
+            // Perform steps based on opts
+            $this->checkClear();
+            $this->checkForce();
+
+            // Run the indexer
+            $this->runIndex();
+
+            return 0;
         } catch (\Exception $e) {
             $this->output->writeln("<error>{$e->getMessage()}</error>");
 
@@ -195,197 +130,60 @@ class Index extends Command
         }
     }
 
-    protected function executeNow(OutputInterface $output): int
+    /**
+     * Check and act on the clear option if set.
+     */
+    protected function checkClear(): void
     {
-        // Refuse to run without exiftool
+        if ($this->opts->clear) {
-        if (!$this->testExif()) {
+            if ($this->input->isInteractive()) {
-            error_log('FATAL: exiftool could not be executed or test failed');
+                $this->output->write('Are you sure you want to clear the existing index? (y/N): ');
-            error_log('Make sure you have perl 5 installed in PATH');
+                if ('y' !== trim(fgets(STDIN))) {
+                    $this->output->writeln('Aborting');
 
-            return 1;
+                    exit;
+                }
             }
 
-        // Time measurement
+            $this->timelineWrite->clear();
-        $startTime = microtime(true);
+            $this->output->writeln('Cleared existing index');
-
+        }
-        if (\OCA\Memories\Util::isEncryptionEnabled()) {
-            // Can work with server-side but not with e2e encryption, see https://github.com/pulsejet/memories/issues/99
-            error_log('FATAL: Only server-side encryption (OC_DEFAULT_MODULE) is supported, but another encryption module is enabled. Aborted.');
-
-            return 1;
     }
-        $this->output = $output;
 
+    /**
+     * Check and act on the force option if set.
+     */
+    protected function checkForce(): void
+    {
+        if ($this->opts->force) {
+            $this->output->writeln('Forcing refresh of existing index entries');
+
+            // TODO
+        }
+    }
+
+    /**
+     * Run the indexer.
+     */
+    protected function runIndex(): void
+    {
         // Call indexing for specified or each user
         if ($uid = $this->opts->user) {
             if ($user = $this->userManager->get($uid)) {
-                $this->indexOneUser($user);
+                $this->indexer->indexUser($user->getUID(), $this->opts->folder);
             } else {
                 throw new \Exception("User {$uid} not found");
             }
         } else {
             $this->userManager->callForSeenUsers(function (IUser $user) {
-                $this->indexOneUser($user);
+                try {
+                    $uid = $user->getUID();
+                    $this->output->writeln("Indexing user {$uid}");
+                    $this->indexer->indexUser($uid, $this->opts->folder);
+                } catch (\Exception $e) {
+                    $this->output->writeln("<error>{$e->getMessage()}</error>");
+                }
             });
         }
-
-        // Clear orphans if asked for this
-        if (($this->opts->cleanup || $this->opts->refresh) && !($this->opts->user || $this->opts->folder)) {
-            $output->write('Deleting orphaned entries ... ');
-            $count = $this->timelineWrite->removeOrphans();
-            $output->writeln("{$count} deleted");
-        }
-
-        // Show some stats
-        $endTime = microtime(true);
-        $execTime = (int) (($endTime - $startTime) * 1000) / 1000;
-        $nTotal = $this->nInvalid + $this->nSkipped + $this->nProcessed + $this->nNoMedia;
-        $this->output->writeln('==========================================');
-        $this->output->writeln("Checked {$nTotal} files of {$this->nUser} users in {$execTime} sec");
-        $this->output->writeln($this->nInvalid.' not valid media items');
-        $this->output->writeln($this->nNoMedia.' .nomedia folders ignored');
-        $this->output->writeln($this->nSkipped.' skipped because unmodified');
-        $this->output->writeln($this->nProcessed.' (re-)processed');
-        $this->output->writeln('==========================================');
-
-        return 0;
-    }
-
-    /** Make sure exiftool is available */
-    private function testExif()
-    {
-        $testfilepath = __DIR__.'/../../exiftest.jpg';
-        $testfile = realpath($testfilepath);
-        if (!$testfile) {
-            error_log("Couldn't find Exif test file {$testfile}");
-
-            return false;
-        }
-
-        $exif = null;
-
-        try {
-            $exif = \OCA\Memories\Exif::getExifFromLocalPath($testfile);
-        } catch (\Exception $e) {
-            error_log("Couldn't read Exif data from test file: ".$e->getMessage());
-
-            return false;
-        }
-
-        if (!$exif) {
-            error_log('Got blank Exif data from test file');
-
-            return false;
-        }
-
-        if ('2004:08:31 19:52:58' !== $exif['DateTimeOriginal']) {
-            error_log('Got unexpected Exif data from test file');
-
-            return false;
-        }
-
-        return true;
-    }
-
-    private function indexOneUser(IUser $user): void
-    {
-        \OC_Util::tearDownFS();
-        \OC_Util::setupFS($user->getUID());
-
-        $uid = $user->getUID();
-        $folder = $this->rootFolder->getUserFolder($uid);
-
-        if ($path = $this->opts->folder) {
-            try {
-                $folder = $folder->get($path);
-            } catch (\OCP\Files\NotFoundException $e) {
-                $this->output->writeln("<error>Folder {$path} not found for user {$uid}</error>");
-
-                return;
-            }
-
-            if (!$folder instanceof Folder) {
-                $this->output->writeln("<error>Path {$path} is not a folder for user {$uid}</error>");
-
-                return;
-            }
-        }
-
-        $this->outputSection = $this->output->section();
-        ++$this->nUser;
-
-        $this->outputSection->overwrite("Scanning files for {$uid}");
-        $this->indexFolder($folder);
-        $this->outputSection->overwrite("Scanned all files for {$uid}");
-    }
-
-    private function indexFolder(Folder $folder): void
-    {
-        try {
-            // Respect the '.nomedia' file. If present don't traverse the folder
-            if ($folder->nodeExists('.nomedia')) {
-                ++$this->nNoMedia;
-
-                return;
-            }
-
-            $nodes = $folder->getDirectoryListing();
-
-            foreach ($nodes as $i => $node) {
-                if ($node instanceof Folder) {
-                    $this->indexFolder($node);
-                } elseif ($node instanceof File) {
-                    $path = $node->getPath();
-                    $path = \strlen($path) > 80 ? '...'.substr($path, -77) : $path;
-
-                    $this->outputSection->overwrite("Scanning {$path}");
-                    $this->indexFile($node);
-                    $this->tempManager->clean();
-                }
-            }
-        } catch (\Exception $e) {
-            $this->output->writeln(sprintf(
-                '<error>Could not scan folder %s: %s</error>',
-                $folder->getPath(),
-                $e->getMessage()
-            ));
-        }
-    }
-
-    private function indexFile(File $file): void
-    {
-        // Process the file
-        $res = 1;
-
-        try {
-            // If refreshing the index, force reprocessing
-            // when the file is still an orphan. this way, the
-            // files are reprocessed exactly once
-            $force = $this->opts->refresh ? 2 : 0;
-
-            // (re-)process the file
-            $res = $this->timelineWrite->processFile($file, $force);
-
-            // If the file was processed successfully,
-            // remove it from the orphan list
-            if ($this->opts->cleanup || $this->opts->refresh) {
-                $this->timelineWrite->unorphan($file);
-            }
-        } catch (\Error $e) {
-            $this->output->writeln(sprintf(
-                '<error>Could not process file %s: %s</error>',
-                $file->getPath(),
-                $e->getMessage()
-            ));
-            $this->output->writeln($e->getTraceAsString());
-        }
-
-        if (2 === $res) {
-            ++$this->nProcessed;
-        } elseif (1 === $res) {
-            ++$this->nSkipped;
-        } else {
-            ++$this->nInvalid;
-        }
     }
 }

lib/Controller/ArchiveController.php

@@ -42,7 +42,6 @@ class ArchiveController extends GenericApiController
     public function archive(string $id): Http\Response
     {
         return Util::guardEx(function () use ($id) {
-            $uid = Util::getUID();
             $userFolder = Util::getUserFolder();
 
             // Check for permissions and get numeric Id
@@ -58,8 +57,7 @@ class ArchiveController extends GenericApiController
             }
 
             // Create archive folder in the root of the user's configured timeline
-            $configPath = Exif::removeExtraSlash(Exif::getPhotosPath($this->config, $uid));
+            $configPaths = Exif::getTimelinePaths(Util::getUID());
-            $configPaths = explode(';', $configPath);
             $timelineFolders = [];
             $timelinePaths = [];
 

lib/Controller/OtherController.php

@@ -128,6 +128,16 @@ class OtherController extends GenericApiController
             // Check for system perl
             $status['perl'] = $this->getExecutableStatus(exec('which perl'));
 
+            // Check number of indexed files
+            $index = \OC::$server->get(\OCA\Memories\Service\Index::class);
+            $status['indexed_count'] = $index->getIndexedCount();
+
+            // Check supported preview mimes
+            $status['mimes'] = $index->getPreviewMimes($index->getAllMimes());
+
+            // Check for bad encryption module
+            $status['bad_encryption'] = \OCA\Memories\Util::isEncryptionEnabled();
+
             // Get GIS status
             $places = \OC::$server->get(\OCA\Memories\Service\Places::class);
 

lib/Db/AddMissingIndices.php

@@ -2,19 +2,18 @@
 
 namespace OCA\Memories\Db;
 
-use OC\DB\Connection;
 use OC\DB\SchemaWrapper;
 
 class AddMissingIndices
 {
     /**
      * Add missing indices to the database schema.
-     *
-     * @param SchemaWrapper   $schema     Schema wrapper
-     * @param null|Connection $connection Connection to db
      */
-    public static function run(SchemaWrapper $schema, $connection)
+    public static function run()
     {
+        $connection = \OC::$server->get(\OC\DB\Connection::class);
+        $schema = new SchemaWrapper($connection);
+
         // Should migrate at end
         $shouldMigrate = false;
 

lib/Db/FsManager.php

@@ -98,14 +98,14 @@ class FsManager
                 $folder = $userFolder->get(Exif::removeExtraSlash($folderPath));
                 $root->addFolder($folder);
             } else {
-                $timelinePath = $this->request->getParam('timelinePath', Exif::getPhotosPath($this->config, $uid));
+                $paths = Exif::getTimelinePaths($uid);
-                $timelinePath = Exif::removeExtraSlash($timelinePath);
+                if ($path = $this->request->getParam('timelinePath', null)) {
+                    $paths = [Exif::removeExtraSlash($path)];
+                }
 
                 // Multiple timeline path support
-                $paths = explode(';', $timelinePath);
+                foreach ($paths as $path) {
-                foreach ($paths as &$path) {
+                    $root->addFolder($userFolder->get($path));
-                    $folder = $userFolder->get(trim($path));
-                    $root->addFolder($folder);
                 }
                 $root->addMountPoints();
             }

lib/Db/LivePhoto.php

@@ -19,7 +19,7 @@ class LivePhoto
     }
 
     /** Check if a given Exif data is the video part of a Live Photo */
-    public function isVideoPart(array &$exif)
+    public function isVideoPart(array $exif)
     {
         return \array_key_exists('MIMEType', $exif)
                && 'video/quicktime' === $exif['MIMEType']
@@ -27,7 +27,7 @@ class LivePhoto
     }
 
     /** Get liveid from photo part */
-    public function getLivePhotoId(File &$file, array &$exif)
+    public function getLivePhotoId(File $file, array $exif)
     {
         // Apple JPEG (MOV has ContentIdentifier)
         if (\array_key_exists('MediaGroupUUID', $exif)) {
@@ -100,7 +100,10 @@ class LivePhoto
         return '';
     }
 
-    public function processVideoPart(File &$file, array &$exif)
+    /**
+     * Process video part of Live Photo.
+     */
+    public function processVideoPart(File $file, array $exif)
     {
         $fileId = $file->getId();
         $mtime = $file->getMTime();
@@ -142,4 +145,16 @@ class LivePhoto
             }
         }
     }
+
+    /**
+     * Delete entry from memories_livephoto table.
+     */
+    public function deleteVideoPart(File $file): void
+    {
+        $query = $this->connection->getQueryBuilder();
+        $query->delete('memories_livephoto')
+            ->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT)))
+        ;
+        $query->executeStatement();
+    }
 }

lib/Db/TimelineWrite.php

@@ -4,8 +4,8 @@ declare(strict_types=1);
 
 namespace OCA\Memories\Db;
 
-use OCA\Memories\AppInfo\Application;
 use OCA\Memories\Exif;
+use OCA\Memories\Service\Index;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\File;
 use OCP\IDBConnection;
@@ -32,105 +32,50 @@ class TimelineWrite
         $this->livePhoto = new LivePhoto($connection);
     }
 
-    /**
-     * Check if a file has a valid mimetype for processing.
-     *
-     * @return int 0 for invalid, 1 for image, 2 for video
-     */
-    public function getFileType(File $file): int
-    {
-        $mime = $file->getMimeType();
-        if (\in_array($mime, Application::IMAGE_MIMES, true)) {
-            // Make sure preview generator supports the mime type
-            if (!$this->preview->isMimeSupported($mime)) {
-                return 0;
-            }
-
-            return 1;
-        }
-        if (\in_array($mime, Application::VIDEO_MIMES, true)) {
-            return 2;
-        }
-
-        return 0;
-    }
-
     /**
      * Process a file to insert Exif data into the database.
      *
      * @param File $file  File node to process
-     * @param int  $force 0 = none, 1 = force, 2 = force if orphan
+     * @param bool $force Update the record even if the file has not changed
-     *
-     * @return int 2 if processed, 1 if skipped, 0 if not valid
      */
-    public function processFile(File $file, int $force = 0): int
+    public function processFile(File $file, bool $force = false): bool
     {
-        // There is no easy way to UPSERT in a standard SQL way, so just
-        // do multiple calls. The worst that can happen is more updates,
-        // but that's not a big deal.
-        // https://stackoverflow.com/questions/15252213/sql-standard-upsert-call
-
         // Check if we want to process this file
-        $fileType = $this->getFileType($file);
+        if (!Index::isSupported($file)) {
-        $isvideo = (2 === $fileType);
+            return false;
-        if (!$fileType) {
-            return 0;
         }
 
         // Get parameters
         $mtime = $file->getMtime();
         $fileId = $file->getId();
+        $isvideo = Index::isVideo($file);
 
-        // Check if need to update
+        // Get previous row
-        $query = $this->connection->getQueryBuilder();
+        $prevRow = $this->getCurrentRow($fileId);
-        $query->select('fileid', 'mtime', 'mapcluster', 'orphan', 'lat', 'lon')
-            ->from('memories')
-            ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
-        ;
-        $cursor = $query->executeQuery();
-        $prevRow = $cursor->fetch();
-        $cursor->closeCursor();
-
-        // Check in live-photo table in case this is a video part of a Live Photo
-        if (!$prevRow) {
-            $query = $this->connection->getQueryBuilder();
-            $query->select('fileid', 'mtime')
-                ->from('memories_livephoto')
-                ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
-            ;
-            $cursor = $query->executeQuery();
-            $prevRow = $cursor->fetch();
-            $cursor->closeCursor();
-        }
-
-        // Check if a forced update is required
-        $isForced = (1 === $force);
-        if (2 === $force) {
-            $isForced = !$prevRow
-                        // Could be live video, force regardless
-                        || !\array_key_exists('orphan', $prevRow)
-                        // If orphan, force for sure
-                        || $prevRow['orphan'];
-        }
 
         // Skip if not forced and file has not changed
-        if (!$isForced && $prevRow && ((int) $prevRow['mtime'] === $mtime)) {
+        if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime)) {
-            return 1;
+            return false;
         }
 
         // Get exif data
-        $exif = [];
-
         try {
             $exif = Exif::getExifFromFile($file);
         } catch (\Exception $e) {
+            $exif = [];
         }
 
         // Hand off if Live Photo video part
         if ($isvideo && $this->livePhoto->isVideoPart($exif)) {
             $this->livePhoto->processVideoPart($file, $exif);
 
-            return 2;
+            return true;
+        }
+
+        // Delete video part if it is no longer valid
+        if ($prevRow && !\array_key_exists('mapcluster', $prevRow)) {
+            $this->livePhoto->deleteVideoPart($file);
+            $prevRow = null;
         }
 
         // Video parameters
@@ -166,6 +111,7 @@ class TimelineWrite
         $exifJson = $this->getExifJson($exif);
 
         // Parameters for insert or update
+        $query = $this->connection->getQueryBuilder();
         $params = [
             'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
             'objectid' => $query->createNamedParameter((string) $fileId, IQueryBuilder::PARAM_STR),
@@ -183,29 +129,26 @@ class TimelineWrite
             'mapcluster' => $query->createNamedParameter($mapCluster, IQueryBuilder::PARAM_INT),
         ];
 
+        // There is no easy way to UPSERT in standard SQL
+        // https://stackoverflow.com/questions/15252213/sql-standard-upsert-call
+        try {
             if ($prevRow) {
-            // Update existing row
-            // No need to set objectid again
                 $query->update('memories')
                     ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
                 ;
                 foreach ($params as $key => $value) {
-                if ('objectid' !== $key && 'fileid' !== $key) {
                     $query->set($key, $value);
                 }
-            }
-            $query->executeStatement();
             } else {
-            // Try to create new row
-            try {
                 $query->insert('memories')->values($params);
-                $query->executeStatement();
-            } catch (\Exception $ex) {
-                error_log('Failed to create memories record: '.$ex->getMessage());
-            }
             }
 
-        return 2;
+            return $query->executeStatement() > 0;
+        } catch (\Exception $ex) {
+            error_log('Failed to create memories record: '.$ex->getMessage());
+
+            return false;
+        }
     }
 
     /**
@@ -249,6 +192,25 @@ class TimelineWrite
         }
     }
 
+    /**
+     * Get the current row for a file_id, from either table.
+     */
+    private function getCurrentRow(int $fileId): ?array
+    {
+        $fetch = function (string $table) use ($fileId) {
+            $query = $this->connection->getQueryBuilder();
+
+            return $query->select('*')
+                ->from($table)
+                ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
+                ->executeQuery()
+                ->fetch()
+            ;
+        };
+
+        return $fetch('memories') ?: $fetch('memories_livephoto') ?: null;
+    }
+
     /**
      * Convert EXIF data to filtered JSON string.
      */

lib/Exif.php

@@ -65,16 +65,13 @@ class Exif
     }
 
     /**
-     * Get the path to the user's configured photos directory.
+     * Get list of timeline paths as array.
      */
-    public static function getPhotosPath(IConfig $config, string &$userId)
+    public static function getTimelinePaths(string $uid): array
     {
-        $p = $config->getUserValue($userId, Application::APPNAME, 'timelinePath', '');
+        $config = \OC::$server->get(IConfig::class);
-        if (empty($p)) {
+        $paths = $config->getUserValue($uid, Application::APPNAME, 'timelinePath', null) ?? 'Photos/';
-            return 'Photos/';
+        return array_map(fn ($p) => self::sanitizePath(trim($p)), explode(';', $paths));
-        }
-
-        return self::sanitizePath($p);
     }
 
     /**

lib/Migration/Repair.php

@@ -26,6 +26,9 @@ class Repair implements IRepairStep
 
     public function run(IOutput $output): void
     {
+        // Add missing indices
+        \OCA\Memories\Db\AddMissingIndices::run();
+
         // kill any instances of go-vod and exiftool
         Util::pkill(BinExt::getName('go-vod'));
         Util::pkill(BinExt::getName('exiftool'));

lib/Migration/Version400308Date20221026151748.php

@@ -52,7 +52,7 @@ class Version400308Date20221026151748 extends SimpleMigrationStep
         $fileCacheTable->addIndex(['parent', 'mimetype'], 'memories_parent_mimetype');
 
         // Add other indices
-        return \OCA\Memories\Db\AddMissingIndices::run($schema, null);
+        return $schema;
     }
 
     /**

lib/Service/Index.php

@@ -0,0 +1,260 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2022, Varun Patil <radialapps@gmail.com>
+ * @author Varun Patil <radialapps@gmail.com>
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Service;
+
+use OCA\Memories\AppInfo\Application;
+use OCA\Memories\Db\TimelineWrite;
+use OCA\Memories\Util;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\File;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\IDBConnection;
+use OCP\IPreview;
+use OCP\ITempManager;
+use Psr\Log\LoggerInterface;
+
+class Index
+{
+    protected IRootFolder $rootFolder;
+    protected TimelineWrite $timelineWrite;
+    protected IDBConnection $db;
+    protected ITempManager $tempManager;
+    protected LoggerInterface $logger;
+
+    private static ?array $mimeList = null;
+
+    public function __construct(
+        IRootFolder $rootFolder,
+        TimelineWrite $timelineWrite,
+        IDBConnection $db,
+        ITempManager $tempManager,
+        LoggerInterface $logger
+    ) {
+        $this->rootFolder = $rootFolder;
+        $this->timelineWrite = $timelineWrite;
+        $this->db = $db;
+        $this->tempManager = $tempManager;
+        $this->logger = $logger;
+    }
+
+    /**
+     * Index all files for a user.
+     */
+    public function indexUser(string $uid, ?string $folder = null): void
+    {
+        \OC_Util::tearDownFS();
+        \OC_Util::setupFS($uid);
+
+        // Get the root folder of the user
+        $root = $this->rootFolder->getUserFolder($uid);
+
+        // Get paths of folders to index
+        $mode = Util::getSystemConfig('memories.index.mode');
+        if (null !== $folder) {
+            $paths = [$folder];
+        }  elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
+            $paths = ['/'];
+        } elseif ('2' === $mode) { // timeline
+            $paths = \OCA\Memories\Exif::getTimelinePaths($uid);
+        } elseif ('3' === $mode) { // custom
+            $paths = [Util::getSystemConfig('memories.index.path')];
+        }
+
+        // If a folder is specified, traverse only that folder
+        foreach ($paths as $path) {
+            try {
+                $node = $root->get($path);
+                if (!$node instanceof Folder) {
+                    throw new \Exception('Not a folder');
+                }
+            } catch (\Exception $e) {
+                if (\OC::$CLI && null !== $folder) { // admin asked for this explicitly
+                    throw new \Exception("The specified folder {$path} does not exist for ${uid}");
+                }
+
+                $this->logger->warning("The specified folder {$path} does not exist for ${uid}");
+                continue;
+            }
+
+            $this->indexFolder($node);
+        }
+    }
+
+    /**
+     * Index all files in a folder.
+     *
+     * @param Folder $folder folder to index
+     */
+    public function indexFolder(Folder $folder): void
+    {
+        // Respect the '.nomedia' file. If present don't traverse the folder
+        if ($folder->nodeExists('.nomedia')) {
+            return;
+        }
+
+        // Get all files and folders in this folders
+        $nodes = $folder->getDirectoryListing();
+
+        // Filter files that are supported
+        $mimes = self::getMimeList();
+        $files = array_filter($nodes, fn ($n) => $n instanceof File && \in_array($n->getMimeType(), $mimes, true));
+
+        // Create an associative array with file ID as key
+        $files = array_combine(array_map(fn ($n) => $n->getId(), $files), $files);
+
+        // Chunk array into some files each (DBs have limitations on IN clause)
+        $chunks = array_chunk($files, 250, true);
+
+        // Check files in each chunk
+        foreach ($chunks as $chunk) {
+            $fileIds = array_keys($chunk);
+
+            // Select all files in filecache
+            $query = $this->db->getQueryBuilder();
+            $query->select('f.fileid')
+                ->from('filecache', 'f')
+                ->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
+            ;
+
+            // TODO: check if forcing a refresh is needed
+            // Check in memories table
+            $query->leftJoin('f', 'memories', 'm', $query->expr()->andX(
+                $query->expr()->eq('f.fileid', 'm.fileid'),
+                $query->expr()->eq('f.mtime', 'm.mtime')
+            ));
+
+            // Check in livephoto table
+            $query->leftJoin('f', 'memories_livephoto', 'mlp', $query->expr()->andX(
+                $query->expr()->eq('f.fileid', 'mlp.fileid'),
+                $query->expr()->eq('f.mtime', 'mlp.mtime')
+            ));
+
+            // Exclude files that are already indexed
+            $query->andWhere($query->expr()->andX(
+                $query->expr()->isNull('m.mtime'),
+                $query->expr()->isNull('mlp.mtime'),
+            ));
+
+            // Get file IDs to actually index
+            $fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+
+            // Index files
+            foreach ($fileIds as $fileId) {
+                $this->indexFile($chunk[$fileId]);
+            }
+        }
+
+        // All folders
+        $folders = array_filter($nodes, fn ($n) => $n instanceof Folder);
+        foreach ($folders as $folder) {
+            try {
+                $this->indexFolder($folder);
+            } catch (\Exception $e) {
+                $this->logger->error('Failed to index folder {folder}: {error}', [
+                    'folder' => $folder->getPath(),
+                    'error' => $e->getMessage(),
+                ]);
+            }
+        }
+    }
+
+    /**
+     * Index a single file.
+     */
+    public function indexFile(File $file): void
+    {
+        try {
+            $this->timelineWrite->processFile($file);
+        } catch (\Exception $e) {
+            $this->logger->error('Failed to index file {file}: {error}', [
+                'file' => $file->getPath(),
+                'error' => $e->getMessage(),
+            ]);
+        }
+
+        $this->tempManager->clean();
+    }
+
+    /**
+     * Get total number of files that are indexed.
+     */
+    public function getIndexedCount()
+    {
+        $query = $this->db->getQueryBuilder();
+        $query->select($query->createFunction('COUNT(DISTINCT fileid)'))
+            ->from('memories')
+        ;
+
+        return (int) $query->executeQuery()->fetchOne();
+    }
+
+    /**
+     * Get list of MIME types to process.
+     */
+    public static function getMimeList(): array
+    {
+        return self::$mimeList ??= array_merge(
+            self::getPreviewMimes(Application::IMAGE_MIMES),
+            Application::VIDEO_MIMES,
+        );
+    }
+
+    /**
+     * Get list of MIME types that have a preview.
+     */
+    public static function getPreviewMimes(array $source): array
+    {
+        $preview = \OC::$server->get(IPreview::class);
+
+        return array_filter($source, fn ($m) => $preview->isMimeSupported($m));
+    }
+
+    /**
+     * Get list of all supported MIME types.
+     */
+    public static function getAllMimes(): array
+    {
+        return array_merge(
+            Application::IMAGE_MIMES,
+            Application::VIDEO_MIMES,
+        );
+    }
+
+    /**
+     * Check if a file is supported.
+     */
+    public static function isSupported(File $file): bool
+    {
+        return \in_array($file->getMimeType(), self::getMimeList(), true);
+    }
+
+    /**
+     * Check if a file is a video.
+     */
+    public static function isVideo(File $file): bool
+    {
+        return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
+    }
+}

lib/Util.php

@@ -346,6 +346,16 @@ class Util
             // This requires perl to be available
             'memories.exiftool_no_local' => false,
 
+            // How to index user directories
+            // 0 = auto-index disabled
+            // 1 = index everything
+            // 2 = index only user timelines
+            // 3 = index only configured path
+            'memories.index.mode' => '1',
+
+            // Path to index (only used if indexing mode is 3)
+            'memories.index.path' => '/',
+
             // Places database type identifier
             'memories.gis_type' => -1,
 

src/Admin.vue

@@ -35,6 +35,120 @@
       }}
     </NcCheckboxRadioSwitch>
 
+    <!----------------------------- Index Settings ----------------------------->
+    <h2>{{ t("memories", "Media Indexing") }}</h2>
+
+    <template v-if="status">
+      <NcNoteCard :type="status.indexed_count > 0 ? 'success' : 'warning'">
+        {{
+          t("memories", "{n} media files have been indexed", {
+            n: status.indexed_count,
+          })
+        }}
+      </NcNoteCard>
+      <NcNoteCard type="error" v-if="status.bad_encryption">
+        {{
+          t(
+            "memories",
+            "Only server-side encryption (OC_DEFAULT_MODULE) is supported, but another encryption module is enabled."
+          )
+        }}
+      </NcNoteCard>
+    </template>
+
+    <p>
+      {{
+        t(
+          "memories",
+          "The EXIF indexes are built and checked in a periodic background task. Be careful when selecting anything other than automatic indexing. For example, setting the indexing to only timeline folders may cause delays before media becomes available to users, since the user configures the timeline only after logging in."
+        )
+      }}
+      {{
+        t(
+          "memories",
+          'Folders with a ".nomedia" file are always excluded from indexing.'
+        )
+      }}
+      <NcCheckboxRadioSwitch
+        :checked.sync="indexingMode"
+        value="1"
+        name="idxm_radio"
+        type="radio"
+        @update:checked="update('indexingMode')"
+        >{{ t("memories", "Index all media automatically (recommended)") }}
+      </NcCheckboxRadioSwitch>
+      <NcCheckboxRadioSwitch
+        :checked.sync="indexingMode"
+        value="2"
+        name="idxm_radio"
+        type="radio"
+        @update:checked="update('indexingMode')"
+        >{{ t("memories", "Only index timeline folders (configured by user)") }}
+      </NcCheckboxRadioSwitch>
+      <NcCheckboxRadioSwitch
+        :checked.sync="indexingMode"
+        value="3"
+        name="idxm_radio"
+        type="radio"
+        @update:checked="update('indexingMode')"
+        >{{ t("memories", "Only index a selected path") }}
+      </NcCheckboxRadioSwitch>
+      <NcCheckboxRadioSwitch
+        :checked.sync="indexingMode"
+        value="0"
+        name="idxm_radio"
+        type="radio"
+        @update:checked="update('indexingMode')"
+        >{{ t("memories", "Disable background indexing") }}
+      </NcCheckboxRadioSwitch>
+
+      <NcTextField
+        :label="t('memories', 'Indexing path (relative, all users)')"
+        :label-visible="true"
+        :value="indexingPath"
+        @change="update('indexingPath', $event.target.value)"
+        v-if="indexingMode === '3'"
+      />
+    </p>
+
+    {{
+      t("memories", "For advanced usage, perform a run of indexing by running:")
+    }}
+    <br />
+    <code>occ memories:index</code>
+    <br />
+    {{ t("memories", "Force re-indexing of all files:") }}
+    <br />
+    <code>occ memories:index --force</code>
+    <br />
+    {{ t("memories", "You can limit indexing by user and/or folder:") }}
+    <br />
+    <code>occ memories:index --user=admin --folder=/Photos/</code>
+    <br />
+    {{ t("memories", "Clear all existing index tables:") }}
+    <br />
+    <code>occ memories:index --clear</code>
+    <br />
+
+    <br />
+    {{
+      t(
+        "memories",
+        "The following MIME types are configured for preview generation correctly. More documentation:"
+      )
+    }}
+    <a
+      href="https://github.com/pulsejet/memories/wiki/File-Type-Support"
+      target="_blank"
+    >
+      {{ t("memories", "External Link") }}
+    </a>
+    <br />
+    <code
+      ><template v-for="mime in status.mimes"
+        >{{ mime }}<br :key="mime" /></template
+    ></code>
+
     <!----------------------------- Places ----------------------------->
     <h2>{{ t("memories", "Reverse Geocoding") }}</h2>
 
@@ -83,10 +197,12 @@
       {{
         t(
           "memories",
-          "If the button below does not work for importing the planet data, use 'occ memories:places-setup'."
+          "If the button below does not work for importing the planet data, use the following command:"
         )
       }}
       <br />
+      <code>occ memories:places-setup</code>
+      <br />
       {{
         t(
           "memories",
@@ -147,6 +263,7 @@
         :label-visible="true"
         :value="ffmpegPath"
         @change="update('ffmpegPath', $event.target.value)"
+        :disabled="!enableTranscoding"
       />
 
       <NcTextField
@@ -154,11 +271,13 @@
         :label-visible="true"
         :value="ffprobePath"
         @change="update('ffprobePath', $event.target.value)"
+        :disabled="!enableTranscoding"
       />
 
       <br />
       {{ t("memories", "Global default video quality (user may override)") }}
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding"
         :checked.sync="videoDefaultQuality"
         value="0"
         name="vdq_radio"
@@ -167,6 +286,7 @@
         >{{ t("memories", "Auto (adaptive transcode)") }}
       </NcCheckboxRadioSwitch>
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding"
         :checked.sync="videoDefaultQuality"
         value="-1"
         name="vdq_radio"
@@ -175,6 +295,7 @@
         >{{ t("memories", "Original (transcode with max quality)") }}
       </NcCheckboxRadioSwitch>
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding"
         :checked.sync="videoDefaultQuality"
         value="-2"
         name="vdq_radio"
@@ -189,14 +310,14 @@
       {{
         t(
           "memories",
-          "Memories uses the go-vod transcoder. You can run go-vod exernally (e.g. in a separate Docker container for hardware acceleration) or use the built-in transcoder. To use an external transcoder, enable the following option and follow the instructions at this link:"
+          "Memories uses the go-vod transcoder. You can run go-vod exernally (e.g. in a separate Docker container for hardware acceleration) or use the built-in transcoder. To use an external transcoder, enable the following option and follow the instructions in the documentation:"
         )
       }}
       <a
         target="_blank"
         href="https://github.com/pulsejet/memories/wiki/HW-Transcoding"
       >
-        {{ t("memories", "external transcoder configuration") }}
+        {{ t("memories", "External Link") }}
       </a>
 
       <template v-if="status">
@@ -206,6 +327,7 @@
       </template>
 
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding"
         :checked.sync="enableExternalTranscoder"
         @update:checked="update('enableExternalTranscoder')"
         type="switch"
@@ -214,6 +336,7 @@
       </NcCheckboxRadioSwitch>
 
       <NcTextField
+        :disabled="!enableTranscoding"
         :label="t('memories', 'Binary path (local only)')"
         :label-visible="true"
         :value="goVodPath"
@@ -221,6 +344,7 @@
       />
 
       <NcTextField
+        :disabled="!enableTranscoding"
         :label="t('memories', 'Bind address (local only)')"
         :label-visible="true"
         :value="goVodBind"
@@ -228,6 +352,7 @@
       />
 
       <NcTextField
+        :disabled="!enableTranscoding"
         :label="t('memories', 'Connection address (same as bind if local)')"
         :label-visible="true"
         :value="goVodConnect"
@@ -271,14 +396,14 @@
       {{
         t(
           "memories",
-          "For more details on driver installation, check the following link:"
+          "For more details on driver installation, check the documentation:"
         )
       }}
       <a
         target="_blank"
         href="https://github.com/pulsejet/memories/wiki/HW-Transcoding#va-api"
       >
-        VA-API configuration
+        {{ t("memories", "External Link") }}
       </a>
 
       <NcNoteCard :type="vaapiStatusType" v-if="status">
@@ -286,6 +411,7 @@
       </NcNoteCard>
 
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding"
         :checked.sync="enableVaapi"
         @update:checked="update('enableVaapi')"
         type="switch"
@@ -294,6 +420,7 @@
       </NcCheckboxRadioSwitch>
 
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding || !enableVaapi"
         :checked.sync="enableVaapiLowPower"
         @update:checked="update('enableVaapiLowPower')"
         type="switch"
@@ -325,6 +452,7 @@
       </NcNoteCard>
 
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding"
         :checked.sync="enableNvenc"
         @update:checked="update('enableNvenc')"
         type="switch"
@@ -332,6 +460,7 @@
         {{ t("memories", "Enable acceleration with NVENC") }}
       </NcCheckboxRadioSwitch>
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding || !enableNvenc"
         :checked.sync="enableNvencTemporalAQ"
         @update:checked="update('enableNvencTemporalAQ')"
         type="switch"
@@ -340,6 +469,7 @@
       </NcCheckboxRadioSwitch>
 
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding || !enableNvenc"
         :checked.sync="nvencScaler"
         value="npp"
         name="nvence_scaler_radio"
@@ -349,6 +479,7 @@
         >{{ t("memories", "NPP scaler") }}
       </NcCheckboxRadioSwitch>
       <NcCheckboxRadioSwitch
+        :disabled="!enableTranscoding || !enableNvenc"
         :checked.sync="nvencScaler"
         value="cuda"
         name="nvence_scaler_radio"
@@ -379,6 +510,8 @@ import NcButton from "@nextcloud/vue/dist/Components/NcButton";
 const settings = {
   exiftoolPath: "memories.exiftool",
   exiftoolPerl: "memories.exiftool_no_local",
+  indexingMode: "memories.index.mode",
+  indexingPath: "memories.index.path",
 
   gisType: "memories.gis_type",
 
@@ -405,6 +538,9 @@ const invertedBooleans = ["enableTranscoding"];
 type BinaryStatus = "ok" | "not_found" | "not_executable" | "test_ok" | string;
 
 type IStatus = {
+  bad_encryption: boolean;
+  indexed_count: number;
+  mimes: string[];
   gis_type: number;
   gis_count?: number;
   exiftool: BinaryStatus;
@@ -430,6 +566,8 @@ export default defineComponent({
 
     exiftoolPath: "",
     exiftoolPerl: false,
+    indexingMode: "0",
+    indexingPath: "",
 
     gisType: 0,
 
@@ -696,5 +834,11 @@ export default defineComponent({
   a {
     color: var(--color-primary-element);
   }
+
+  code {
+    padding-left: 10px;
+    -webkit-box-decoration-break: clone;
+    box-decoration-break: clone;
+  }
 }
 </style>