index: add background job (close #110)

Signed-off-by: Varun Patil <varunpatil@ucla.edu>

CHANGELOG.md

@@ -15,6 +15,9 @@ Note: this is a major release and may introduce breaking changes to your workflo
   Make sure your temp directory is writable by the web server.
 - **Breaking**: The `--cleanup` flag to `memories:index` has been removed and is no longer necessary.  
   Folders having a `.nomedia` file will automatically be excluded from the timeline.
+- **Feature**: Indexing will now build and check indices automatically in the backgroud.  
+  Make sure Nextcloud cron is configured correctly. You can disable automatic indexing in the admin panel.
+  Note that files are still indexed immediately on upload.
 - **Feature**: You can now choose which folders to index by default.  
   This can be configured from the admin panel. The available options are:
   - All media files (excluding folders with `.nomedia` files, default and recommended)

appinfo/info.xml

@@ -64,4 +64,7 @@ Memories is a *batteries-included* photo management solution for Nextcloud with
             <step>OCA\Memories\Migration\Repair</step>
         </install>
     </repair-steps>
+    <background-jobs>
+        <job>OCA\Memories\Cron\IndexJob</job>
+    </background-jobs>
 </info>

lib/Cron/IndexJob.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace OCA\Memories\Cron;
+
+use OCA\Memories\Service;
+use OCA\Memories\Util;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
+use OCP\IUserManager;
+use Psr\Log\LoggerInterface;
+
+const MAX_RUN_TIME = 10; // seconds
+const INTERVAL = 600; // seconds (don't set this too low)
+
+class IndexJob extends TimedJob
+{
+    private Service\Index $service;
+    private IUserManager $userManager;
+    private LoggerInterface $logger;
+
+    public function __construct(
+        ITimeFactory $time,
+        Service\Index $service,
+        IUserManager $userManager,
+        LoggerInterface $logger
+    ) {
+        parent::__construct($time);
+        $this->service = $service;
+        $this->userManager = $userManager;
+        $this->logger = $logger;
+
+        $this->setInterval(INTERVAL);
+    }
+
+    protected function run($arguments)
+    {
+        // Check if indexing is enabled
+        if ('0' === Util::getSystemConfig('memories.index.mode')) {
+            return;
+        }
+
+        // Run for a maximum of 5 minutes
+        $startTime = microtime(true);
+        $this->service->continueCheck = function () use ($startTime) {
+            return (microtime(true) - $startTime) < MAX_RUN_TIME;
+        };
+
+        // Index with static exiftool process
+        // This is sub-optimal: the process may not be required at all.
+        try {
+            \OCA\Memories\Exif::ensureStaticExiftoolProc();
+            $this->indexAllUsers();
+        } catch (Service\ProcessClosedException $e) {
+            $this->logger->warning('Memories: Indexing process closed before completion, will continue on next run.');
+        } finally {
+            \OCA\Memories\Exif::closeStaticExiftoolProc();
+        }
+    }
+
+    /**
+     * Index all users.
+     *
+     * @throws Service\ProcessClosedException if the process was closed before completion
+     */
+    private function indexAllUsers(): void
+    {
+        $this->userManager->callForSeenUsers(function ($user) {
+            try {
+                $this->service->indexUser($user->getUID());
+            } catch (Service\ProcessClosedException $e) {
+                throw $e;
+            } catch (\Exception $e) {
+                $this->logger->error('Indexing failed for user '.$user->getUID().': '.$e->getMessage());
+            } catch (\Throwable $e) {
+                $this->logger->error('[BUG] uncaught exception in memories: '.$e->getMessage());
+            }
+        });
+    }
+}

lib/Service/Index.php

@@ -40,8 +40,14 @@ use Symfony\Component\Console\Output\OutputInterface;
 
 class Index
 {
-    public ?OutputInterface $output;
-    public ?ConsoleSectionOutput $section;
+    public ?OutputInterface $output = null;
+    public ?ConsoleSectionOutput $section = null;
+
+    /**
+     * Callback to check if the process should continue.
+     * This is called before every file is indexed.
+     */
+    public ?\Closure $continueCheck = null;
 
     protected IRootFolder $rootFolder;
     protected TimelineWrite $timelineWrite;
@@ -162,6 +168,7 @@ class Index
 
             // Index files
             foreach ($fileIds as $fileId) {
+                $this->ensureContinueOk();
                 $this->indexFile($chunk[$fileId]);
             }
         }
@@ -169,8 +176,12 @@ class Index
         // All folders
         $folders = array_filter($nodes, fn ($n) => $n instanceof Folder);
         foreach ($folders as $folder) {
+            $this->ensureContinueOk();
+
             try {
                 $this->indexFolder($folder);
+            } catch (ProcessClosedException $e) {
+                throw $e;
             } catch (\Exception $e) {
                 $this->logger->error('Failed to index folder {folder}: {error}', [
                     'folder' => $folder->getPath(),
@@ -280,4 +291,14 @@ class Index
             $this->section->write($message);
         }
     }
+
+    /**
+     * Ensure that the process should go on.
+     */
+    private function ensureContinueOk(): void
+    {
+        if (null !== $this->continueCheck && !($this->continueCheck)()) {
+            throw new ProcessClosedException();
+        }
+    }
 }

lib/Service/ProcessClosedException.php

@@ -0,0 +1,9 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\Memories\Service;
+
+class ProcessClosedException extends \Exception
+{
+}