diff --git a/lib/Command/Index.php b/lib/Command/Index.php index b00eb64f..fb66fe50 100644 --- a/lib/Command/Index.php +++ b/lib/Command/Index.php @@ -105,6 +105,7 @@ class Index extends Command $this->input = $input; $this->output = $output; $this->opts = new IndexOpts($input); + $this->indexer->output = $output; try { // Use static exiftool process @@ -135,19 +136,21 @@ class Index extends Command */ protected function checkClear(): void { - if ($this->opts->clear) { - if ($this->input->isInteractive()) { - $this->output->write('Are you sure you want to clear the existing index? (y/N): '); - if ('y' !== trim(fgets(STDIN))) { - $this->output->writeln('Aborting'); - - exit; - } - } - - $this->timelineWrite->clear(); - $this->output->writeln('Cleared existing index'); + if (!$this->opts->clear) { + return; } + + if ($this->input->isInteractive()) { + $this->output->write('Are you sure you want to clear the existing index? (y/N): '); + if ('y' !== trim(fgets(STDIN))) { + $this->output->writeln('Aborting'); + + exit; + } + } + + $this->timelineWrite->clear(); + $this->output->writeln('Cleared existing index'); } /** @@ -155,11 +158,13 @@ class Index extends Command */ protected function checkForce(): void { - if ($this->opts->force) { - $this->output->writeln('Forcing refresh of existing index entries'); - - // TODO + if (!$this->opts->force) { + return; } + + $this->output->writeln('Forcing refresh of existing index entries'); + + $this->timelineWrite->orphanAll(); } /** @@ -167,23 +172,32 @@ class Index extends Command */ protected function runIndex(): void { - // Call indexing for specified or each user + $this->runForUsers(function (IUser $user) { + try { + $uid = $user->getUID(); + $this->output->writeln("Indexing user {$uid}"); + $this->indexer->indexUser($uid, $this->opts->folder); + } catch (\Exception $e) { + $this->output->writeln("{$e->getMessage()}"); + } + }); + } + + /** + * Run function for all users (or selected user if set). + * + * @param mixed $closure + */ + private function runForUsers($closure) + { if ($uid = $this->opts->user) { if ($user = $this->userManager->get($uid)) { - $this->indexer->indexUser($user->getUID(), $this->opts->folder); + $closure($user); } else { - throw new \Exception("User {$uid} not found"); + $this->output->writeln("User {$uid} not found"); } } else { - $this->userManager->callForSeenUsers(function (IUser $user) { - try { - $uid = $user->getUID(); - $this->output->writeln("Indexing user {$uid}"); - $this->indexer->indexUser($uid, $this->opts->folder); - } catch (\Exception $e) { - $this->output->writeln("{$e->getMessage()}"); - } - }); + $this->userManager->callForSeenUsers(fn (IUser $user) => $closure($user)); } } } diff --git a/lib/Command/MigrateGoogleTakeout.php b/lib/Command/MigrateGoogleTakeout.php index 88397e54..575620cd 100644 --- a/lib/Command/MigrateGoogleTakeout.php +++ b/lib/Command/MigrateGoogleTakeout.php @@ -63,7 +63,8 @@ class MigrateGoogleTakeout extends Command IUserManager $userManager, IConfig $config, IDBConnection $connection, - ITempManager $tempManager + ITempManager $tempManager, + TimelineWrite $timelineWrite ) { parent::__construct(); @@ -72,7 +73,7 @@ class MigrateGoogleTakeout extends Command $this->config = $config; $this->connection = $connection; $this->tempManager = $tempManager; - $this->timelineWrite = new TimelineWrite($connection); + $this->timelineWrite = $timelineWrite; } protected function configure(): void diff --git a/lib/Db/LivePhoto.php b/lib/Db/LivePhoto.php index 0bffa83b..4bb79099 100644 --- a/lib/Db/LivePhoto.php +++ b/lib/Db/LivePhoto.php @@ -103,13 +103,13 @@ class LivePhoto /** * Process video part of Live Photo. */ - public function processVideoPart(File $file, array $exif) + public function processVideoPart(File $file, array $exif): bool { $fileId = $file->getId(); $mtime = $file->getMTime(); $liveid = $exif['ContentIdentifier']; if (empty($liveid)) { - return; + return false; } $query = $this->connection->getQueryBuilder(); @@ -117,32 +117,30 @@ class LivePhoto ->from('memories_livephoto') ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) ; - $cursor = $query->executeQuery(); - $prevRow = $cursor->fetch(); - $cursor->closeCursor(); + $prevRow = $query->executeQuery()->fetch(); - if ($prevRow) { - // Update existing row - $query->update('memories_livephoto') - ->set('liveid', $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR)) - ->set('mtime', $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT)) - ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) - ; - $query->executeStatement(); - } else { - // Try to create new row - try { - $query->insert('memories_livephoto') - ->values([ - 'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR), - 'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT), - 'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT), - ]) + $params = [ + 'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR), + 'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT), + 'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT), + 'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL), + ]; + + try { + if ($prevRow) { + $query->update('memories_livephoto') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) ; - $query->executeStatement(); - } catch (\Exception $ex) { - error_log('Failed to create memories_livephoto record: '.$ex->getMessage()); + foreach ($params as $key => $value) { + $query->set($key, $value); + } + } else { + $query->insert('memories_livephoto')->values($params); } + + return $query->executeStatement() > 0; + } catch (\Exception $ex) { + throw new \Exception('Failed to create livephoto record: '.$ex->getMessage()); } } diff --git a/lib/Db/TimelineWrite.php b/lib/Db/TimelineWrite.php index bb0b6941..b1918545 100644 --- a/lib/Db/TimelineWrite.php +++ b/lib/Db/TimelineWrite.php @@ -9,7 +9,6 @@ use OCA\Memories\Service\Index; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\File; use OCP\IDBConnection; -use OCP\IPreview; require_once __DIR__.'/../ExifFields.php'; @@ -22,14 +21,14 @@ class TimelineWrite use TimelineWriteOrphans; use TimelineWritePlaces; protected IDBConnection $connection; - protected IPreview $preview; protected LivePhoto $livePhoto; - public function __construct(IDBConnection $connection) - { + public function __construct( + IDBConnection $connection, + LivePhoto $livePhoto + ) { $this->connection = $connection; - $this->preview = \OC::$server->get(IPreview::class); - $this->livePhoto = new LivePhoto($connection); + $this->livePhoto = $livePhoto; } /** @@ -53,8 +52,12 @@ class TimelineWrite // Get previous row $prevRow = $this->getCurrentRow($fileId); - // Skip if not forced and file has not changed - if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime)) { + // Skip if all of the following: + // - not forced + // - the record exists + // - the file has not changed + // - the record is not an orphan + if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime) && (!(bool) $prevRow['orphan'])) { return false; } @@ -127,6 +130,7 @@ class TimelineWrite 'lat' => $query->createNamedParameter($lat, IQueryBuilder::PARAM_STR), 'lon' => $query->createNamedParameter($lon, IQueryBuilder::PARAM_STR), 'mapcluster' => $query->createNamedParameter($mapCluster, IQueryBuilder::PARAM_INT), + 'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL), ]; // There is no easy way to UPSERT in standard SQL @@ -145,9 +149,7 @@ class TimelineWrite return $query->executeStatement() > 0; } catch (\Exception $ex) { - error_log('Failed to create memories record: '.$ex->getMessage()); - - return false; + throw new \Exception('Failed to create memories record: '.$ex->getMessage()); } } diff --git a/lib/Db/TimelineWriteOrphans.php b/lib/Db/TimelineWriteOrphans.php index 1d1980f0..a61209b0 100644 --- a/lib/Db/TimelineWriteOrphans.php +++ b/lib/Db/TimelineWriteOrphans.php @@ -32,12 +32,16 @@ trait TimelineWriteOrphans */ public function orphanAll(): int { - $query = $this->connection->getQueryBuilder(); - $query->update('memories') - ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)) - ; + $do = function (string $table) { + $query = $this->connection->getQueryBuilder(); + $query->update($table) + ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)) + ; - return $query->executeStatement(); + return $query->executeStatement(); + }; + + return $do('memories') + $do('memories_livephoto'); } /** diff --git a/lib/Exif.php b/lib/Exif.php index 3f26b37c..d5bf00c0 100644 --- a/lib/Exif.php +++ b/lib/Exif.php @@ -71,6 +71,7 @@ class Exif { $config = \OC::$server->get(IConfig::class); $paths = $config->getUserValue($uid, Application::APPNAME, 'timelinePath', null) ?? 'Photos/'; + return array_map(fn ($p) => self::sanitizePath(trim($p)), explode(';', $paths)); } diff --git a/lib/Listeners/PostDeleteListener.php b/lib/Listeners/PostDeleteListener.php index 065682da..d2bf80e3 100644 --- a/lib/Listeners/PostDeleteListener.php +++ b/lib/Listeners/PostDeleteListener.php @@ -26,15 +26,14 @@ use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; use OCP\Files\Events\Node\NodeDeletedEvent; use OCP\Files\Folder; -use OCP\IDBConnection; class PostDeleteListener implements IEventListener { private TimelineWrite $util; - public function __construct(IDBConnection $connection) + public function __construct(TimelineWrite $util) { - $this->util = new TimelineWrite($connection); + $this->util = $util; } public function handle(Event $event): void diff --git a/lib/Listeners/PostWriteListener.php b/lib/Listeners/PostWriteListener.php index 3ff0a566..3a9b2f53 100644 --- a/lib/Listeners/PostWriteListener.php +++ b/lib/Listeners/PostWriteListener.php @@ -22,20 +22,19 @@ declare(strict_types=1); namespace OCA\Memories\Listeners; use OCA\Memories\Db\TimelineWrite; +use OCA\Memories\Service\Index; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; use OCP\Files\Events\Node\NodeTouchedEvent; use OCP\Files\Events\Node\NodeWrittenEvent; -use OCP\Files\Folder; -use OCP\IDBConnection; class PostWriteListener implements IEventListener { private TimelineWrite $timelineWrite; - public function __construct(IDBConnection $connection) + public function __construct(TimelineWrite $timelineWrite) { - $this->timelineWrite = new TimelineWrite($connection); + $this->timelineWrite = $timelineWrite; } public function handle(Event $event): void @@ -46,12 +45,9 @@ class PostWriteListener implements IEventListener } $node = $event->getNode(); - if ($node instanceof Folder) { - return; - } // Check the mime type first - if (!$this->timelineWrite->getFileType($node)) { + if (!Index::isSupported($node)) { return; } diff --git a/lib/Migration/Version500000Date20230414042534.php b/lib/Migration/Version500000Date20230414042534.php new file mode 100644 index 00000000..e51ec3d4 --- /dev/null +++ b/lib/Migration/Version500000Date20230414042534.php @@ -0,0 +1,69 @@ + + * @author Your name + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCA\Memories\Migration; + +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +/** + * Auto-generated migration step: Please modify to your needs! + */ +class Version500000Date20230414042534 extends SimpleMigrationStep +{ + /** + * @param \Closure(): ISchemaWrapper $schemaClosure + */ + public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void + { + } + + /** + * @param \Closure(): ISchemaWrapper $schemaClosure + */ + public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper + { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + $table = $schema->getTable('memories_livephoto'); + + if (!$table->hasColumn('orphan')) { + $table->addColumn('orphan', Types::BOOLEAN, [ + 'notnull' => false, + 'default' => false, + ]); + } + + return $schema; + } + + /** + * @param \Closure(): ISchemaWrapper $schemaClosure + */ + public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void + { + } +} diff --git a/lib/Service/Index.php b/lib/Service/Index.php index e9fb7790..6d102901 100644 --- a/lib/Service/Index.php +++ b/lib/Service/Index.php @@ -30,13 +30,17 @@ use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\File; use OCP\Files\Folder; use OCP\Files\IRootFolder; +use OCP\Files\Node; use OCP\IDBConnection; use OCP\IPreview; use OCP\ITempManager; use Psr\Log\LoggerInterface; +use Symfony\Component\Console\Output\OutputInterface; class Index { + public ?OutputInterface $output; + protected IRootFolder $rootFolder; protected TimelineWrite $timelineWrite; protected IDBConnection $db; @@ -74,12 +78,14 @@ class Index $mode = Util::getSystemConfig('memories.index.mode'); if (null !== $folder) { $paths = [$folder]; - } elseif ('1' === $mode || '0' === $mode) { // everything (or nothing) + } elseif ('1' === $mode || '0' === $mode) { // everything (or nothing) $paths = ['/']; } elseif ('2' === $mode) { // timeline $paths = \OCA\Memories\Exif::getTimelinePaths($uid); } elseif ('3' === $mode) { // custom $paths = [Util::getSystemConfig('memories.index.path')]; + } else { + throw new \Exception('Invalid index mode'); } // If a folder is specified, traverse only that folder @@ -90,11 +96,8 @@ class Index throw new \Exception('Not a folder'); } } catch (\Exception $e) { - if (\OC::$CLI && null !== $folder) { // admin asked for this explicitly - throw new \Exception("The specified folder {$path} does not exist for ${uid}"); - } + $this->error("The specified folder {$path} does not exist for {$uid}"); - $this->logger->warning("The specified folder {$path} does not exist for ${uid}"); continue; } @@ -138,24 +141,18 @@ class Index ->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) ; - // TODO: check if forcing a refresh is needed - // Check in memories table - $query->leftJoin('f', 'memories', 'm', $query->expr()->andX( - $query->expr()->eq('f.fileid', 'm.fileid'), - $query->expr()->eq('f.mtime', 'm.mtime') - )); + // Filter out files that are already indexed + $addFilter = function (string $table, string $alias) use (&$query) { + $query->leftJoin('f', $table, $alias, $query->expr()->andX( + $query->expr()->eq('f.fileid', "$alias.fileid"), + $query->expr()->eq('f.mtime', "$alias.mtime"), + $query->expr()->eq("$alias.orphan", $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL)) + )); - // Check in livephoto table - $query->leftJoin('f', 'memories_livephoto', 'mlp', $query->expr()->andX( - $query->expr()->eq('f.fileid', 'mlp.fileid'), - $query->expr()->eq('f.mtime', 'mlp.mtime') - )); - - // Exclude files that are already indexed - $query->andWhere($query->expr()->andX( - $query->expr()->isNull('m.mtime'), - $query->expr()->isNull('mlp.mtime'), - )); + $query->andWhere($query->expr()->isNull("$alias.fileid")); + }; + $addFilter('memories', 'm'); + $addFilter('memories_livephoto', 'lp'); // Get file IDs to actually index $fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); @@ -188,10 +185,7 @@ class Index try { $this->timelineWrite->processFile($file); } catch (\Exception $e) { - $this->logger->error('Failed to index file {file}: {error}', [ - 'file' => $file->getPath(), - 'error' => $e->getMessage(), - ]); + $this->error("Failed to index file {$file->getPath()}: {$e->getMessage()}"); } $this->tempManager->clean(); @@ -245,7 +239,7 @@ class Index /** * Check if a file is supported. */ - public static function isSupported(File $file): bool + public static function isSupported(Node $file): bool { return \in_array($file->getMimeType(), self::getMimeList(), true); } @@ -257,4 +251,22 @@ class Index { return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true); } + + /** Log to console if CLI or logger */ + private function error(string $message) + { + $this->logger->error($message); + + if ($this->output) { + $this->output->writeln("{$message}"); + } + } + + /** Log to console if CLI */ + private function log(string $message) + { + if ($this->output) { + $this->output->writeln($message); + } + } }