diff --git a/lib/Command/Index.php b/lib/Command/Index.php
index b00eb64f..fb66fe50 100644
--- a/lib/Command/Index.php
+++ b/lib/Command/Index.php
@@ -105,6 +105,7 @@ class Index extends Command
$this->input = $input;
$this->output = $output;
$this->opts = new IndexOpts($input);
+ $this->indexer->output = $output;
try {
// Use static exiftool process
@@ -135,19 +136,21 @@ class Index extends Command
*/
protected function checkClear(): void
{
- if ($this->opts->clear) {
- if ($this->input->isInteractive()) {
- $this->output->write('Are you sure you want to clear the existing index? (y/N): ');
- if ('y' !== trim(fgets(STDIN))) {
- $this->output->writeln('Aborting');
-
- exit;
- }
- }
-
- $this->timelineWrite->clear();
- $this->output->writeln('Cleared existing index');
+ if (!$this->opts->clear) {
+ return;
}
+
+ if ($this->input->isInteractive()) {
+ $this->output->write('Are you sure you want to clear the existing index? (y/N): ');
+ if ('y' !== trim(fgets(STDIN))) {
+ $this->output->writeln('Aborting');
+
+ exit;
+ }
+ }
+
+ $this->timelineWrite->clear();
+ $this->output->writeln('Cleared existing index');
}
/**
@@ -155,11 +158,13 @@ class Index extends Command
*/
protected function checkForce(): void
{
- if ($this->opts->force) {
- $this->output->writeln('Forcing refresh of existing index entries');
-
- // TODO
+ if (!$this->opts->force) {
+ return;
}
+
+ $this->output->writeln('Forcing refresh of existing index entries');
+
+ $this->timelineWrite->orphanAll();
}
/**
@@ -167,23 +172,32 @@ class Index extends Command
*/
protected function runIndex(): void
{
- // Call indexing for specified or each user
+ $this->runForUsers(function (IUser $user) {
+ try {
+ $uid = $user->getUID();
+ $this->output->writeln("Indexing user {$uid}");
+ $this->indexer->indexUser($uid, $this->opts->folder);
+ } catch (\Exception $e) {
+ $this->output->writeln("{$e->getMessage()}");
+ }
+ });
+ }
+
+ /**
+ * Run function for all users (or selected user if set).
+ *
+ * @param mixed $closure
+ */
+ private function runForUsers($closure)
+ {
if ($uid = $this->opts->user) {
if ($user = $this->userManager->get($uid)) {
- $this->indexer->indexUser($user->getUID(), $this->opts->folder);
+ $closure($user);
} else {
- throw new \Exception("User {$uid} not found");
+ $this->output->writeln("User {$uid} not found");
}
} else {
- $this->userManager->callForSeenUsers(function (IUser $user) {
- try {
- $uid = $user->getUID();
- $this->output->writeln("Indexing user {$uid}");
- $this->indexer->indexUser($uid, $this->opts->folder);
- } catch (\Exception $e) {
- $this->output->writeln("{$e->getMessage()}");
- }
- });
+ $this->userManager->callForSeenUsers(fn (IUser $user) => $closure($user));
}
}
}
diff --git a/lib/Command/MigrateGoogleTakeout.php b/lib/Command/MigrateGoogleTakeout.php
index 88397e54..575620cd 100644
--- a/lib/Command/MigrateGoogleTakeout.php
+++ b/lib/Command/MigrateGoogleTakeout.php
@@ -63,7 +63,8 @@ class MigrateGoogleTakeout extends Command
IUserManager $userManager,
IConfig $config,
IDBConnection $connection,
- ITempManager $tempManager
+ ITempManager $tempManager,
+ TimelineWrite $timelineWrite
) {
parent::__construct();
@@ -72,7 +73,7 @@ class MigrateGoogleTakeout extends Command
$this->config = $config;
$this->connection = $connection;
$this->tempManager = $tempManager;
- $this->timelineWrite = new TimelineWrite($connection);
+ $this->timelineWrite = $timelineWrite;
}
protected function configure(): void
diff --git a/lib/Db/LivePhoto.php b/lib/Db/LivePhoto.php
index 0bffa83b..4bb79099 100644
--- a/lib/Db/LivePhoto.php
+++ b/lib/Db/LivePhoto.php
@@ -103,13 +103,13 @@ class LivePhoto
/**
* Process video part of Live Photo.
*/
- public function processVideoPart(File $file, array $exif)
+ public function processVideoPart(File $file, array $exif): bool
{
$fileId = $file->getId();
$mtime = $file->getMTime();
$liveid = $exif['ContentIdentifier'];
if (empty($liveid)) {
- return;
+ return false;
}
$query = $this->connection->getQueryBuilder();
@@ -117,32 +117,30 @@ class LivePhoto
->from('memories_livephoto')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
- $cursor = $query->executeQuery();
- $prevRow = $cursor->fetch();
- $cursor->closeCursor();
+ $prevRow = $query->executeQuery()->fetch();
- if ($prevRow) {
- // Update existing row
- $query->update('memories_livephoto')
- ->set('liveid', $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR))
- ->set('mtime', $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT))
- ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
- ;
- $query->executeStatement();
- } else {
- // Try to create new row
- try {
- $query->insert('memories_livephoto')
- ->values([
- 'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
- 'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
- 'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
- ])
+ $params = [
+ 'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
+ 'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
+ 'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
+ 'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL),
+ ];
+
+ try {
+ if ($prevRow) {
+ $query->update('memories_livephoto')
+ ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
- $query->executeStatement();
- } catch (\Exception $ex) {
- error_log('Failed to create memories_livephoto record: '.$ex->getMessage());
+ foreach ($params as $key => $value) {
+ $query->set($key, $value);
+ }
+ } else {
+ $query->insert('memories_livephoto')->values($params);
}
+
+ return $query->executeStatement() > 0;
+ } catch (\Exception $ex) {
+ throw new \Exception('Failed to create livephoto record: '.$ex->getMessage());
}
}
diff --git a/lib/Db/TimelineWrite.php b/lib/Db/TimelineWrite.php
index bb0b6941..b1918545 100644
--- a/lib/Db/TimelineWrite.php
+++ b/lib/Db/TimelineWrite.php
@@ -9,7 +9,6 @@ use OCA\Memories\Service\Index;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\File;
use OCP\IDBConnection;
-use OCP\IPreview;
require_once __DIR__.'/../ExifFields.php';
@@ -22,14 +21,14 @@ class TimelineWrite
use TimelineWriteOrphans;
use TimelineWritePlaces;
protected IDBConnection $connection;
- protected IPreview $preview;
protected LivePhoto $livePhoto;
- public function __construct(IDBConnection $connection)
- {
+ public function __construct(
+ IDBConnection $connection,
+ LivePhoto $livePhoto
+ ) {
$this->connection = $connection;
- $this->preview = \OC::$server->get(IPreview::class);
- $this->livePhoto = new LivePhoto($connection);
+ $this->livePhoto = $livePhoto;
}
/**
@@ -53,8 +52,12 @@ class TimelineWrite
// Get previous row
$prevRow = $this->getCurrentRow($fileId);
- // Skip if not forced and file has not changed
- if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime)) {
+ // Skip if all of the following:
+ // - not forced
+ // - the record exists
+ // - the file has not changed
+ // - the record is not an orphan
+ if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime) && (!(bool) $prevRow['orphan'])) {
return false;
}
@@ -127,6 +130,7 @@ class TimelineWrite
'lat' => $query->createNamedParameter($lat, IQueryBuilder::PARAM_STR),
'lon' => $query->createNamedParameter($lon, IQueryBuilder::PARAM_STR),
'mapcluster' => $query->createNamedParameter($mapCluster, IQueryBuilder::PARAM_INT),
+ 'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL),
];
// There is no easy way to UPSERT in standard SQL
@@ -145,9 +149,7 @@ class TimelineWrite
return $query->executeStatement() > 0;
} catch (\Exception $ex) {
- error_log('Failed to create memories record: '.$ex->getMessage());
-
- return false;
+ throw new \Exception('Failed to create memories record: '.$ex->getMessage());
}
}
diff --git a/lib/Db/TimelineWriteOrphans.php b/lib/Db/TimelineWriteOrphans.php
index 1d1980f0..a61209b0 100644
--- a/lib/Db/TimelineWriteOrphans.php
+++ b/lib/Db/TimelineWriteOrphans.php
@@ -32,12 +32,16 @@ trait TimelineWriteOrphans
*/
public function orphanAll(): int
{
- $query = $this->connection->getQueryBuilder();
- $query->update('memories')
- ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
- ;
+ $do = function (string $table) {
+ $query = $this->connection->getQueryBuilder();
+ $query->update($table)
+ ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
+ ;
- return $query->executeStatement();
+ return $query->executeStatement();
+ };
+
+ return $do('memories') + $do('memories_livephoto');
}
/**
diff --git a/lib/Exif.php b/lib/Exif.php
index 3f26b37c..d5bf00c0 100644
--- a/lib/Exif.php
+++ b/lib/Exif.php
@@ -71,6 +71,7 @@ class Exif
{
$config = \OC::$server->get(IConfig::class);
$paths = $config->getUserValue($uid, Application::APPNAME, 'timelinePath', null) ?? 'Photos/';
+
return array_map(fn ($p) => self::sanitizePath(trim($p)), explode(';', $paths));
}
diff --git a/lib/Listeners/PostDeleteListener.php b/lib/Listeners/PostDeleteListener.php
index 065682da..d2bf80e3 100644
--- a/lib/Listeners/PostDeleteListener.php
+++ b/lib/Listeners/PostDeleteListener.php
@@ -26,15 +26,14 @@ use OCP\EventDispatcher\Event;
use OCP\EventDispatcher\IEventListener;
use OCP\Files\Events\Node\NodeDeletedEvent;
use OCP\Files\Folder;
-use OCP\IDBConnection;
class PostDeleteListener implements IEventListener
{
private TimelineWrite $util;
- public function __construct(IDBConnection $connection)
+ public function __construct(TimelineWrite $util)
{
- $this->util = new TimelineWrite($connection);
+ $this->util = $util;
}
public function handle(Event $event): void
diff --git a/lib/Listeners/PostWriteListener.php b/lib/Listeners/PostWriteListener.php
index 3ff0a566..3a9b2f53 100644
--- a/lib/Listeners/PostWriteListener.php
+++ b/lib/Listeners/PostWriteListener.php
@@ -22,20 +22,19 @@ declare(strict_types=1);
namespace OCA\Memories\Listeners;
use OCA\Memories\Db\TimelineWrite;
+use OCA\Memories\Service\Index;
use OCP\EventDispatcher\Event;
use OCP\EventDispatcher\IEventListener;
use OCP\Files\Events\Node\NodeTouchedEvent;
use OCP\Files\Events\Node\NodeWrittenEvent;
-use OCP\Files\Folder;
-use OCP\IDBConnection;
class PostWriteListener implements IEventListener
{
private TimelineWrite $timelineWrite;
- public function __construct(IDBConnection $connection)
+ public function __construct(TimelineWrite $timelineWrite)
{
- $this->timelineWrite = new TimelineWrite($connection);
+ $this->timelineWrite = $timelineWrite;
}
public function handle(Event $event): void
@@ -46,12 +45,9 @@ class PostWriteListener implements IEventListener
}
$node = $event->getNode();
- if ($node instanceof Folder) {
- return;
- }
// Check the mime type first
- if (!$this->timelineWrite->getFileType($node)) {
+ if (!Index::isSupported($node)) {
return;
}
diff --git a/lib/Migration/Version500000Date20230414042534.php b/lib/Migration/Version500000Date20230414042534.php
new file mode 100644
index 00000000..e51ec3d4
--- /dev/null
+++ b/lib/Migration/Version500000Date20230414042534.php
@@ -0,0 +1,69 @@
+
+ * @author Your name
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+namespace OCA\Memories\Migration;
+
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version500000Date20230414042534 extends SimpleMigrationStep
+{
+ /**
+ * @param \Closure(): ISchemaWrapper $schemaClosure
+ */
+ public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+ {
+ }
+
+ /**
+ * @param \Closure(): ISchemaWrapper $schemaClosure
+ */
+ public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
+ {
+ /** @var ISchemaWrapper $schema */
+ $schema = $schemaClosure();
+
+ $table = $schema->getTable('memories_livephoto');
+
+ if (!$table->hasColumn('orphan')) {
+ $table->addColumn('orphan', Types::BOOLEAN, [
+ 'notnull' => false,
+ 'default' => false,
+ ]);
+ }
+
+ return $schema;
+ }
+
+ /**
+ * @param \Closure(): ISchemaWrapper $schemaClosure
+ */
+ public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+ {
+ }
+}
diff --git a/lib/Service/Index.php b/lib/Service/Index.php
index e9fb7790..6d102901 100644
--- a/lib/Service/Index.php
+++ b/lib/Service/Index.php
@@ -30,13 +30,17 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
+use OCP\Files\Node;
use OCP\IDBConnection;
use OCP\IPreview;
use OCP\ITempManager;
use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Output\OutputInterface;
class Index
{
+ public ?OutputInterface $output;
+
protected IRootFolder $rootFolder;
protected TimelineWrite $timelineWrite;
protected IDBConnection $db;
@@ -74,12 +78,14 @@ class Index
$mode = Util::getSystemConfig('memories.index.mode');
if (null !== $folder) {
$paths = [$folder];
- } elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
+ } elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
$paths = ['/'];
} elseif ('2' === $mode) { // timeline
$paths = \OCA\Memories\Exif::getTimelinePaths($uid);
} elseif ('3' === $mode) { // custom
$paths = [Util::getSystemConfig('memories.index.path')];
+ } else {
+ throw new \Exception('Invalid index mode');
}
// If a folder is specified, traverse only that folder
@@ -90,11 +96,8 @@ class Index
throw new \Exception('Not a folder');
}
} catch (\Exception $e) {
- if (\OC::$CLI && null !== $folder) { // admin asked for this explicitly
- throw new \Exception("The specified folder {$path} does not exist for ${uid}");
- }
+ $this->error("The specified folder {$path} does not exist for {$uid}");
- $this->logger->warning("The specified folder {$path} does not exist for ${uid}");
continue;
}
@@ -138,24 +141,18 @@ class Index
->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
;
- // TODO: check if forcing a refresh is needed
- // Check in memories table
- $query->leftJoin('f', 'memories', 'm', $query->expr()->andX(
- $query->expr()->eq('f.fileid', 'm.fileid'),
- $query->expr()->eq('f.mtime', 'm.mtime')
- ));
+ // Filter out files that are already indexed
+ $addFilter = function (string $table, string $alias) use (&$query) {
+ $query->leftJoin('f', $table, $alias, $query->expr()->andX(
+ $query->expr()->eq('f.fileid', "$alias.fileid"),
+ $query->expr()->eq('f.mtime', "$alias.mtime"),
+ $query->expr()->eq("$alias.orphan", $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL))
+ ));
- // Check in livephoto table
- $query->leftJoin('f', 'memories_livephoto', 'mlp', $query->expr()->andX(
- $query->expr()->eq('f.fileid', 'mlp.fileid'),
- $query->expr()->eq('f.mtime', 'mlp.mtime')
- ));
-
- // Exclude files that are already indexed
- $query->andWhere($query->expr()->andX(
- $query->expr()->isNull('m.mtime'),
- $query->expr()->isNull('mlp.mtime'),
- ));
+ $query->andWhere($query->expr()->isNull("$alias.fileid"));
+ };
+ $addFilter('memories', 'm');
+ $addFilter('memories_livephoto', 'lp');
// Get file IDs to actually index
$fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
@@ -188,10 +185,7 @@ class Index
try {
$this->timelineWrite->processFile($file);
} catch (\Exception $e) {
- $this->logger->error('Failed to index file {file}: {error}', [
- 'file' => $file->getPath(),
- 'error' => $e->getMessage(),
- ]);
+ $this->error("Failed to index file {$file->getPath()}: {$e->getMessage()}");
}
$this->tempManager->clean();
@@ -245,7 +239,7 @@ class Index
/**
* Check if a file is supported.
*/
- public static function isSupported(File $file): bool
+ public static function isSupported(Node $file): bool
{
return \in_array($file->getMimeType(), self::getMimeList(), true);
}
@@ -257,4 +251,22 @@ class Index
{
return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
}
+
+ /** Log to console if CLI or logger */
+ private function error(string $message)
+ {
+ $this->logger->error($message);
+
+ if ($this->output) {
+ $this->output->writeln("{$message}");
+ }
+ }
+
+ /** Log to console if CLI */
+ private function log(string $message)
+ {
+ if ($this->output) {
+ $this->output->writeln($message);
+ }
+ }
}