RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Restore refresh command 

Signed-off-by: Varun Patil <varunpatil@ucla.edu>
pull/579/head
Varun Patil 2023-04-13 21:59:01 -07:00
parent 641574ccd2
commit a611169e7d
10 changed files with 205 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
lib/Command/Index.php
View File

@ -105,6 +105,7 @@ class Index extends Command
$this->input = $input;
$this->output = $output;
$this->opts = new IndexOpts($input);
$this->indexer->output = $output;
try {
// Use static exiftool process
@ -135,19 +136,21 @@ class Index extends Command
*/
protected function checkClear(): void
{
if ($this->opts->clear) {
if ($this->input->isInteractive()) {
$this->output->write('Are you sure you want to clear the existing index? (y/N): ');
if ('y' !== trim(fgets(STDIN))) {
$this->output->writeln('Aborting');
exit;
}
}
$this->timelineWrite->clear();
$this->output->writeln('Cleared existing index');
if (!$this->opts->clear) {
return;
}
if ($this->input->isInteractive()) {
$this->output->write('Are you sure you want to clear the existing index? (y/N): ');
if ('y' !== trim(fgets(STDIN))) {
$this->output->writeln('Aborting');
exit;
}
}
$this->timelineWrite->clear();
$this->output->writeln('Cleared existing index');
}
/**
@ -155,11 +158,13 @@ class Index extends Command
*/
protected function checkForce(): void
{
if ($this->opts->force) {
$this->output->writeln('Forcing refresh of existing index entries');
// TODO
if (!$this->opts->force) {
return;
}
$this->output->writeln('Forcing refresh of existing index entries');
$this->timelineWrite->orphanAll();
}
/**
@ -167,23 +172,32 @@ class Index extends Command
*/
protected function runIndex(): void
{
// Call indexing for specified or each user
$this->runForUsers(function (IUser $user) {
try {
$uid = $user->getUID();
$this->output->writeln("Indexing user {$uid}");
$this->indexer->indexUser($uid, $this->opts->folder);
} catch (\Exception $e) {
$this->output->writeln("<error>{$e->getMessage()}</error>");
}
});
}
/**
* Run function for all users (or selected user if set).
*
* @param mixed $closure
*/
private function runForUsers($closure)
{
if ($uid = $this->opts->user) {
if ($user = $this->userManager->get($uid)) {
$this->indexer->indexUser($user->getUID(), $this->opts->folder);
$closure($user);
} else {
throw new \Exception("User {$uid} not found");
$this->output->writeln("<error>User {$uid} not found</error>");
}
} else {
$this->userManager->callForSeenUsers(function (IUser $user) {
try {
$uid = $user->getUID();
$this->output->writeln("Indexing user {$uid}");
$this->indexer->indexUser($uid, $this->opts->folder);
} catch (\Exception $e) {
$this->output->writeln("<error>{$e->getMessage()}</error>");
}
});
$this->userManager->callForSeenUsers(fn (IUser $user) => $closure($user));
}
}
}

5
lib/Command/MigrateGoogleTakeout.php
View File

@ -63,7 +63,8 @@ class MigrateGoogleTakeout extends Command
IUserManager $userManager,
IConfig $config,
IDBConnection $connection,
ITempManager $tempManager
ITempManager $tempManager,
TimelineWrite $timelineWrite
) {
parent::__construct();
@ -72,7 +73,7 @@ class MigrateGoogleTakeout extends Command
$this->config = $config;
$this->connection = $connection;
$this->tempManager = $tempManager;
$this->timelineWrite = new TimelineWrite($connection);
$this->timelineWrite = $timelineWrite;
}
protected function configure(): void

48
lib/Db/LivePhoto.php
View File

@ -103,13 +103,13 @@ class LivePhoto
/**
* Process video part of Live Photo.
*/
public function processVideoPart(File $file, array $exif)
public function processVideoPart(File $file, array $exif): bool
{
$fileId = $file->getId();
$mtime = $file->getMTime();
$liveid = $exif['ContentIdentifier'];
if (empty($liveid)) {
return;
return false;
}
$query = $this->connection->getQueryBuilder();
@ -117,32 +117,30 @@ class LivePhoto
->from('memories_livephoto')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
$cursor = $query->executeQuery();
$prevRow = $cursor->fetch();
$cursor->closeCursor();
$prevRow = $query->executeQuery()->fetch();
if ($prevRow) {
// Update existing row
$query->update('memories_livephoto')
->set('liveid', $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR))
->set('mtime', $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT))
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
$query->executeStatement();
} else {
// Try to create new row
try {
$query->insert('memories_livephoto')
->values([
'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
])
$params = [
'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL),
];
try {
if ($prevRow) {
$query->update('memories_livephoto')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
$query->executeStatement();
} catch (\Exception $ex) {
error_log('Failed to create memories_livephoto record: '.$ex->getMessage());
foreach ($params as $key => $value) {
$query->set($key, $value);
}
} else {
$query->insert('memories_livephoto')->values($params);
}
return $query->executeStatement() > 0;
} catch (\Exception $ex) {
throw new \Exception('Failed to create livephoto record: '.$ex->getMessage());
}
}

24
lib/Db/TimelineWrite.php
View File

@ -9,7 +9,6 @@ use OCA\Memories\Service\Index;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\File;
use OCP\IDBConnection;
use OCP\IPreview;
require_once __DIR__.'/../ExifFields.php';
@ -22,14 +21,14 @@ class TimelineWrite
use TimelineWriteOrphans;
use TimelineWritePlaces;
protected IDBConnection $connection;
protected IPreview $preview;
protected LivePhoto $livePhoto;
public function __construct(IDBConnection $connection)
{
public function __construct(
IDBConnection $connection,
LivePhoto $livePhoto
) {
$this->connection = $connection;
$this->preview = \OC::$server->get(IPreview::class);
$this->livePhoto = new LivePhoto($connection);
$this->livePhoto = $livePhoto;
}
/**
@ -53,8 +52,12 @@ class TimelineWrite
// Get previous row
$prevRow = $this->getCurrentRow($fileId);
// Skip if not forced and file has not changed
if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime)) {
// Skip if all of the following:
// - not forced
// - the record exists
// - the file has not changed
// - the record is not an orphan
if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime) && (!(bool) $prevRow['orphan'])) {
return false;
}
@ -127,6 +130,7 @@ class TimelineWrite
'lat' => $query->createNamedParameter($lat, IQueryBuilder::PARAM_STR),
'lon' => $query->createNamedParameter($lon, IQueryBuilder::PARAM_STR),
'mapcluster' => $query->createNamedParameter($mapCluster, IQueryBuilder::PARAM_INT),
'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL),
];
// There is no easy way to UPSERT in standard SQL
@ -145,9 +149,7 @@ class TimelineWrite
return $query->executeStatement() > 0;
} catch (\Exception $ex) {
error_log('Failed to create memories record: '.$ex->getMessage());
return false;
throw new \Exception('Failed to create memories record: '.$ex->getMessage());
}
}

14
lib/Db/TimelineWriteOrphans.php
View File

@ -32,12 +32,16 @@ trait TimelineWriteOrphans
*/
public function orphanAll(): int
{
$query = $this->connection->getQueryBuilder();
$query->update('memories')
->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
;
$do = function (string $table) {
$query = $this->connection->getQueryBuilder();
$query->update($table)
->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
;
return $query->executeStatement();
return $query->executeStatement();
};
return $do('memories') + $do('memories_livephoto');
}
/**

1
lib/Exif.php
View File

@ -71,6 +71,7 @@ class Exif
{
$config = \OC::$server->get(IConfig::class);
$paths = $config->getUserValue($uid, Application::APPNAME, 'timelinePath', null) ?? 'Photos/';
return array_map(fn ($p) => self::sanitizePath(trim($p)), explode(';', $paths));
}

5
lib/Listeners/PostDeleteListener.php
View File

@ -26,15 +26,14 @@ use OCP\EventDispatcher\Event;
use OCP\EventDispatcher\IEventListener;
use OCP\Files\Events\Node\NodeDeletedEvent;
use OCP\Files\Folder;
use OCP\IDBConnection;
class PostDeleteListener implements IEventListener
{
private TimelineWrite $util;
public function __construct(IDBConnection $connection)
public function __construct(TimelineWrite $util)
{
$this->util = new TimelineWrite($connection);
$this->util = $util;
}
public function handle(Event $event): void

12
lib/Listeners/PostWriteListener.php
View File

@ -22,20 +22,19 @@ declare(strict_types=1);
namespace OCA\Memories\Listeners;
use OCA\Memories\Db\TimelineWrite;
use OCA\Memories\Service\Index;
use OCP\EventDispatcher\Event;
use OCP\EventDispatcher\IEventListener;
use OCP\Files\Events\Node\NodeTouchedEvent;
use OCP\Files\Events\Node\NodeWrittenEvent;
use OCP\Files\Folder;
use OCP\IDBConnection;
class PostWriteListener implements IEventListener
{
private TimelineWrite $timelineWrite;
public function __construct(IDBConnection $connection)
public function __construct(TimelineWrite $timelineWrite)
{
$this->timelineWrite = new TimelineWrite($connection);
$this->timelineWrite = $timelineWrite;
}
public function handle(Event $event): void
@ -46,12 +45,9 @@ class PostWriteListener implements IEventListener
}
$node = $event->getNode();
if ($node instanceof Folder) {
return;
}
// Check the mime type first
if (!$this->timelineWrite->getFileType($node)) {
if (!Index::isSupported($node)) {
return;
}

69
lib/Migration/Version500000Date20230414042534.php 100644
View File

@ -0,0 +1,69 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Your name <your@email.com>
* @author Your name <your@email.com>
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Migration;
use OCP\DB\ISchemaWrapper;
use OCP\DB\Types;
use OCP\Migration\IOutput;
use OCP\Migration\SimpleMigrationStep;
/**
* Auto-generated migration step: Please modify to your needs!
*/
class Version500000Date20230414042534 extends SimpleMigrationStep
{
/**
* @param \Closure(): ISchemaWrapper $schemaClosure
*/
public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
{
}
/**
* @param \Closure(): ISchemaWrapper $schemaClosure
*/
public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
{
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
$table = $schema->getTable('memories_livephoto');
if (!$table->hasColumn('orphan')) {
$table->addColumn('orphan', Types::BOOLEAN, [
'notnull' => false,
'default' => false,
]);
}
return $schema;
}
/**
* @param \Closure(): ISchemaWrapper $schemaClosure
*/
public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
{
}
}

66
lib/Service/Index.php
View File

@ -30,13 +30,17 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\Files\Node;
use OCP\IDBConnection;
use OCP\IPreview;
use OCP\ITempManager;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Output\OutputInterface;
class Index
{
public ?OutputInterface $output;
protected IRootFolder $rootFolder;
protected TimelineWrite $timelineWrite;
protected IDBConnection $db;
@ -74,12 +78,14 @@ class Index
$mode = Util::getSystemConfig('memories.index.mode');
if (null !== $folder) {
$paths = [$folder];
} elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
} elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
$paths = ['/'];
} elseif ('2' === $mode) { // timeline
$paths = \OCA\Memories\Exif::getTimelinePaths($uid);
} elseif ('3' === $mode) { // custom
$paths = [Util::getSystemConfig('memories.index.path')];
} else {
throw new \Exception('Invalid index mode');
}
// If a folder is specified, traverse only that folder
@ -90,11 +96,8 @@ class Index
throw new \Exception('Not a folder');
}
} catch (\Exception $e) {
if (\OC::$CLI && null !== $folder) { // admin asked for this explicitly
throw new \Exception("The specified folder {$path} does not exist for ${uid}");
}
$this->error("The specified folder {$path} does not exist for {$uid}");
$this->logger->warning("The specified folder {$path} does not exist for ${uid}");
continue;
}
@ -138,24 +141,18 @@ class Index
->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
;
// TODO: check if forcing a refresh is needed
// Check in memories table
$query->leftJoin('f', 'memories', 'm', $query->expr()->andX(
$query->expr()->eq('f.fileid', 'm.fileid'),
$query->expr()->eq('f.mtime', 'm.mtime')
));
// Filter out files that are already indexed
$addFilter = function (string $table, string $alias) use (&$query) {
$query->leftJoin('f', $table, $alias, $query->expr()->andX(
$query->expr()->eq('f.fileid', "$alias.fileid"),
$query->expr()->eq('f.mtime', "$alias.mtime"),
$query->expr()->eq("$alias.orphan", $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL))
));
// Check in livephoto table
$query->leftJoin('f', 'memories_livephoto', 'mlp', $query->expr()->andX(
$query->expr()->eq('f.fileid', 'mlp.fileid'),
$query->expr()->eq('f.mtime', 'mlp.mtime')
));
// Exclude files that are already indexed
$query->andWhere($query->expr()->andX(
$query->expr()->isNull('m.mtime'),
$query->expr()->isNull('mlp.mtime'),
));
$query->andWhere($query->expr()->isNull("$alias.fileid"));
};
$addFilter('memories', 'm');
$addFilter('memories_livephoto', 'lp');
// Get file IDs to actually index
$fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
@ -188,10 +185,7 @@ class Index
try {
$this->timelineWrite->processFile($file);
} catch (\Exception $e) {
$this->logger->error('Failed to index file {file}: {error}', [
'file' => $file->getPath(),
'error' => $e->getMessage(),
]);
$this->error("Failed to index file {$file->getPath()}: {$e->getMessage()}");
}
$this->tempManager->clean();
@ -245,7 +239,7 @@ class Index
/**
* Check if a file is supported.
*/
public static function isSupported(File $file): bool
public static function isSupported(Node $file): bool
{
return \in_array($file->getMimeType(), self::getMimeList(), true);
}
@ -257,4 +251,22 @@ class Index
{
return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
}
/** Log to console if CLI or logger */
private function error(string $message)
{
$this->logger->error($message);
if ($this->output) {
$this->output->writeln("<error>{$message}</error>");
}
}
/** Log to console if CLI */
private function log(string $message)
{
if ($this->output) {
$this->output->writeln($message);
}
}
}