RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

index: move to service (partial) 

Signed-off-by: Varun Patil <varunpatil@ucla.edu>
pull/579/head
Varun Patil 2023-04-13 19:43:13 -07:00
parent d2273cc76e
commit 641574ccd2
14 changed files with 609 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/BinExt.php
View File

@ -55,12 +55,33 @@ class BinExt
throw new \Exception("failed to run exiftool: {$cmd}");
}
// Check version
$version = trim($out);
$target = self::EXIFTOOL_VER;
if (!version_compare($version, $target, '=')) {
throw new \Exception("version does not match {$version} <==> {$target}");
}
// Test with actual file
$file = realpath(__DIR__.'/../exiftest.jpg');
if (!$file) {
throw new \Exception('Could not find EXIF test file');
}
try {
$exif = \OCA\Memories\Exif::getExifFromLocalPath($file);
} catch (\Exception $e) {
throw new \Exception("Couldn't read Exif data from test file: ".$e->getMessage());
}
if (!$exif) {
throw new \Exception('Got no Exif data from test file');
}
if (($exp = '2004:08:31 19:52:58') !== ($got = $exif['DateTimeOriginal'])) {
throw new \Exception("Got wrong Exif data from test file {$exp} <==> {$got}");
}
return true;
}

338
lib/Command/Index.php
View File

@ -23,17 +23,11 @@ declare(strict_types=1);
namespace OCA\Memories\Command;
use OC\DB\Connection;
use OC\DB\SchemaWrapper;
use OCA\Memories\AppInfo\Application;
use OCA\Memories\BinExt;
use OCA\Memories\Db\TimelineWrite;
use OCP\Files\File;
use OCP\Files\Folder;
use OCA\Memories\Service;
use OCP\Files\IRootFolder;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IPreview;
use OCP\ITempManager;
use OCP\IUser;
use OCP\IUserManager;
use Symfony\Component\Console\Command\Command;
@ -44,17 +38,15 @@ use Symfony\Component\Console\Output\OutputInterface;
class IndexOpts
{
public bool $refresh = false;
public bool $force = false;
public bool $clear = false;
public bool $cleanup = false;
public ?string $user = null;
public ?string $folder = null;
public function __construct(InputInterface $input)
{
$this->refresh = (bool) $input->getOption('refresh');
$this->force = (bool) $input->getOption('force');
$this->clear = (bool) $input->getOption('clear');
$this->cleanup = (bool) $input->getOption('cleanup');
$this->user = $input->getOption('user');
$this->folder = $input->getOption('folder');
}
@ -67,22 +59,13 @@ class Index extends Command
protected IUserManager $userManager;
protected IRootFolder $rootFolder;
protected IPreview $preview;
protected IConfig $config;
protected OutputInterface $output;
protected IDBConnection $connection;
protected Connection $connectionForSchema;
protected Service\Index $indexer;
protected TimelineWrite $timelineWrite;
protected ITempManager $tempManager;
// Stats
private int $nUser = 0;
private int $nProcessed = 0;
private int $nSkipped = 0;
private int $nInvalid = 0;
private int $nNoMedia = 0;
// Helper for the progress bar
// IO
private InputInterface $input;
private OutputInterface $output;
private ConsoleSectionOutput $outputSection;
// Command options
@ -91,22 +74,17 @@ class Index extends Command
public function __construct(
IRootFolder $rootFolder,
IUserManager $userManager,
IPreview $preview,
IConfig $config,
IDBConnection $connection,
Connection $connectionForSchema,
ITempManager $tempManager
Service\Index $indexer,
TimelineWrite $timelineWrite
) {
parent::__construct();
$this->userManager = $userManager;
$this->rootFolder = $rootFolder;
$this->preview = $preview;
$this->config = $config;
$this->connection = $connection;
$this->connectionForSchema = $connectionForSchema;
$this->tempManager = $tempManager;
$this->timelineWrite = new TimelineWrite($connection);
$this->indexer = $indexer;
$this->timelineWrite = $timelineWrite;
}
protected function configure(): void
@ -115,77 +93,34 @@ class Index extends Command
->setName('memories:index')
->setDescription('Generate photo entries')
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Index only the specified user')
->addOption('folder', null, InputOption::VALUE_REQUIRED, 'Index only the specified folder')
->addOption('refresh', 'f', InputOption::VALUE_NONE, 'Refresh existing entries')
->addOption('clear', null, InputOption::VALUE_NONE, 'Clear existing index before creating a new one (slow)')
->addOption('cleanup', null, InputOption::VALUE_NONE, 'Remove orphaned entries from index (e.g. from .nomedia files)')
->addOption('folder', null, InputOption::VALUE_REQUIRED, 'Index only the specified folder (relative to the user\'s root)')
->addOption('force', 'f', InputOption::VALUE_NONE, 'Force refresh of existing index entries')
->addOption('clear', null, InputOption::VALUE_NONE, 'Clear all existing index entries')
;
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
// Add missing indices
$output->writeln('Checking database indices');
\OCA\Memories\Db\AddMissingIndices::run(new SchemaWrapper($this->connectionForSchema), $this->connectionForSchema);
// Print mime type support information
$output->writeln("\nMIME Type support:");
$mimes = array_merge(Application::IMAGE_MIMES, Application::VIDEO_MIMES);
$someUnsupported = false;
foreach ($mimes as $mimeType) {
if ($this->preview->isMimeSupported($mimeType)) {
$output->writeln(" {$mimeType}: supported");
} else {
$output->writeln(" {$mimeType}: <error>not supported</error>");
$someUnsupported = true;
}
}
// Print file type support info
if ($someUnsupported) {
$output->writeln("\nSome file types are not supported by your preview provider.\nPlease see https://github.com/pulsejet/memories/wiki/File-Type-Support\n");
} else {
$output->writeln("\nAll file types are supported by your preview provider.\n");
}
// Get options and arguments
// Store input/output/opts for later use
$this->input = $input;
$this->output = $output;
$this->opts = new IndexOpts($input);
// Clear index if asked for this
if ($this->opts->clear && $input->isInteractive()) {
$output->write('Are you sure you want to clear the existing index? (y/N): ');
$answer = trim(fgets(STDIN));
if ('y' !== $answer) {
$output->writeln('Aborting');
return 1;
}
}
if ($this->opts->clear) {
$this->timelineWrite->clear();
$output->writeln('Cleared existing index');
}
// Detect incompatible options
if ($this->opts->cleanup && ($this->opts->user || $this->opts->folder)) {
$output->writeln('<error>Cannot use --cleanup with --user or --folder</error>');
return 1;
}
// Orphan all entries so we can delete them later
// Refresh works similarly, with a different flag on the process call
if ($this->opts->cleanup || $this->opts->refresh) {
$output->write('Marking all entries for refresh / cleanup ... ');
$count = $this->timelineWrite->orphanAll();
$output->writeln("{$count} marked");
}
// Run with the static process
try {
// Use static exiftool process
\OCA\Memories\Exif::ensureStaticExiftoolProc();
if (!BinExt::testExiftool()) { // throws
throw new \Exception('exiftool could not be executed or test failed');
}
return $this->executeNow($output);
// Perform steps based on opts
$this->checkClear();
$this->checkForce();
// Run the indexer
$this->runIndex();
return 0;
} catch (\Exception $e) {
$this->output->writeln("<error>{$e->getMessage()}</error>");
@ -195,197 +130,60 @@ class Index extends Command
}
}
protected function executeNow(OutputInterface $output): int
/**
* Check and act on the clear option if set.
*/
protected function checkClear(): void
{
// Refuse to run without exiftool
if (!$this->testExif()) {
error_log('FATAL: exiftool could not be executed or test failed');
error_log('Make sure you have perl 5 installed in PATH');
if ($this->opts->clear) {
if ($this->input->isInteractive()) {
$this->output->write('Are you sure you want to clear the existing index? (y/N): ');
if ('y' !== trim(fgets(STDIN))) {
$this->output->writeln('Aborting');
return 1;
exit;
}
}
$this->timelineWrite->clear();
$this->output->writeln('Cleared existing index');
}
}
// Time measurement
$startTime = microtime(true);
/**
* Check and act on the force option if set.
*/
protected function checkForce(): void
{
if ($this->opts->force) {
$this->output->writeln('Forcing refresh of existing index entries');
if (\OCA\Memories\Util::isEncryptionEnabled()) {
// Can work with server-side but not with e2e encryption, see https://github.com/pulsejet/memories/issues/99
error_log('FATAL: Only server-side encryption (OC_DEFAULT_MODULE) is supported, but another encryption module is enabled. Aborted.');
return 1;
// TODO
}
$this->output = $output;
}
/**
* Run the indexer.
*/
protected function runIndex(): void
{
// Call indexing for specified or each user
if ($uid = $this->opts->user) {
if ($user = $this->userManager->get($uid)) {
$this->indexOneUser($user);
$this->indexer->indexUser($user->getUID(), $this->opts->folder);
} else {
throw new \Exception("User {$uid} not found");
}
} else {
$this->userManager->callForSeenUsers(function (IUser $user) {
$this->indexOneUser($user);
});
}
// Clear orphans if asked for this
if (($this->opts->cleanup || $this->opts->refresh) && !($this->opts->user || $this->opts->folder)) {
$output->write('Deleting orphaned entries ... ');
$count = $this->timelineWrite->removeOrphans();
$output->writeln("{$count} deleted");
}
// Show some stats
$endTime = microtime(true);
$execTime = (int) (($endTime - $startTime) * 1000) / 1000;
$nTotal = $this->nInvalid + $this->nSkipped + $this->nProcessed + $this->nNoMedia;
$this->output->writeln('==========================================');
$this->output->writeln("Checked {$nTotal} files of {$this->nUser} users in {$execTime} sec");
$this->output->writeln($this->nInvalid.' not valid media items');
$this->output->writeln($this->nNoMedia.' .nomedia folders ignored');
$this->output->writeln($this->nSkipped.' skipped because unmodified');
$this->output->writeln($this->nProcessed.' (re-)processed');
$this->output->writeln('==========================================');
return 0;
}
/** Make sure exiftool is available */
private function testExif()
{
$testfilepath = __DIR__.'/../../exiftest.jpg';
$testfile = realpath($testfilepath);
if (!$testfile) {
error_log("Couldn't find Exif test file {$testfile}");
return false;
}
$exif = null;
try {
$exif = \OCA\Memories\Exif::getExifFromLocalPath($testfile);
} catch (\Exception $e) {
error_log("Couldn't read Exif data from test file: ".$e->getMessage());
return false;
}
if (!$exif) {
error_log('Got blank Exif data from test file');
return false;
}
if ('2004:08:31 19:52:58' !== $exif['DateTimeOriginal']) {
error_log('Got unexpected Exif data from test file');
return false;
}
return true;
}
private function indexOneUser(IUser $user): void
{
\OC_Util::tearDownFS();
\OC_Util::setupFS($user->getUID());
$uid = $user->getUID();
$folder = $this->rootFolder->getUserFolder($uid);
if ($path = $this->opts->folder) {
try {
$folder = $folder->get($path);
} catch (\OCP\Files\NotFoundException $e) {
$this->output->writeln("<error>Folder {$path} not found for user {$uid}</error>");
return;
}
if (!$folder instanceof Folder) {
$this->output->writeln("<error>Path {$path} is not a folder for user {$uid}</error>");
return;
}
}
$this->outputSection = $this->output->section();
++$this->nUser;
$this->outputSection->overwrite("Scanning files for {$uid}");
$this->indexFolder($folder);
$this->outputSection->overwrite("Scanned all files for {$uid}");
}
private function indexFolder(Folder $folder): void
{
try {
// Respect the '.nomedia' file. If present don't traverse the folder
if ($folder->nodeExists('.nomedia')) {
++$this->nNoMedia;
return;
}
$nodes = $folder->getDirectoryListing();
foreach ($nodes as $i => $node) {
if ($node instanceof Folder) {
$this->indexFolder($node);
} elseif ($node instanceof File) {
$path = $node->getPath();
$path = \strlen($path) > 80 ? '...'.substr($path, -77) : $path;
$this->outputSection->overwrite("Scanning {$path}");
$this->indexFile($node);
$this->tempManager->clean();
try {
$uid = $user->getUID();
$this->output->writeln("Indexing user {$uid}");
$this->indexer->indexUser($uid, $this->opts->folder);
} catch (\Exception $e) {
$this->output->writeln("<error>{$e->getMessage()}</error>");
}
}
} catch (\Exception $e) {
$this->output->writeln(sprintf(
'<error>Could not scan folder %s: %s</error>',
$folder->getPath(),
$e->getMessage()
));
}
}
private function indexFile(File $file): void
{
// Process the file
$res = 1;
try {
// If refreshing the index, force reprocessing
// when the file is still an orphan. this way, the
// files are reprocessed exactly once
$force = $this->opts->refresh ? 2 : 0;
// (re-)process the file
$res = $this->timelineWrite->processFile($file, $force);
// If the file was processed successfully,
// remove it from the orphan list
if ($this->opts->cleanup || $this->opts->refresh) {
$this->timelineWrite->unorphan($file);
}
} catch (\Error $e) {
$this->output->writeln(sprintf(
'<error>Could not process file %s: %s</error>',
$file->getPath(),
$e->getMessage()
));
$this->output->writeln($e->getTraceAsString());
}
if (2 === $res) {
++$this->nProcessed;
} elseif (1 === $res) {
++$this->nSkipped;
} else {
++$this->nInvalid;
});
}
}
}

4
lib/Controller/ArchiveController.php
View File

@ -42,7 +42,6 @@ class ArchiveController extends GenericApiController
public function archive(string $id): Http\Response
{
return Util::guardEx(function () use ($id) {
$uid = Util::getUID();
$userFolder = Util::getUserFolder();
// Check for permissions and get numeric Id
@ -58,8 +57,7 @@ class ArchiveController extends GenericApiController
}
// Create archive folder in the root of the user's configured timeline
$configPath = Exif::removeExtraSlash(Exif::getPhotosPath($this->config, $uid));
$configPaths = explode(';', $configPath);
$configPaths = Exif::getTimelinePaths(Util::getUID());
$timelineFolders = [];
$timelinePaths = [];

10
lib/Controller/OtherController.php
View File

@ -128,6 +128,16 @@ class OtherController extends GenericApiController
// Check for system perl
$status['perl'] = $this->getExecutableStatus(exec('which perl'));
// Check number of indexed files
$index = \OC::$server->get(\OCA\Memories\Service\Index::class);
$status['indexed_count'] = $index->getIndexedCount();
// Check supported preview mimes
$status['mimes'] = $index->getPreviewMimes($index->getAllMimes());
// Check for bad encryption module
$status['bad_encryption'] = \OCA\Memories\Util::isEncryptionEnabled();
// Get GIS status
$places = \OC::$server->get(\OCA\Memories\Service\Places::class);

9
lib/Db/AddMissingIndices.php
View File

@ -2,19 +2,18 @@
namespace OCA\Memories\Db;
use OC\DB\Connection;
use OC\DB\SchemaWrapper;
class AddMissingIndices
{
/**
* Add missing indices to the database schema.
*
* @param SchemaWrapper $schema Schema wrapper
* @param null|Connection $connection Connection to db
*/
public static function run(SchemaWrapper $schema, $connection)
public static function run()
{
$connection = \OC::$server->get(\OC\DB\Connection::class);
$schema = new SchemaWrapper($connection);
// Should migrate at end
$shouldMigrate = false;

12
lib/Db/FsManager.php
View File

@ -98,14 +98,14 @@ class FsManager
$folder = $userFolder->get(Exif::removeExtraSlash($folderPath));
$root->addFolder($folder);
} else {
$timelinePath = $this->request->getParam('timelinePath', Exif::getPhotosPath($this->config, $uid));
$timelinePath = Exif::removeExtraSlash($timelinePath);
$paths = Exif::getTimelinePaths($uid);
if ($path = $this->request->getParam('timelinePath', null)) {
$paths = [Exif::removeExtraSlash($path)];
}
// Multiple timeline path support
$paths = explode(';', $timelinePath);
foreach ($paths as &$path) {
$folder = $userFolder->get(trim($path));
$root->addFolder($folder);
foreach ($paths as $path) {
$root->addFolder($userFolder->get($path));
}
$root->addMountPoints();
}

21
lib/Db/LivePhoto.php
View File

@ -19,7 +19,7 @@ class LivePhoto
}
/** Check if a given Exif data is the video part of a Live Photo */
public function isVideoPart(array &$exif)
public function isVideoPart(array $exif)
{
return \array_key_exists('MIMEType', $exif)
&& 'video/quicktime' === $exif['MIMEType']
@ -27,7 +27,7 @@ class LivePhoto
}
/** Get liveid from photo part */
public function getLivePhotoId(File &$file, array &$exif)
public function getLivePhotoId(File $file, array $exif)
{
// Apple JPEG (MOV has ContentIdentifier)
if (\array_key_exists('MediaGroupUUID', $exif)) {
@ -100,7 +100,10 @@ class LivePhoto
return '';
}
public function processVideoPart(File &$file, array &$exif)
/**
* Process video part of Live Photo.
*/
public function processVideoPart(File $file, array $exif)
{
$fileId = $file->getId();
$mtime = $file->getMTime();
@ -142,4 +145,16 @@ class LivePhoto
}
}
}
/**
* Delete entry from memories_livephoto table.
*/
public function deleteVideoPart(File $file): void
{
$query = $this->connection->getQueryBuilder();
$query->delete('memories_livephoto')
->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT)))
;
$query->executeStatement();
}
}

144
lib/Db/TimelineWrite.php
View File

@ -4,8 +4,8 @@ declare(strict_types=1);
namespace OCA\Memories\Db;
use OCA\Memories\AppInfo\Application;
use OCA\Memories\Exif;
use OCA\Memories\Service\Index;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\File;
use OCP\IDBConnection;
@ -32,105 +32,50 @@ class TimelineWrite
$this->livePhoto = new LivePhoto($connection);
}
/**
* Check if a file has a valid mimetype for processing.
*
* @return int 0 for invalid, 1 for image, 2 for video
*/
public function getFileType(File $file): int
{
$mime = $file->getMimeType();
if (\in_array($mime, Application::IMAGE_MIMES, true)) {
// Make sure preview generator supports the mime type
if (!$this->preview->isMimeSupported($mime)) {
return 0;
}
return 1;
}
if (\in_array($mime, Application::VIDEO_MIMES, true)) {
return 2;
}
return 0;
}
/**
* Process a file to insert Exif data into the database.
*
* @param File $file File node to process
* @param int $force 0 = none, 1 = force, 2 = force if orphan
*
* @return int 2 if processed, 1 if skipped, 0 if not valid
* @param bool $force Update the record even if the file has not changed
*/
public function processFile(File $file, int $force = 0): int
public function processFile(File $file, bool $force = false): bool
{
// There is no easy way to UPSERT in a standard SQL way, so just
// do multiple calls. The worst that can happen is more updates,
// but that's not a big deal.
// https://stackoverflow.com/questions/15252213/sql-standard-upsert-call
// Check if we want to process this file
$fileType = $this->getFileType($file);
$isvideo = (2 === $fileType);
if (!$fileType) {
return 0;
if (!Index::isSupported($file)) {
return false;
}
// Get parameters
$mtime = $file->getMtime();
$fileId = $file->getId();
$isvideo = Index::isVideo($file);
// Check if need to update
$query = $this->connection->getQueryBuilder();
$query->select('fileid', 'mtime', 'mapcluster', 'orphan', 'lat', 'lon')
->from('memories')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
$cursor = $query->executeQuery();
$prevRow = $cursor->fetch();
$cursor->closeCursor();
// Check in live-photo table in case this is a video part of a Live Photo
if (!$prevRow) {
$query = $this->connection->getQueryBuilder();
$query->select('fileid', 'mtime')
->from('memories_livephoto')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
$cursor = $query->executeQuery();
$prevRow = $cursor->fetch();
$cursor->closeCursor();
}
// Check if a forced update is required
$isForced = (1 === $force);
if (2 === $force) {
$isForced = !$prevRow
// Could be live video, force regardless
|| !\array_key_exists('orphan', $prevRow)
// If orphan, force for sure
|| $prevRow['orphan'];
}
// Get previous row
$prevRow = $this->getCurrentRow($fileId);
// Skip if not forced and file has not changed
if (!$isForced && $prevRow && ((int) $prevRow['mtime'] === $mtime)) {
return 1;
if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime)) {
return false;
}
// Get exif data
$exif = [];
try {
$exif = Exif::getExifFromFile($file);
} catch (\Exception $e) {
$exif = [];
}
// Hand off if Live Photo video part
if ($isvideo && $this->livePhoto->isVideoPart($exif)) {
$this->livePhoto->processVideoPart($file, $exif);
return 2;
return true;
}
// Delete video part if it is no longer valid
if ($prevRow && !\array_key_exists('mapcluster', $prevRow)) {
$this->livePhoto->deleteVideoPart($file);
$prevRow = null;
}
// Video parameters
@ -166,6 +111,7 @@ class TimelineWrite
$exifJson = $this->getExifJson($exif);
// Parameters for insert or update
$query = $this->connection->getQueryBuilder();
$params = [
'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
'objectid' => $query->createNamedParameter((string) $fileId, IQueryBuilder::PARAM_STR),
@ -183,29 +129,26 @@ class TimelineWrite
'mapcluster' => $query->createNamedParameter($mapCluster, IQueryBuilder::PARAM_INT),
];
if ($prevRow) {
// Update existing row
// No need to set objectid again
$query->update('memories')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
foreach ($params as $key => $value) {
if ('objectid' !== $key && 'fileid' !== $key) {
// There is no easy way to UPSERT in standard SQL
// https://stackoverflow.com/questions/15252213/sql-standard-upsert-call
try {
if ($prevRow) {
$query->update('memories')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
foreach ($params as $key => $value) {
$query->set($key, $value);
}
}
$query->executeStatement();
} else {
// Try to create new row
try {
} else {
$query->insert('memories')->values($params);
$query->executeStatement();
} catch (\Exception $ex) {
error_log('Failed to create memories record: '.$ex->getMessage());
}
}
return 2;
return $query->executeStatement() > 0;
} catch (\Exception $ex) {
error_log('Failed to create memories record: '.$ex->getMessage());
return false;
}
}
/**
@ -249,6 +192,25 @@ class TimelineWrite
}
}
/**
* Get the current row for a file_id, from either table.
*/
private function getCurrentRow(int $fileId): ?array
{
$fetch = function (string $table) use ($fileId) {
$query = $this->connection->getQueryBuilder();
return $query->select('*')
->from($table)
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
->executeQuery()
->fetch()
;
};
return $fetch('memories') ?: $fetch('memories_livephoto') ?: null;
}
/**
* Convert EXIF data to filtered JSON string.
*/

13
lib/Exif.php
View File

@ -65,16 +65,13 @@ class Exif
}
/**
* Get the path to the user's configured photos directory.
* Get list of timeline paths as array.
*/
public static function getPhotosPath(IConfig $config, string &$userId)
public static function getTimelinePaths(string $uid): array
{
$p = $config->getUserValue($userId, Application::APPNAME, 'timelinePath', '');
if (empty($p)) {
return 'Photos/';
}
return self::sanitizePath($p);
$config = \OC::$server->get(IConfig::class);
$paths = $config->getUserValue($uid, Application::APPNAME, 'timelinePath', null) ?? 'Photos/';
return array_map(fn ($p) => self::sanitizePath(trim($p)), explode(';', $paths));
}
/**

3
lib/Migration/Repair.php
View File

@ -26,6 +26,9 @@ class Repair implements IRepairStep
public function run(IOutput $output): void
{
// Add missing indices
\OCA\Memories\Db\AddMissingIndices::run();
// kill any instances of go-vod and exiftool
Util::pkill(BinExt::getName('go-vod'));
Util::pkill(BinExt::getName('exiftool'));

2
lib/Migration/Version400308Date20221026151748.php
View File

@ -52,7 +52,7 @@ class Version400308Date20221026151748 extends SimpleMigrationStep
$fileCacheTable->addIndex(['parent', 'mimetype'], 'memories_parent_mimetype');
// Add other indices
return \OCA\Memories\Db\AddMissingIndices::run($schema, null);
return $schema;
}
/**

260
lib/Service/Index.php 100644
View File

@ -0,0 +1,260 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022, Varun Patil <radialapps@gmail.com>
* @author Varun Patil <radialapps@gmail.com>
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Service;
use OCA\Memories\AppInfo\Application;
use OCA\Memories\Db\TimelineWrite;
use OCA\Memories\Util;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\IDBConnection;
use OCP\IPreview;
use OCP\ITempManager;
use Psr\Log\LoggerInterface;
class Index
{
protected IRootFolder $rootFolder;
protected TimelineWrite $timelineWrite;
protected IDBConnection $db;
protected ITempManager $tempManager;
protected LoggerInterface $logger;
private static ?array $mimeList = null;
public function __construct(
IRootFolder $rootFolder,
TimelineWrite $timelineWrite,
IDBConnection $db,
ITempManager $tempManager,
LoggerInterface $logger
) {
$this->rootFolder = $rootFolder;
$this->timelineWrite = $timelineWrite;
$this->db = $db;
$this->tempManager = $tempManager;
$this->logger = $logger;
}
/**
* Index all files for a user.
*/
public function indexUser(string $uid, ?string $folder = null): void
{
\OC_Util::tearDownFS();
\OC_Util::setupFS($uid);
// Get the root folder of the user
$root = $this->rootFolder->getUserFolder($uid);
// Get paths of folders to index
$mode = Util::getSystemConfig('memories.index.mode');
if (null !== $folder) {
$paths = [$folder];
} elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
$paths = ['/'];
} elseif ('2' === $mode) { // timeline
$paths = \OCA\Memories\Exif::getTimelinePaths($uid);
} elseif ('3' === $mode) { // custom
$paths = [Util::getSystemConfig('memories.index.path')];
}
// If a folder is specified, traverse only that folder
foreach ($paths as $path) {
try {
$node = $root->get($path);
if (!$node instanceof Folder) {
throw new \Exception('Not a folder');
}
} catch (\Exception $e) {
if (\OC::$CLI && null !== $folder) { // admin asked for this explicitly
throw new \Exception("The specified folder {$path} does not exist for ${uid}");
}
$this->logger->warning("The specified folder {$path} does not exist for ${uid}");
continue;
}
$this->indexFolder($node);
}
}
/**
* Index all files in a folder.
*
* @param Folder $folder folder to index
*/
public function indexFolder(Folder $folder): void
{
// Respect the '.nomedia' file. If present don't traverse the folder
if ($folder->nodeExists('.nomedia')) {
return;
}
// Get all files and folders in this folders
$nodes = $folder->getDirectoryListing();
// Filter files that are supported
$mimes = self::getMimeList();
$files = array_filter($nodes, fn ($n) => $n instanceof File && \in_array($n->getMimeType(), $mimes, true));
// Create an associative array with file ID as key
$files = array_combine(array_map(fn ($n) => $n->getId(), $files), $files);
// Chunk array into some files each (DBs have limitations on IN clause)
$chunks = array_chunk($files, 250, true);
// Check files in each chunk
foreach ($chunks as $chunk) {
$fileIds = array_keys($chunk);
// Select all files in filecache
$query = $this->db->getQueryBuilder();
$query->select('f.fileid')
->from('filecache', 'f')
->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
;
// TODO: check if forcing a refresh is needed
// Check in memories table
$query->leftJoin('f', 'memories', 'm', $query->expr()->andX(
$query->expr()->eq('f.fileid', 'm.fileid'),
$query->expr()->eq('f.mtime', 'm.mtime')
));
// Check in livephoto table
$query->leftJoin('f', 'memories_livephoto', 'mlp', $query->expr()->andX(
$query->expr()->eq('f.fileid', 'mlp.fileid'),
$query->expr()->eq('f.mtime', 'mlp.mtime')
));
// Exclude files that are already indexed
$query->andWhere($query->expr()->andX(
$query->expr()->isNull('m.mtime'),
$query->expr()->isNull('mlp.mtime'),
));
// Get file IDs to actually index
$fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
// Index files
foreach ($fileIds as $fileId) {
$this->indexFile($chunk[$fileId]);
}
}
// All folders
$folders = array_filter($nodes, fn ($n) => $n instanceof Folder);
foreach ($folders as $folder) {
try {
$this->indexFolder($folder);
} catch (\Exception $e) {
$this->logger->error('Failed to index folder {folder}: {error}', [
'folder' => $folder->getPath(),
'error' => $e->getMessage(),
]);
}
}
}
/**
* Index a single file.
*/
public function indexFile(File $file): void
{
try {
$this->timelineWrite->processFile($file);
} catch (\Exception $e) {
$this->logger->error('Failed to index file {file}: {error}', [
'file' => $file->getPath(),
'error' => $e->getMessage(),
]);
}
$this->tempManager->clean();
}
/**
* Get total number of files that are indexed.
*/
public function getIndexedCount()
{
$query = $this->db->getQueryBuilder();
$query->select($query->createFunction('COUNT(DISTINCT fileid)'))
->from('memories')
;
return (int) $query->executeQuery()->fetchOne();
}
/**
* Get list of MIME types to process.
*/
public static function getMimeList(): array
{
return self::$mimeList ??= array_merge(
self::getPreviewMimes(Application::IMAGE_MIMES),
Application::VIDEO_MIMES,
);
}
/**
* Get list of MIME types that have a preview.
*/
public static function getPreviewMimes(array $source): array
{
$preview = \OC::$server->get(IPreview::class);
return array_filter($source, fn ($m) => $preview->isMimeSupported($m));
}
/**
* Get list of all supported MIME types.
*/
public static function getAllMimes(): array
{
return array_merge(
Application::IMAGE_MIMES,
Application::VIDEO_MIMES,
);
}
/**
* Check if a file is supported.
*/
public static function isSupported(File $file): bool
{
return \in_array($file->getMimeType(), self::getMimeList(), true);
}
/**
* Check if a file is a video.
*/
public static function isVideo(File $file): bool
{
return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
}
}

10
lib/Util.php
View File

@ -346,6 +346,16 @@ class Util
// This requires perl to be available
'memories.exiftool_no_local' => false,
// How to index user directories
// 0 = auto-index disabled
// 1 = index everything
// 2 = index only user timelines
// 3 = index only configured path
'memories.index.mode' => '1',
// Path to index (only used if indexing mode is 3)
'memories.index.path' => '/',
// Places database type identifier
'memories.gis_type' => -1,

154
src/Admin.vue
View File

@ -35,6 +35,120 @@
}}
</NcCheckboxRadioSwitch>
<!----------------------------- Index Settings ----------------------------->
<h2>{{ t("memories", "Media Indexing") }}</h2>
<template v-if="status">
<NcNoteCard :type="status.indexed_count > 0 ? 'success' : 'warning'">
{{
t("memories", "{n} media files have been indexed", {
n: status.indexed_count,
})
}}
</NcNoteCard>
<NcNoteCard type="error" v-if="status.bad_encryption">
{{
t(
"memories",
"Only server-side encryption (OC_DEFAULT_MODULE) is supported, but another encryption module is enabled."
)
}}
</NcNoteCard>
</template>
<p>
{{
t(
"memories",
"The EXIF indexes are built and checked in a periodic background task. Be careful when selecting anything other than automatic indexing. For example, setting the indexing to only timeline folders may cause delays before media becomes available to users, since the user configures the timeline only after logging in."
)
}}
{{
t(
"memories",
'Folders with a ".nomedia" file are always excluded from indexing.'
)
}}
<NcCheckboxRadioSwitch
:checked.sync="indexingMode"
value="1"
name="idxm_radio"
type="radio"
@update:checked="update('indexingMode')"
>{{ t("memories", "Index all media automatically (recommended)") }}
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:checked.sync="indexingMode"
value="2"
name="idxm_radio"
type="radio"
@update:checked="update('indexingMode')"
>{{ t("memories", "Only index timeline folders (configured by user)") }}
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:checked.sync="indexingMode"
value="3"
name="idxm_radio"
type="radio"
@update:checked="update('indexingMode')"
>{{ t("memories", "Only index a selected path") }}
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:checked.sync="indexingMode"
value="0"
name="idxm_radio"
type="radio"
@update:checked="update('indexingMode')"
>{{ t("memories", "Disable background indexing") }}
</NcCheckboxRadioSwitch>
<NcTextField
:label="t('memories', 'Indexing path (relative, all users)')"
:label-visible="true"
:value="indexingPath"
@change="update('indexingPath', $event.target.value)"
v-if="indexingMode === '3'"
/>
</p>
{{
t("memories", "For advanced usage, perform a run of indexing by running:")
}}
<br />
<code>occ memories:index</code>
<br />
{{ t("memories", "Force re-indexing of all files:") }}
<br />
<code>occ memories:index --force</code>
<br />
{{ t("memories", "You can limit indexing by user and/or folder:") }}
<br />
<code>occ memories:index --user=admin --folder=/Photos/</code>
<br />
{{ t("memories", "Clear all existing index tables:") }}
<br />
<code>occ memories:index --clear</code>
<br />
<br />
{{
t(
"memories",
"The following MIME types are configured for preview generation correctly. More documentation:"
)
}}
<a
href="https://github.com/pulsejet/memories/wiki/File-Type-Support"
target="_blank"
>
{{ t("memories", "External Link") }}
</a>
<br />
<code
><template v-for="mime in status.mimes"
>{{ mime }}<br :key="mime" /></template
></code>
<!----------------------------- Places ----------------------------->
<h2>{{ t("memories", "Reverse Geocoding") }}</h2>
@ -83,10 +197,12 @@
{{
t(
"memories",
"If the button below does not work for importing the planet data, use 'occ memories:places-setup'."
"If the button below does not work for importing the planet data, use the following command:"
)
}}
<br />
<code>occ memories:places-setup</code>
<br />
{{
t(
"memories",
@ -147,6 +263,7 @@
:label-visible="true"
:value="ffmpegPath"
@change="update('ffmpegPath', $event.target.value)"
:disabled="!enableTranscoding"
/>
<NcTextField
@ -154,11 +271,13 @@
:label-visible="true"
:value="ffprobePath"
@change="update('ffprobePath', $event.target.value)"
:disabled="!enableTranscoding"
/>
<br />
{{ t("memories", "Global default video quality (user may override)") }}
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding"
:checked.sync="videoDefaultQuality"
value="0"
name="vdq_radio"
@ -167,6 +286,7 @@
>{{ t("memories", "Auto (adaptive transcode)") }}
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding"
:checked.sync="videoDefaultQuality"
value="-1"
name="vdq_radio"
@ -175,6 +295,7 @@
>{{ t("memories", "Original (transcode with max quality)") }}
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding"
:checked.sync="videoDefaultQuality"
value="-2"
name="vdq_radio"
@ -189,14 +310,14 @@
{{
t(
"memories",
"Memories uses the go-vod transcoder. You can run go-vod exernally (e.g. in a separate Docker container for hardware acceleration) or use the built-in transcoder. To use an external transcoder, enable the following option and follow the instructions at this link:"
"Memories uses the go-vod transcoder. You can run go-vod exernally (e.g. in a separate Docker container for hardware acceleration) or use the built-in transcoder. To use an external transcoder, enable the following option and follow the instructions in the documentation:"
)
}}
<a
target="_blank"
href="https://github.com/pulsejet/memories/wiki/HW-Transcoding"
>
{{ t("memories", "external transcoder configuration") }}
{{ t("memories", "External Link") }}
</a>
<template v-if="status">
@ -206,6 +327,7 @@
</template>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding"
:checked.sync="enableExternalTranscoder"
@update:checked="update('enableExternalTranscoder')"
type="switch"
@ -214,6 +336,7 @@
</NcCheckboxRadioSwitch>
<NcTextField
:disabled="!enableTranscoding"
:label="t('memories', 'Binary path (local only)')"
:label-visible="true"
:value="goVodPath"
@ -221,6 +344,7 @@
/>
<NcTextField
:disabled="!enableTranscoding"
:label="t('memories', 'Bind address (local only)')"
:label-visible="true"
:value="goVodBind"
@ -228,6 +352,7 @@
/>
<NcTextField
:disabled="!enableTranscoding"
:label="t('memories', 'Connection address (same as bind if local)')"
:label-visible="true"
:value="goVodConnect"
@ -271,14 +396,14 @@
{{
t(
"memories",
"For more details on driver installation, check the following link:"
"For more details on driver installation, check the documentation:"
)
}}
<a
target="_blank"
href="https://github.com/pulsejet/memories/wiki/HW-Transcoding#va-api"
>
VA-API configuration
{{ t("memories", "External Link") }}
</a>
<NcNoteCard :type="vaapiStatusType" v-if="status">
@ -286,6 +411,7 @@
</NcNoteCard>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding"
:checked.sync="enableVaapi"
@update:checked="update('enableVaapi')"
type="switch"
@ -294,6 +420,7 @@
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding || !enableVaapi"
:checked.sync="enableVaapiLowPower"
@update:checked="update('enableVaapiLowPower')"
type="switch"
@ -325,6 +452,7 @@
</NcNoteCard>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding"
:checked.sync="enableNvenc"
@update:checked="update('enableNvenc')"
type="switch"
@ -332,6 +460,7 @@
{{ t("memories", "Enable acceleration with NVENC") }}
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding || !enableNvenc"
:checked.sync="enableNvencTemporalAQ"
@update:checked="update('enableNvencTemporalAQ')"
type="switch"
@ -340,6 +469,7 @@
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding || !enableNvenc"
:checked.sync="nvencScaler"
value="npp"
name="nvence_scaler_radio"
@ -349,6 +479,7 @@
>{{ t("memories", "NPP scaler") }}
</NcCheckboxRadioSwitch>
<NcCheckboxRadioSwitch
:disabled="!enableTranscoding || !enableNvenc"
:checked.sync="nvencScaler"
value="cuda"
name="nvence_scaler_radio"
@ -379,6 +510,8 @@ import NcButton from "@nextcloud/vue/dist/Components/NcButton";
const settings = {
exiftoolPath: "memories.exiftool",
exiftoolPerl: "memories.exiftool_no_local",
indexingMode: "memories.index.mode",
indexingPath: "memories.index.path",
gisType: "memories.gis_type",
@ -405,6 +538,9 @@ const invertedBooleans = ["enableTranscoding"];
type BinaryStatus = "ok" | "not_found" | "not_executable" | "test_ok" | string;
type IStatus = {
bad_encryption: boolean;
indexed_count: number;
mimes: string[];
gis_type: number;
gis_count?: number;
exiftool: BinaryStatus;
@ -430,6 +566,8 @@ export default defineComponent({
exiftoolPath: "",
exiftoolPerl: false,
indexingMode: "0",
indexingPath: "",
gisType: 0,
@ -696,5 +834,11 @@ export default defineComponent({
a {
color: var(--color-primary-element);
}
code {
padding-left: 10px;
-webkit-box-decoration-break: clone;
box-decoration-break: clone;
}
}
</style>