memories/lib/Command/Index.php

349 lines
12 KiB
PHP
Raw Normal View History

2022-08-13 01:58:37 +00:00
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022, Varun Patil <radialapps@gmail.com>
* @author Varun Patil <radialapps@gmail.com>
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Command;
2022-08-13 01:58:37 +00:00
2022-10-26 17:06:45 +00:00
use OC\DB\Connection;
use OC\DB\SchemaWrapper;
2022-10-25 00:47:25 +00:00
use OCA\Memories\AppInfo\Application;
2022-10-19 17:10:36 +00:00
use OCA\Memories\Db\TimelineWrite;
2022-08-13 01:58:37 +00:00
use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IPreview;
use OCP\IUser;
use OCP\IUserManager;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\ConsoleSectionOutput;
2022-11-21 10:18:06 +00:00
use Symfony\Component\Console\Output\OutputInterface;
2022-08-13 01:58:37 +00:00
class IndexOpts
{
public bool $refresh = false;
public bool $clear = false;
public bool $cleanup = false;
public function __construct(InputInterface $input)
{
$this->refresh = (bool) $input->getOption('refresh');
$this->clear = (bool) $input->getOption('clear');
$this->cleanup = (bool) $input->getOption('cleanup');
}
}
2022-10-19 17:10:36 +00:00
class Index extends Command
{
2022-09-09 07:31:42 +00:00
/** @var int[][] */
protected array $sizes;
protected IUserManager $userManager;
protected IRootFolder $rootFolder;
2022-10-25 00:47:25 +00:00
protected IPreview $preview;
2022-09-09 07:31:42 +00:00
protected IConfig $config;
protected OutputInterface $output;
protected IDBConnection $connection;
2022-10-26 17:06:45 +00:00
protected Connection $connectionForSchema;
2022-09-09 07:31:42 +00:00
protected TimelineWrite $timelineWrite;
// Stats
private int $nUser = 0;
2022-09-09 07:31:42 +00:00
private int $nProcessed = 0;
private int $nSkipped = 0;
private int $nInvalid = 0;
private int $nNoMedia = 0;
2022-09-09 07:31:42 +00:00
2022-10-25 17:49:42 +00:00
// Helper for the progress bar
private ConsoleSectionOutput $outputSection;
2022-10-25 17:49:42 +00:00
2022-10-19 17:10:36 +00:00
public function __construct(
IRootFolder $rootFolder,
IUserManager $userManager,
2022-10-25 00:47:25 +00:00
IPreview $preview,
2022-10-19 17:10:36 +00:00
IConfig $config,
IDBConnection $connection,
2022-11-09 09:23:12 +00:00
Connection $connectionForSchema
2022-10-19 17:15:14 +00:00
) {
2022-09-09 07:31:42 +00:00
parent::__construct();
$this->userManager = $userManager;
$this->rootFolder = $rootFolder;
2022-10-25 00:47:25 +00:00
$this->preview = $preview;
2022-09-09 07:31:42 +00:00
$this->config = $config;
$this->connection = $connection;
2022-10-26 17:06:45 +00:00
$this->connectionForSchema = $connectionForSchema;
2022-12-04 17:40:58 +00:00
$this->timelineWrite = new TimelineWrite($connection);
2022-09-09 07:31:42 +00:00
}
2022-10-19 17:10:36 +00:00
protected function configure(): void
{
2022-09-09 07:31:42 +00:00
$this
->setName('memories:index')
->setDescription('Generate photo entries')
->addOption(
'refresh',
'f',
InputOption::VALUE_NONE,
'Refresh existing entries'
2022-09-11 00:15:40 +00:00
)
->addOption(
'clear',
null,
InputOption::VALUE_NONE,
'Clear existing index before creating a new one (slow)'
)
->addOption(
'cleanup',
null,
InputOption::VALUE_NONE,
'Remove orphaned entries from index (e.g. from .nomedia files)'
2022-10-19 17:10:36 +00:00
)
;
2022-09-09 07:31:42 +00:00
}
2022-10-19 17:10:36 +00:00
protected function execute(InputInterface $input, OutputInterface $output): int
{
2022-10-26 17:06:45 +00:00
// Add missing indices
$output->writeln('Checking database indices');
\OCA\Memories\Db\AddMissingIndices::run(new SchemaWrapper($this->connectionForSchema), $this->connectionForSchema);
2022-10-25 00:47:25 +00:00
// Print mime type support information
2022-10-26 17:06:45 +00:00
$output->writeln("\nMIME Type support:");
2022-10-25 17:20:50 +00:00
$mimes = array_merge(Application::IMAGE_MIMES, Application::VIDEO_MIMES);
$someUnsupported = false;
foreach ($mimes as &$mimeType) {
2022-10-25 00:47:25 +00:00
if ($this->preview->isMimeSupported($mimeType)) {
$output->writeln(" {$mimeType}: supported");
} else {
2022-10-25 17:20:50 +00:00
$output->writeln(" {$mimeType}: <error>not supported</error>");
$someUnsupported = true;
2022-10-25 00:47:25 +00:00
}
}
2022-10-25 17:20:50 +00:00
// Print file type support info
if ($someUnsupported) {
$output->writeln("\nSome file types are not supported by your preview provider.\nPlease see https://github.com/pulsejet/memories/wiki/File-Type-Support\n");
2022-10-25 17:30:17 +00:00
} else {
$output->writeln("\nAll file types are supported by your preview provider.\n");
2022-10-25 17:20:50 +00:00
}
2022-10-25 00:47:25 +00:00
// Get options and arguments
$opts = new IndexOpts($input);
2022-09-11 00:15:40 +00:00
// Clear index if asked for this
if ($opts->clear && $input->isInteractive()) {
2022-10-19 17:10:36 +00:00
$output->write('Are you sure you want to clear the existing index? (y/N): ');
2022-09-11 00:15:40 +00:00
$answer = trim(fgets(STDIN));
2022-10-19 17:10:36 +00:00
if ('y' !== $answer) {
$output->writeln('Aborting');
2022-09-11 00:15:40 +00:00
return 1;
}
}
if ($opts->clear) {
2022-09-11 00:15:40 +00:00
$this->timelineWrite->clear();
2022-10-19 17:10:36 +00:00
$output->writeln('Cleared existing index');
2022-09-11 00:15:40 +00:00
}
// Orphan all entries so we can delete them later
if ($opts->cleanup) {
$output->write('Marking all entries for cleanup ... ');
$count = $this->timelineWrite->orphanAll();
$output->writeln("{$count} marked");
}
2022-09-09 15:18:55 +00:00
// Run with the static process
try {
\OCA\Memories\Exif::ensureStaticExiftoolProc();
2022-10-19 17:10:36 +00:00
return $this->executeWithOpts($output, $opts);
2022-09-09 15:18:55 +00:00
} catch (\Exception $e) {
2022-10-19 17:10:36 +00:00
error_log('FATAL: '.$e->getMessage());
2022-09-09 15:18:55 +00:00
return 1;
} finally {
\OCA\Memories\Exif::closeStaticExiftoolProc();
}
}
protected function executeWithOpts(OutputInterface $output, IndexOpts &$opts): int
2022-10-19 17:10:36 +00:00
{
2022-09-09 07:31:42 +00:00
// Refuse to run without exiftool
if (!$this->testExif()) {
2022-10-20 20:41:34 +00:00
error_log('FATAL: exiftool could not be executed or test failed');
error_log('Make sure you have perl 5 installed in PATH');
2022-10-19 17:10:36 +00:00
2022-09-09 15:18:55 +00:00
return 1;
2022-09-09 07:31:42 +00:00
}
// Time measurement
$startTime = microtime(true);
2022-12-04 17:33:20 +00:00
if (\OCA\Memories\Util::isEncryptionEnabled()) {
// Can work with server-side but not with e2e encryption, see https://github.com/pulsejet/memories/issues/99
error_log('FATAL: Only server-side encryption (OC_DEFAULT_MODULE) is supported, but another encryption module is enabled. Aborted.');
2022-10-19 17:10:36 +00:00
2022-09-09 07:31:42 +00:00
return 1;
}
$this->output = $output;
2022-08-13 01:58:37 +00:00
$this->userManager->callForSeenUsers(function (IUser &$user) use (&$opts) {
$this->generateUserEntries($user, $opts);
2022-08-13 01:58:37 +00:00
});
// Clear orphans if asked for this
if ($opts->cleanup) {
$output->write('Deleting orphaned entries ... ');
$count = $this->timelineWrite->removeOrphans();
$output->writeln("{$count} deleted");
}
2022-09-09 07:31:42 +00:00
// Show some stats
$endTime = microtime(true);
2022-10-19 17:10:36 +00:00
$execTime = (int) (($endTime - $startTime) * 1000) / 1000;
$nTotal = $this->nInvalid + $this->nSkipped + $this->nProcessed + $this->nNoMedia;
2022-10-25 18:48:54 +00:00
$this->output->writeln('==========================================');
$this->output->writeln("Checked {$nTotal} files of {$this->nUser} users in {$execTime} sec");
2022-10-19 17:10:36 +00:00
$this->output->writeln($this->nInvalid.' not valid media items');
$this->output->writeln($this->nNoMedia.' .nomedia folders ignored');
2022-10-19 17:10:36 +00:00
$this->output->writeln($this->nSkipped.' skipped because unmodified');
$this->output->writeln($this->nProcessed.' (re-)processed');
$this->output->writeln('==========================================');
2022-09-09 07:31:42 +00:00
return 0;
}
2022-10-19 17:10:36 +00:00
/** Make sure exiftool is available */
private function testExif()
{
2022-11-24 11:13:34 +00:00
$testfilepath = __DIR__.'/../../exiftest.jpg';
$testfile = realpath($testfilepath);
if (!$testfile) {
error_log("Couldn't find Exif test file {$testfile}");
2022-10-19 17:10:36 +00:00
return false;
}
$exif = null;
try {
$exif = \OCA\Memories\Exif::getExifFromLocalPath($testfile);
2022-10-19 17:10:36 +00:00
} catch (\Exception $e) {
error_log("Couldn't read Exif data from test file: ".$e->getMessage());
return false;
}
if (!$exif) {
error_log('Got blank Exif data from test file');
return false;
}
if ('2004:08:31 19:52:58' !== $exif['DateTimeOriginal']) {
error_log('Got unexpected Exif data from test file');
return false;
}
return true;
}
private function generateUserEntries(IUser &$user, IndexOpts &$opts): void
2022-10-19 17:10:36 +00:00
{
2022-09-09 07:31:42 +00:00
\OC_Util::tearDownFS();
\OC_Util::setupFS($user->getUID());
2022-09-13 17:39:38 +00:00
$uid = $user->getUID();
$userFolder = $this->rootFolder->getUserFolder($uid);
$this->outputSection = $this->output->section();
$this->parseFolder($userFolder, $opts, (float) $this->nUser, (float) $this->userManager->countSeenUsers());
$this->outputSection->overwrite('Scanned '.$userFolder->getPath());
++$this->nUser;
2022-09-09 07:31:42 +00:00
}
private function parseFolder(Folder &$folder, IndexOpts &$opts, float $progress_i, float $progress_n): void
2022-10-19 17:10:36 +00:00
{
2022-09-09 07:31:42 +00:00
try {
2022-09-11 00:22:05 +00:00
// Respect the '.nomedia' file. If present don't traverse the folder
if ($folder->nodeExists('.nomedia')) {
++$this->nNoMedia;
2022-11-21 10:18:06 +00:00
2022-09-11 00:22:05 +00:00
return;
}
2022-09-09 07:31:42 +00:00
$nodes = $folder->getDirectoryListing();
2022-11-21 10:18:06 +00:00
foreach ($nodes as $i => &$node) {
2022-09-09 07:31:42 +00:00
if ($node instanceof Folder) {
$new_progress_i = (float) ($progress_i * \count($nodes) + $i);
$new_progress_n = (float) ($progress_n * \count($nodes));
$this->parseFolder($node, $opts, $new_progress_i, $new_progress_n);
2022-09-09 07:31:42 +00:00
} elseif ($node instanceof File) {
$progress = (float) (($progress_i / $progress_n) * 100);
$this->outputSection->overwrite(sprintf('%.2f%%', $progress).' scanning '.$node->getPath());
$this->parseFile($node, $opts);
2022-09-09 07:31:42 +00:00
}
}
2022-11-24 10:54:07 +00:00
} catch (\Exception $e) {
2022-10-19 17:10:36 +00:00
$this->output->writeln(sprintf(
2022-11-24 10:54:07 +00:00
'<error>Could not scan folder %s: %s</error>',
2022-09-09 07:31:42 +00:00
$folder->getPath(),
2022-11-24 10:54:07 +00:00
$e->getMessage()
2022-09-09 07:31:42 +00:00
));
}
}
private function parseFile(File &$file, IndexOpts &$opts): void
2022-10-19 17:10:36 +00:00
{
2022-11-07 20:46:43 +00:00
// Process the file
2022-11-24 10:54:07 +00:00
$res = 1;
try {
$res = $this->timelineWrite->processFile($file, $opts->refresh);
if ($opts->cleanup) {
$this->timelineWrite->unorphan($file);
}
2022-12-22 18:49:06 +00:00
} catch (\Error $e) {
2022-11-24 10:54:07 +00:00
$this->output->writeln(sprintf(
'<error>Could not process file %s: %s</error>',
$file->getPath(),
$e->getMessage()
));
2022-12-22 18:49:06 +00:00
$this->output->writeln($e->getTraceAsString());
2022-11-24 10:54:07 +00:00
}
2022-10-19 17:10:36 +00:00
if (2 === $res) {
++$this->nProcessed;
} elseif (1 === $res) {
2022-10-19 17:10:36 +00:00
++$this->nSkipped;
2022-09-09 07:31:42 +00:00
} else {
2022-10-19 17:10:36 +00:00
++$this->nInvalid;
2022-09-09 07:31:42 +00:00
}
}
2022-10-19 17:10:36 +00:00
}