* @author Varun Patil * @license AGPL-3.0-or-later * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ namespace OCA\Memories\Command; use OC\DB\Connection; use OC\DB\SchemaWrapper; use OCA\Memories\AppInfo\Application; use OCA\Memories\Db\TimelineWrite; use OCP\Files\File; use OCP\Files\Folder; use OCP\Files\IRootFolder; use OCP\IConfig; use OCP\IDBConnection; use OCP\IPreview; use OCP\ITempManager; use OCP\IUser; use OCP\IUserManager; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\ConsoleSectionOutput; use Symfony\Component\Console\Output\OutputInterface; class IndexOpts { public bool $refresh = false; public bool $clear = false; public bool $cleanup = false; public function __construct(InputInterface $input) { $this->refresh = (bool) $input->getOption('refresh'); $this->clear = (bool) $input->getOption('clear'); $this->cleanup = (bool) $input->getOption('cleanup'); } } class Index extends Command { /** @var int[][] */ protected array $sizes; protected IUserManager $userManager; protected IRootFolder $rootFolder; protected IPreview $preview; protected IConfig $config; protected OutputInterface $output; protected IDBConnection $connection; protected Connection $connectionForSchema; protected TimelineWrite $timelineWrite; protected ITempManager $tempManager; // Stats private int $nUser = 0; private int $nProcessed = 0; private int $nSkipped = 0; private int $nInvalid = 0; private int $nNoMedia = 0; // Helper for the progress bar private ConsoleSectionOutput $outputSection; public function __construct( IRootFolder $rootFolder, IUserManager $userManager, IPreview $preview, IConfig $config, IDBConnection $connection, Connection $connectionForSchema, ITempManager $tempManager ) { parent::__construct(); $this->userManager = $userManager; $this->rootFolder = $rootFolder; $this->preview = $preview; $this->config = $config; $this->connection = $connection; $this->connectionForSchema = $connectionForSchema; $this->tempManager = $tempManager; $this->timelineWrite = new TimelineWrite($connection); } protected function configure(): void { $this ->setName('memories:index') ->setDescription('Generate photo entries') ->addOption( 'refresh', 'f', InputOption::VALUE_NONE, 'Refresh existing entries' ) ->addOption( 'clear', null, InputOption::VALUE_NONE, 'Clear existing index before creating a new one (slow)' ) ->addOption( 'cleanup', null, InputOption::VALUE_NONE, 'Remove orphaned entries from index (e.g. from .nomedia files)' ) ; } protected function execute(InputInterface $input, OutputInterface $output): int { // Add missing indices $output->writeln('Checking database indices'); \OCA\Memories\Db\AddMissingIndices::run(new SchemaWrapper($this->connectionForSchema), $this->connectionForSchema); // Print mime type support information $output->writeln("\nMIME Type support:"); $mimes = array_merge(Application::IMAGE_MIMES, Application::VIDEO_MIMES); $someUnsupported = false; foreach ($mimes as &$mimeType) { if ($this->preview->isMimeSupported($mimeType)) { $output->writeln(" {$mimeType}: supported"); } else { $output->writeln(" {$mimeType}: not supported"); $someUnsupported = true; } } // Print file type support info if ($someUnsupported) { $output->writeln("\nSome file types are not supported by your preview provider.\nPlease see https://github.com/pulsejet/memories/wiki/File-Type-Support\n"); } else { $output->writeln("\nAll file types are supported by your preview provider.\n"); } // Get options and arguments $opts = new IndexOpts($input); // Clear index if asked for this if ($opts->clear && $input->isInteractive()) { $output->write('Are you sure you want to clear the existing index? (y/N): '); $answer = trim(fgets(STDIN)); if ('y' !== $answer) { $output->writeln('Aborting'); return 1; } } if ($opts->clear) { $this->timelineWrite->clear(); $output->writeln('Cleared existing index'); } // Orphan all entries so we can delete them later // Refresh works similarly, with a different flag on the process call if ($opts->cleanup || $opts->refresh) { $output->write('Marking all entries for refresh / cleanup ... '); $count = $this->timelineWrite->orphanAll(); $output->writeln("{$count} marked"); } // Run with the static process try { \OCA\Memories\Exif::ensureStaticExiftoolProc(); return $this->executeWithOpts($output, $opts); } catch (\Exception $e) { error_log('FATAL: '.$e->getMessage()); return 1; } finally { \OCA\Memories\Exif::closeStaticExiftoolProc(); } } protected function executeWithOpts(OutputInterface $output, IndexOpts &$opts): int { // Refuse to run without exiftool if (!$this->testExif()) { error_log('FATAL: exiftool could not be executed or test failed'); error_log('Make sure you have perl 5 installed in PATH'); return 1; } // Time measurement $startTime = microtime(true); if (\OCA\Memories\Util::isEncryptionEnabled()) { // Can work with server-side but not with e2e encryption, see https://github.com/pulsejet/memories/issues/99 error_log('FATAL: Only server-side encryption (OC_DEFAULT_MODULE) is supported, but another encryption module is enabled. Aborted.'); return 1; } $this->output = $output; // Call indexing for each user $this->userManager->callForSeenUsers(function (IUser &$user) use (&$opts) { $this->generateUserEntries($user, $opts); }); // Clear orphans if asked for this if ($opts->cleanup || $opts->refresh) { $output->write('Deleting orphaned entries ... '); $count = $this->timelineWrite->removeOrphans(); $output->writeln("{$count} deleted"); } // Show some stats $endTime = microtime(true); $execTime = (int) (($endTime - $startTime) * 1000) / 1000; $nTotal = $this->nInvalid + $this->nSkipped + $this->nProcessed + $this->nNoMedia; $this->output->writeln('=========================================='); $this->output->writeln("Checked {$nTotal} files of {$this->nUser} users in {$execTime} sec"); $this->output->writeln($this->nInvalid.' not valid media items'); $this->output->writeln($this->nNoMedia.' .nomedia folders ignored'); $this->output->writeln($this->nSkipped.' skipped because unmodified'); $this->output->writeln($this->nProcessed.' (re-)processed'); $this->output->writeln('=========================================='); return 0; } /** Make sure exiftool is available */ private function testExif() { $testfilepath = __DIR__.'/../../exiftest.jpg'; $testfile = realpath($testfilepath); if (!$testfile) { error_log("Couldn't find Exif test file {$testfile}"); return false; } $exif = null; try { $exif = \OCA\Memories\Exif::getExifFromLocalPath($testfile); } catch (\Exception $e) { error_log("Couldn't read Exif data from test file: ".$e->getMessage()); return false; } if (!$exif) { error_log('Got blank Exif data from test file'); return false; } if ('2004:08:31 19:52:58' !== $exif['DateTimeOriginal']) { error_log('Got unexpected Exif data from test file'); return false; } return true; } private function generateUserEntries(IUser &$user, IndexOpts &$opts): void { \OC_Util::tearDownFS(); \OC_Util::setupFS($user->getUID()); $uid = $user->getUID(); $userFolder = $this->rootFolder->getUserFolder($uid); $this->outputSection = $this->output->section(); ++$this->nUser; $this->outputSection->overwrite("Scanning files for {$uid}"); $this->parseFolder($userFolder, $opts); $this->outputSection->overwrite("Scanned all files for {$uid}"); } private function parseFolder(Folder &$folder, IndexOpts &$opts): void { try { // Respect the '.nomedia' file. If present don't traverse the folder if ($folder->nodeExists('.nomedia')) { ++$this->nNoMedia; return; } $nodes = $folder->getDirectoryListing(); foreach ($nodes as $i => &$node) { if ($node instanceof Folder) { $this->parseFolder($node, $opts); } elseif ($node instanceof File) { $path = $node->getPath(); $path = \strlen($path) > 80 ? '...'.substr($path, -77) : $path; $this->outputSection->overwrite("Scanning {$path}"); $this->parseFile($node, $opts); $this->tempManager->clean(); } } } catch (\Exception $e) { $this->output->writeln(sprintf( 'Could not scan folder %s: %s', $folder->getPath(), $e->getMessage() )); } } private function parseFile(File &$file, IndexOpts &$opts): void { // Process the file $res = 1; try { // If refreshing the index, force reprocessing // when the file is still an orphan. this way, the // files are reprocessed exactly once $force = $opts->refresh ? 2 : 0; // (re-)process the file $res = $this->timelineWrite->processFile($file, $force); // If the file was processed successfully, // remove it from the orphan list if ($opts->cleanup || $opts->refresh) { $this->timelineWrite->unorphan($file); } } catch (\Error $e) { $this->output->writeln(sprintf( 'Could not process file %s: %s', $file->getPath(), $e->getMessage() )); $this->output->writeln($e->getTraceAsString()); } if (2 === $res) { ++$this->nProcessed; } elseif (1 === $res) { ++$this->nSkipped; } else { ++$this->nInvalid; } } }