2023-04-14 02:43:13 +00:00
|
|
|
<?php
|
|
|
|
|
|
|
|
declare(strict_types=1);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @copyright Copyright (c) 2022, Varun Patil <radialapps@gmail.com>
|
|
|
|
* @author Varun Patil <radialapps@gmail.com>
|
|
|
|
* @license AGPL-3.0-or-later
|
|
|
|
*
|
|
|
|
* This program is free software: you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU Affero General Public License as
|
|
|
|
* published by the Free Software Foundation, either version 3 of the
|
|
|
|
* License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU Affero General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
namespace OCA\Memories\Service;
|
|
|
|
|
|
|
|
use OCA\Memories\AppInfo\Application;
|
|
|
|
use OCA\Memories\Db\TimelineWrite;
|
2023-10-22 19:58:33 +00:00
|
|
|
use OCA\Memories\Settings\SystemConfig;
|
2023-04-14 02:43:13 +00:00
|
|
|
use OCA\Memories\Util;
|
|
|
|
use OCP\DB\QueryBuilder\IQueryBuilder;
|
|
|
|
use OCP\Files\File;
|
|
|
|
use OCP\Files\Folder;
|
|
|
|
use OCP\Files\IRootFolder;
|
2023-04-14 04:59:01 +00:00
|
|
|
use OCP\Files\Node;
|
2023-04-14 02:43:13 +00:00
|
|
|
use OCP\IDBConnection;
|
|
|
|
use OCP\IPreview;
|
|
|
|
use OCP\ITempManager;
|
|
|
|
use Psr\Log\LoggerInterface;
|
2023-04-14 05:24:01 +00:00
|
|
|
use Symfony\Component\Console\Output\ConsoleSectionOutput;
|
2023-04-14 04:59:01 +00:00
|
|
|
use Symfony\Component\Console\Output\OutputInterface;
|
2023-04-14 02:43:13 +00:00
|
|
|
|
|
|
|
class Index
|
|
|
|
{
|
2023-04-17 01:07:57 +00:00
|
|
|
public ?OutputInterface $output = null;
|
|
|
|
public ?ConsoleSectionOutput $section = null;
|
2023-11-17 18:20:49 +00:00
|
|
|
public bool $verbose = false;
|
2023-04-17 01:07:57 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Callback to check if the process should continue.
|
|
|
|
* This is called before every file is indexed.
|
2023-10-15 19:46:35 +00:00
|
|
|
*
|
|
|
|
* @var null|\Closure(): bool
|
2023-04-17 01:07:57 +00:00
|
|
|
*/
|
|
|
|
public ?\Closure $continueCheck = null;
|
2023-04-14 04:59:01 +00:00
|
|
|
|
2023-10-15 19:46:35 +00:00
|
|
|
/** @var string[] */
|
2023-04-14 02:43:13 +00:00
|
|
|
private static ?array $mimeList = null;
|
|
|
|
|
|
|
|
public function __construct(
|
2023-10-15 01:51:17 +00:00
|
|
|
protected IRootFolder $rootFolder,
|
|
|
|
protected TimelineWrite $tw,
|
|
|
|
protected IDBConnection $db,
|
|
|
|
protected ITempManager $tempManager,
|
2023-10-15 01:59:00 +00:00
|
|
|
protected LoggerInterface $logger,
|
2023-10-15 01:51:17 +00:00
|
|
|
) {}
|
2023-04-14 02:43:13 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Index all files for a user.
|
|
|
|
*/
|
|
|
|
public function indexUser(string $uid, ?string $folder = null): void
|
|
|
|
{
|
2023-10-14 02:03:13 +00:00
|
|
|
$this->log("<info>Indexing user {$uid}</info>".PHP_EOL);
|
|
|
|
|
2023-04-14 02:43:13 +00:00
|
|
|
\OC_Util::tearDownFS();
|
|
|
|
\OC_Util::setupFS($uid);
|
|
|
|
|
|
|
|
// Get the root folder of the user
|
|
|
|
$root = $this->rootFolder->getUserFolder($uid);
|
|
|
|
|
|
|
|
// Get paths of folders to index
|
2023-10-22 19:58:33 +00:00
|
|
|
$mode = SystemConfig::get('memories.index.mode');
|
2023-04-14 02:43:13 +00:00
|
|
|
if (null !== $folder) {
|
|
|
|
$paths = [$folder];
|
2023-04-14 04:59:01 +00:00
|
|
|
} elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
|
2023-04-14 02:43:13 +00:00
|
|
|
$paths = ['/'];
|
|
|
|
} elseif ('2' === $mode) { // timeline
|
2023-04-16 23:03:59 +00:00
|
|
|
$paths = Util::getTimelinePaths($uid);
|
2023-04-14 02:43:13 +00:00
|
|
|
} elseif ('3' === $mode) { // custom
|
2023-10-22 19:58:33 +00:00
|
|
|
$paths = [SystemConfig::get('memories.index.path')];
|
2023-04-14 04:59:01 +00:00
|
|
|
} else {
|
|
|
|
throw new \Exception('Invalid index mode');
|
2023-04-14 02:43:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// If a folder is specified, traverse only that folder
|
|
|
|
foreach ($paths as $path) {
|
|
|
|
try {
|
|
|
|
$node = $root->get($path);
|
|
|
|
if (!$node instanceof Folder) {
|
|
|
|
throw new \Exception('Not a folder');
|
|
|
|
}
|
|
|
|
} catch (\Exception $e) {
|
2023-04-14 04:59:01 +00:00
|
|
|
$this->error("The specified folder {$path} does not exist for {$uid}");
|
2023-04-14 02:43:13 +00:00
|
|
|
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
$this->indexFolder($node);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Index all files in a folder.
|
|
|
|
*
|
|
|
|
* @param Folder $folder folder to index
|
|
|
|
*/
|
|
|
|
public function indexFolder(Folder $folder): void
|
|
|
|
{
|
2023-11-17 18:20:49 +00:00
|
|
|
$path = $folder->getPath();
|
|
|
|
$this->log("Indexing folder {$path}", true);
|
|
|
|
|
2023-08-20 16:42:04 +00:00
|
|
|
if ($folder->nodeExists('.nomedia') || $folder->nodeExists('.nomemories')) {
|
2023-11-17 18:20:49 +00:00
|
|
|
$this->log("Skipping folder {$path} (.nomedia / .nomemories)\n", true);
|
2023-04-14 05:24:01 +00:00
|
|
|
|
2023-04-14 02:43:13 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get all files and folders in this folders
|
|
|
|
$nodes = $folder->getDirectoryListing();
|
|
|
|
|
|
|
|
// Filter files that are supported
|
|
|
|
$mimes = self::getMimeList();
|
2023-08-30 18:10:08 +00:00
|
|
|
$files = array_filter($nodes, static fn ($n) => $n instanceof File && \in_array($n->getMimeType(), $mimes, true));
|
2023-04-14 02:43:13 +00:00
|
|
|
|
|
|
|
// Create an associative array with file ID as key
|
2023-08-30 18:10:08 +00:00
|
|
|
$files = array_combine(array_map(static fn ($n) => $n->getId(), $files), $files);
|
2023-04-14 02:43:13 +00:00
|
|
|
|
|
|
|
// Chunk array into some files each (DBs have limitations on IN clause)
|
|
|
|
$chunks = array_chunk($files, 250, true);
|
|
|
|
|
|
|
|
// Check files in each chunk
|
|
|
|
foreach ($chunks as $chunk) {
|
|
|
|
$fileIds = array_keys($chunk);
|
|
|
|
|
|
|
|
// Select all files in filecache
|
|
|
|
$query = $this->db->getQueryBuilder();
|
|
|
|
$query->select('f.fileid')
|
|
|
|
->from('filecache', 'f')
|
|
|
|
->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
|
|
|
|
;
|
|
|
|
|
2023-04-14 04:59:01 +00:00
|
|
|
// Filter out files that are already indexed
|
2023-10-14 08:25:50 +00:00
|
|
|
$addFilter = static function (string $table, string $alias) use (&$query): void {
|
2023-04-14 04:59:01 +00:00
|
|
|
$query->leftJoin('f', $table, $alias, $query->expr()->andX(
|
2023-04-14 04:59:45 +00:00
|
|
|
$query->expr()->eq('f.fileid', "{$alias}.fileid"),
|
|
|
|
$query->expr()->eq('f.mtime', "{$alias}.mtime"),
|
2023-10-15 01:59:00 +00:00
|
|
|
$query->expr()->eq("{$alias}.orphan", $query->expr()->literal(0)),
|
2023-04-14 04:59:01 +00:00
|
|
|
));
|
|
|
|
|
2023-04-14 04:59:45 +00:00
|
|
|
$query->andWhere($query->expr()->isNull("{$alias}.fileid"));
|
2023-04-14 04:59:01 +00:00
|
|
|
};
|
|
|
|
$addFilter('memories', 'm');
|
|
|
|
$addFilter('memories_livephoto', 'lp');
|
2023-04-14 02:43:13 +00:00
|
|
|
|
|
|
|
// Get file IDs to actually index
|
|
|
|
$fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
|
|
|
|
|
|
|
|
// Index files
|
|
|
|
foreach ($fileIds as $fileId) {
|
2023-04-17 01:07:57 +00:00
|
|
|
$this->ensureContinueOk();
|
2023-04-14 02:43:13 +00:00
|
|
|
$this->indexFile($chunk[$fileId]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// All folders
|
2023-08-30 18:10:08 +00:00
|
|
|
$folders = array_filter($nodes, static fn ($n) => $n instanceof Folder);
|
2023-04-14 02:43:13 +00:00
|
|
|
foreach ($folders as $folder) {
|
2023-04-17 01:07:57 +00:00
|
|
|
$this->ensureContinueOk();
|
|
|
|
|
2023-04-14 02:43:13 +00:00
|
|
|
try {
|
|
|
|
$this->indexFolder($folder);
|
2023-04-17 01:07:57 +00:00
|
|
|
} catch (ProcessClosedException $e) {
|
|
|
|
throw $e;
|
2023-04-14 02:43:13 +00:00
|
|
|
} catch (\Exception $e) {
|
2023-11-17 18:20:49 +00:00
|
|
|
$this->error("Failed to index folder {$folder->getPath()}: {$e->getMessage()}");
|
2023-04-14 02:43:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Index a single file.
|
|
|
|
*/
|
|
|
|
public function indexFile(File $file): void
|
|
|
|
{
|
2023-04-15 19:02:07 +00:00
|
|
|
$path = $file->getPath();
|
|
|
|
|
2023-04-14 02:43:13 +00:00
|
|
|
try {
|
2023-04-15 19:02:07 +00:00
|
|
|
$this->log("Indexing file {$path}", true);
|
2023-10-15 01:51:17 +00:00
|
|
|
$this->tw->processFile($file);
|
2023-04-15 19:02:07 +00:00
|
|
|
} catch (\OCP\Lock\LockedException $e) {
|
2023-04-16 19:55:34 +00:00
|
|
|
$this->log("Skipping file {$path} due to lock", true);
|
2023-04-14 02:43:13 +00:00
|
|
|
} catch (\Exception $e) {
|
2023-04-15 19:02:07 +00:00
|
|
|
$this->error("Failed to index file {$path}: {$e->getMessage()}");
|
2023-04-15 18:41:15 +00:00
|
|
|
} finally {
|
|
|
|
$this->tempManager->clean();
|
2023-04-14 02:43:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-04-25 00:59:26 +00:00
|
|
|
/**
|
|
|
|
* Cleanup all stale entries (passthrough to timeline write).
|
|
|
|
*/
|
|
|
|
public function cleanupStale(): void
|
|
|
|
{
|
2023-10-14 02:03:13 +00:00
|
|
|
$this->log('<info>Cleaning up stale index entries</info>'.PHP_EOL);
|
2023-10-15 01:51:17 +00:00
|
|
|
$this->tw->cleanupStale();
|
2023-04-25 00:59:26 +00:00
|
|
|
}
|
|
|
|
|
2023-04-14 02:43:13 +00:00
|
|
|
/**
|
|
|
|
* Get total number of files that are indexed.
|
|
|
|
*/
|
2023-10-14 08:25:50 +00:00
|
|
|
public function getIndexedCount(): int
|
2023-04-14 02:43:13 +00:00
|
|
|
{
|
|
|
|
$query = $this->db->getQueryBuilder();
|
|
|
|
$query->select($query->createFunction('COUNT(DISTINCT fileid)'))
|
|
|
|
->from('memories')
|
|
|
|
;
|
|
|
|
|
|
|
|
return (int) $query->executeQuery()->fetchOne();
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get list of MIME types to process.
|
|
|
|
*/
|
|
|
|
public static function getMimeList(): array
|
|
|
|
{
|
|
|
|
return self::$mimeList ??= array_merge(
|
|
|
|
self::getPreviewMimes(Application::IMAGE_MIMES),
|
|
|
|
Application::VIDEO_MIMES,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get list of MIME types that have a preview.
|
|
|
|
*/
|
|
|
|
public static function getPreviewMimes(array $source): array
|
|
|
|
{
|
|
|
|
$preview = \OC::$server->get(IPreview::class);
|
|
|
|
|
2023-08-30 18:10:08 +00:00
|
|
|
return array_filter($source, static fn ($m) => $preview->isMimeSupported($m));
|
2023-04-14 02:43:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get list of all supported MIME types.
|
|
|
|
*/
|
|
|
|
public static function getAllMimes(): array
|
|
|
|
{
|
|
|
|
return array_merge(
|
|
|
|
Application::IMAGE_MIMES,
|
|
|
|
Application::VIDEO_MIMES,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Check if a file is supported.
|
|
|
|
*/
|
2023-04-14 04:59:01 +00:00
|
|
|
public static function isSupported(Node $file): bool
|
2023-04-14 02:43:13 +00:00
|
|
|
{
|
|
|
|
return \in_array($file->getMimeType(), self::getMimeList(), true);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Check if a file is a video.
|
|
|
|
*/
|
|
|
|
public static function isVideo(File $file): bool
|
|
|
|
{
|
|
|
|
return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
|
|
|
|
}
|
2023-04-14 04:59:01 +00:00
|
|
|
|
2023-10-14 08:25:50 +00:00
|
|
|
/**
|
|
|
|
* Log error to console if CLI or logger.
|
|
|
|
*/
|
|
|
|
private function error(string $message): void
|
2023-04-14 04:59:01 +00:00
|
|
|
{
|
2023-10-22 19:07:22 +00:00
|
|
|
$this->logger->error($message, ['app' => 'memories']);
|
2023-04-14 04:59:01 +00:00
|
|
|
|
|
|
|
if ($this->output) {
|
2023-09-30 01:48:52 +00:00
|
|
|
$this->output->writeln("<error>{$message}</error>\n");
|
2023-04-14 04:59:01 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-10-14 08:25:50 +00:00
|
|
|
/**
|
|
|
|
* Log to console if CLI.
|
|
|
|
*/
|
|
|
|
private function log(string $message, bool $overwrite = false): void
|
2023-04-14 04:59:01 +00:00
|
|
|
{
|
2023-10-14 09:07:18 +00:00
|
|
|
if ($this->section) {
|
2023-11-17 18:20:49 +00:00
|
|
|
if ($overwrite && !$this->verbose) {
|
2023-04-14 05:24:01 +00:00
|
|
|
$this->section->clear(1);
|
|
|
|
}
|
|
|
|
$this->section->write($message);
|
2023-04-14 04:59:01 +00:00
|
|
|
}
|
|
|
|
}
|
2023-04-17 01:07:57 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Ensure that the process should go on.
|
|
|
|
*/
|
|
|
|
private function ensureContinueOk(): void
|
|
|
|
{
|
|
|
|
if (null !== $this->continueCheck && !($this->continueCheck)()) {
|
|
|
|
throw new ProcessClosedException();
|
|
|
|
}
|
|
|
|
}
|
2023-04-14 02:43:13 +00:00
|
|
|
}
|