+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+namespace OCA\Memories\Service;
+
+use OCA\Memories\AppInfo\Application;
+use OCA\Memories\Db\TimelineWrite;
+use OCA\Memories\Util;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\File;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\IDBConnection;
+use OCP\IPreview;
+use OCP\ITempManager;
+use Psr\Log\LoggerInterface;
+
+class Index
+{
+ protected IRootFolder $rootFolder;
+ protected TimelineWrite $timelineWrite;
+ protected IDBConnection $db;
+ protected ITempManager $tempManager;
+ protected LoggerInterface $logger;
+
+ private static ?array $mimeList = null;
+
+ public function __construct(
+ IRootFolder $rootFolder,
+ TimelineWrite $timelineWrite,
+ IDBConnection $db,
+ ITempManager $tempManager,
+ LoggerInterface $logger
+ ) {
+ $this->rootFolder = $rootFolder;
+ $this->timelineWrite = $timelineWrite;
+ $this->db = $db;
+ $this->tempManager = $tempManager;
+ $this->logger = $logger;
+ }
+
+ /**
+ * Index all files for a user.
+ */
+ public function indexUser(string $uid, ?string $folder = null): void
+ {
+ \OC_Util::tearDownFS();
+ \OC_Util::setupFS($uid);
+
+ // Get the root folder of the user
+ $root = $this->rootFolder->getUserFolder($uid);
+
+ // Get paths of folders to index
+ $mode = Util::getSystemConfig('memories.index.mode');
+ if (null !== $folder) {
+ $paths = [$folder];
+ } elseif ('1' === $mode || '0' === $mode) { // everything (or nothing)
+ $paths = ['/'];
+ } elseif ('2' === $mode) { // timeline
+ $paths = \OCA\Memories\Exif::getTimelinePaths($uid);
+ } elseif ('3' === $mode) { // custom
+ $paths = [Util::getSystemConfig('memories.index.path')];
+ }
+
+ // If a folder is specified, traverse only that folder
+ foreach ($paths as $path) {
+ try {
+ $node = $root->get($path);
+ if (!$node instanceof Folder) {
+ throw new \Exception('Not a folder');
+ }
+ } catch (\Exception $e) {
+ if (\OC::$CLI && null !== $folder) { // admin asked for this explicitly
+ throw new \Exception("The specified folder {$path} does not exist for ${uid}");
+ }
+
+ $this->logger->warning("The specified folder {$path} does not exist for ${uid}");
+ continue;
+ }
+
+ $this->indexFolder($node);
+ }
+ }
+
+ /**
+ * Index all files in a folder.
+ *
+ * @param Folder $folder folder to index
+ */
+ public function indexFolder(Folder $folder): void
+ {
+ // Respect the '.nomedia' file. If present don't traverse the folder
+ if ($folder->nodeExists('.nomedia')) {
+ return;
+ }
+
+ // Get all files and folders in this folders
+ $nodes = $folder->getDirectoryListing();
+
+ // Filter files that are supported
+ $mimes = self::getMimeList();
+ $files = array_filter($nodes, fn ($n) => $n instanceof File && \in_array($n->getMimeType(), $mimes, true));
+
+ // Create an associative array with file ID as key
+ $files = array_combine(array_map(fn ($n) => $n->getId(), $files), $files);
+
+ // Chunk array into some files each (DBs have limitations on IN clause)
+ $chunks = array_chunk($files, 250, true);
+
+ // Check files in each chunk
+ foreach ($chunks as $chunk) {
+ $fileIds = array_keys($chunk);
+
+ // Select all files in filecache
+ $query = $this->db->getQueryBuilder();
+ $query->select('f.fileid')
+ ->from('filecache', 'f')
+ ->where($query->expr()->in('f.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
+ ;
+
+ // TODO: check if forcing a refresh is needed
+ // Check in memories table
+ $query->leftJoin('f', 'memories', 'm', $query->expr()->andX(
+ $query->expr()->eq('f.fileid', 'm.fileid'),
+ $query->expr()->eq('f.mtime', 'm.mtime')
+ ));
+
+ // Check in livephoto table
+ $query->leftJoin('f', 'memories_livephoto', 'mlp', $query->expr()->andX(
+ $query->expr()->eq('f.fileid', 'mlp.fileid'),
+ $query->expr()->eq('f.mtime', 'mlp.mtime')
+ ));
+
+ // Exclude files that are already indexed
+ $query->andWhere($query->expr()->andX(
+ $query->expr()->isNull('m.mtime'),
+ $query->expr()->isNull('mlp.mtime'),
+ ));
+
+ // Get file IDs to actually index
+ $fileIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+
+ // Index files
+ foreach ($fileIds as $fileId) {
+ $this->indexFile($chunk[$fileId]);
+ }
+ }
+
+ // All folders
+ $folders = array_filter($nodes, fn ($n) => $n instanceof Folder);
+ foreach ($folders as $folder) {
+ try {
+ $this->indexFolder($folder);
+ } catch (\Exception $e) {
+ $this->logger->error('Failed to index folder {folder}: {error}', [
+ 'folder' => $folder->getPath(),
+ 'error' => $e->getMessage(),
+ ]);
+ }
+ }
+ }
+
+ /**
+ * Index a single file.
+ */
+ public function indexFile(File $file): void
+ {
+ try {
+ $this->timelineWrite->processFile($file);
+ } catch (\Exception $e) {
+ $this->logger->error('Failed to index file {file}: {error}', [
+ 'file' => $file->getPath(),
+ 'error' => $e->getMessage(),
+ ]);
+ }
+
+ $this->tempManager->clean();
+ }
+
+ /**
+ * Get total number of files that are indexed.
+ */
+ public function getIndexedCount()
+ {
+ $query = $this->db->getQueryBuilder();
+ $query->select($query->createFunction('COUNT(DISTINCT fileid)'))
+ ->from('memories')
+ ;
+
+ return (int) $query->executeQuery()->fetchOne();
+ }
+
+ /**
+ * Get list of MIME types to process.
+ */
+ public static function getMimeList(): array
+ {
+ return self::$mimeList ??= array_merge(
+ self::getPreviewMimes(Application::IMAGE_MIMES),
+ Application::VIDEO_MIMES,
+ );
+ }
+
+ /**
+ * Get list of MIME types that have a preview.
+ */
+ public static function getPreviewMimes(array $source): array
+ {
+ $preview = \OC::$server->get(IPreview::class);
+
+ return array_filter($source, fn ($m) => $preview->isMimeSupported($m));
+ }
+
+ /**
+ * Get list of all supported MIME types.
+ */
+ public static function getAllMimes(): array
+ {
+ return array_merge(
+ Application::IMAGE_MIMES,
+ Application::VIDEO_MIMES,
+ );
+ }
+
+ /**
+ * Check if a file is supported.
+ */
+ public static function isSupported(File $file): bool
+ {
+ return \in_array($file->getMimeType(), self::getMimeList(), true);
+ }
+
+ /**
+ * Check if a file is a video.
+ */
+ public static function isVideo(File $file): bool
+ {
+ return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
+ }
+}
diff --git a/lib/Util.php b/lib/Util.php
index 8d579a9e..9246619a 100644
--- a/lib/Util.php
+++ b/lib/Util.php
@@ -346,6 +346,16 @@ class Util
// This requires perl to be available
'memories.exiftool_no_local' => false,
+ // How to index user directories
+ // 0 = auto-index disabled
+ // 1 = index everything
+ // 2 = index only user timelines
+ // 3 = index only configured path
+ 'memories.index.mode' => '1',
+
+ // Path to index (only used if indexing mode is 3)
+ 'memories.index.path' => '/',
+
// Places database type identifier
'memories.gis_type' => -1,
diff --git a/src/Admin.vue b/src/Admin.vue
index 32d31847..c3757c21 100644
--- a/src/Admin.vue
+++ b/src/Admin.vue
@@ -35,6 +35,120 @@
}}
+
+ {{ t("memories", "Media Indexing") }}
+
+
+
+ {{
+ t("memories", "{n} media files have been indexed", {
+ n: status.indexed_count,
+ })
+ }}
+
+
+ {{
+ t(
+ "memories",
+ "Only server-side encryption (OC_DEFAULT_MODULE) is supported, but another encryption module is enabled."
+ )
+ }}
+
+
+
+
+ {{
+ t(
+ "memories",
+ "The EXIF indexes are built and checked in a periodic background task. Be careful when selecting anything other than automatic indexing. For example, setting the indexing to only timeline folders may cause delays before media becomes available to users, since the user configures the timeline only after logging in."
+ )
+ }}
+ {{
+ t(
+ "memories",
+ 'Folders with a ".nomedia" file are always excluded from indexing.'
+ )
+ }}
+ {{ t("memories", "Index all media automatically (recommended)") }}
+
+ {{ t("memories", "Only index timeline folders (configured by user)") }}
+
+ {{ t("memories", "Only index a selected path") }}
+
+ {{ t("memories", "Disable background indexing") }}
+
+
+
+
+
+ {{
+ t("memories", "For advanced usage, perform a run of indexing by running:")
+ }}
+
+ occ memories:index
+
+ {{ t("memories", "Force re-indexing of all files:") }}
+
+ occ memories:index --force
+
+ {{ t("memories", "You can limit indexing by user and/or folder:") }}
+
+ occ memories:index --user=admin --folder=/Photos/
+
+ {{ t("memories", "Clear all existing index tables:") }}
+
+ occ memories:index --clear
+
+
+
+ {{
+ t(
+ "memories",
+ "The following MIME types are configured for preview generation correctly. More documentation:"
+ )
+ }}
+
+ {{ t("memories", "External Link") }}
+
+
+ {{ mime }}
+
{{ t("memories", "Reverse Geocoding") }}
@@ -83,10 +197,12 @@
{{
t(
"memories",
- "If the button below does not work for importing the planet data, use 'occ memories:places-setup'."
+ "If the button below does not work for importing the planet data, use the following command:"
)
}}
+ occ memories:places-setup
+
{{
t(
"memories",
@@ -147,6 +263,7 @@
:label-visible="true"
:value="ffmpegPath"
@change="update('ffmpegPath', $event.target.value)"
+ :disabled="!enableTranscoding"
/>
{{ t("memories", "Global default video quality (user may override)") }}
{{ t("memories", "Auto (adaptive transcode)") }}
{{ t("memories", "Original (transcode with max quality)") }}
- {{ t("memories", "external transcoder configuration") }}
+ {{ t("memories", "External Link") }}
@@ -206,6 +327,7 @@
- VA-API configuration
+ {{ t("memories", "External Link") }}
@@ -286,6 +411,7 @@
{{ t("memories", "NPP scaler") }}