From 290c550f2f3482fe86ed597f8be3324b84b2862f Mon Sep 17 00:00:00 2001 From: Varun Patil Date: Sun, 20 Aug 2023 23:07:07 -0700 Subject: [PATCH] native: implement AUID Signed-off-by: Varun Patil --- lib/Controller/DaysController.php | 5 + lib/Db/TimelineQuery.php | 8 +- lib/Db/TimelineQueryDays.php | 15 +- lib/Db/TimelineQueryNativeX.php | 17 +++ lib/Db/TimelineWrite.php | 3 +- lib/Exif.php | 11 ++ .../Version505000Date20230821044807.php | 128 ++++++++++++++++++ src/native.ts | 10 +- src/types.ts | 10 +- 9 files changed, 193 insertions(+), 14 deletions(-) create mode 100644 lib/Db/TimelineQueryNativeX.php create mode 100644 lib/Migration/Version505000Date20230821044807.php diff --git a/lib/Controller/DaysController.php b/lib/Controller/DaysController.php index 45d81306..c0ec0360 100644 --- a/lib/Controller/DaysController.php +++ b/lib/Controller/DaysController.php @@ -166,6 +166,11 @@ class DaysController extends GenericApiController $transforms[] = [$this->timelineQuery, 'transformLimit', (int) $limit]; } + // Add extra fields for native callers + if (Util::callerIsNative()) { + $transforms[] = [$this->timelineQuery, 'transformNativeQuery']; + } + return $transforms; } diff --git a/lib/Db/TimelineQuery.php b/lib/Db/TimelineQuery.php index f94b92b2..13f98f0e 100644 --- a/lib/Db/TimelineQuery.php +++ b/lib/Db/TimelineQuery.php @@ -15,11 +15,15 @@ class TimelineQuery use TimelineQueryFolders; use TimelineQueryLivePhoto; use TimelineQueryMap; + use TimelineQueryNativeX; use TimelineQuerySingleItem; public const TIMELINE_SELECT = [ - 'm.isvideo', 'm.video_duration', 'm.datetaken', 'm.dayid', 'm.w', 'm.h', 'm.liveid', - 'f.etag', 'f.name AS basename', 'mimetypes.mimetype', + 'm.datetaken', 'm.dayid', + 'm.w', 'm.h', 'm.liveid', + 'm.isvideo', 'm.video_duration', + 'f.etag', 'f.name AS basename', + 'mimetypes.mimetype', ]; protected IDBConnection $connection; diff --git a/lib/Db/TimelineQueryDays.php b/lib/Db/TimelineQueryDays.php index 462d306b..df8f474d 100644 --- a/lib/Db/TimelineQueryDays.php +++ b/lib/Db/TimelineQueryDays.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace OCA\Memories\Db; use OCA\Memories\ClustersBackend; -use OCA\Memories\Util; +use OCA\Memories\Exif; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IDBConnection; @@ -221,11 +221,14 @@ trait TimelineQueryDays // All cluster transformations ClustersBackend\Manager::applyDayPostTransforms($this->request, $row); - // Remove datetaken unless native (for sorting) - if (Util::callerIsNative()) { - $row['datetaken'] = Util::sqlUtcToTimestamp($row['datetaken']); - } else { - unset($row['datetaken']); + // This field is only required due to the GROUP BY clause + unset($row['datetaken']); + + // Calculate the AUID if we can + if (\array_key_exists('epoch', $row) && \array_key_exists('size', $row) + && ($epoch = (int) $row['epoch']) && ($size = (int) $row['size'])) { + // compute AUID and discard epoch and size + $row['auid'] = Exif::getAUID($epoch, $size); } } diff --git a/lib/Db/TimelineQueryNativeX.php b/lib/Db/TimelineQueryNativeX.php new file mode 100644 index 00000000..68378b0d --- /dev/null +++ b/lib/Db/TimelineQueryNativeX.php @@ -0,0 +1,17 @@ +addSelect('m.epoch', 'f.size'); + } + } +} diff --git a/lib/Db/TimelineWrite.php b/lib/Db/TimelineWrite.php index 4b262c89..9cab29bd 100644 --- a/lib/Db/TimelineWrite.php +++ b/lib/Db/TimelineWrite.php @@ -122,7 +122,7 @@ class TimelineWrite $dateTaken = Exif::getDateTaken($file, $exif); // Store the acutal epoch with the EXIF data - $exif['DateTimeEpoch'] = $dateTaken->getTimestamp(); + $epoch = $exif['DateTimeEpoch'] = $dateTaken->getTimestamp(); // Store the date taken in the database as UTC (local date) only // Basically, assume everything happens in Greenwich @@ -150,6 +150,7 @@ class TimelineWrite 'objectid' => $query->createNamedParameter((string) $fileId, IQueryBuilder::PARAM_STR), 'dayid' => $query->createNamedParameter($dayId, IQueryBuilder::PARAM_INT), 'datetaken' => $query->createNamedParameter($dateTakenStr, IQueryBuilder::PARAM_STR), + 'epoch' => $query->createNamedParameter($epoch, IQueryBuilder::PARAM_INT), 'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT), 'isvideo' => $query->createNamedParameter($isvideo, IQueryBuilder::PARAM_INT), 'video_duration' => $query->createNamedParameter($videoDuration, IQueryBuilder::PARAM_INT), diff --git a/lib/Exif.php b/lib/Exif.php index 956f9891..93e84732 100644 --- a/lib/Exif.php +++ b/lib/Exif.php @@ -248,6 +248,17 @@ class Exif return [$width, $height]; } + /** + * Get the Approximate Unique ID (AUID) from parameters. + * + * @param int $epoch the date taken as a unix timestamp (seconds) + * @param int $size the file size in bytes + */ + public static function getAUID(int $epoch, int $size): int + { + return crc32($epoch.$size); + } + /** * Get the list of MIME Types that are allowed to be edited. */ diff --git a/lib/Migration/Version505000Date20230821044807.php b/lib/Migration/Version505000Date20230821044807.php new file mode 100644 index 00000000..c1578fa9 --- /dev/null +++ b/lib/Migration/Version505000Date20230821044807.php @@ -0,0 +1,128 @@ + + * @author Varun Patil + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCA\Memories\Migration; + +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +/** + * Auto-generated migration step: Please modify to your needs! + */ +class Version505000Date20230821044807 extends SimpleMigrationStep +{ + /** @var IDBConnection */ + private $dbc; + + public function __construct(IDBConnection $dbc) + { + $this->dbc = $dbc; + } + + /** + * @param \Closure(): ISchemaWrapper $schemaClosure + */ + public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void + { + } + + /** + * @param \Closure(): ISchemaWrapper $schemaClosure + */ + public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper + { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + $table = $schema->getTable('memories'); + $table->addColumn('epoch', Types::BIGINT, [ + 'notnull' => true, + 'default' => 0, + ]); + + return $schema; + } + + /** + * @param \Closure(): ISchemaWrapper $schemaClosure + */ + public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void + { + // extracts the epoch value from the EXIF json and stores it in the epoch column + try { + // get the required records + $result = $this->dbc->getQueryBuilder() + ->select('m.id', 'm.exif') + ->from('memories', 'm') + ->executeQuery() + ; + $count = 0; + + // iterate the memories table and update the epoch column + $this->dbc->beginTransaction(); + while ($row = $result->fetch()) { + try { + // get the epoch from the exif data + $exif = json_decode($row['exif'], true); + if (!\is_array($exif) || !\array_key_exists('DateTimeEpoch', $exif)) { + continue; + } + + // get epoch from exif if available + if ($epoch = (int) $exif['DateTimeEpoch']) { + // update the epoch column + $query = $this->dbc->getQueryBuilder(); + $query->update('memories') + ->set('epoch', $query->createNamedParameter($epoch)) + ->where($query->expr()->eq('id', $query->createNamedParameter((int) $row['id'], \PDO::PARAM_INT))) + ->executeStatement() + ; + + // increment the counter + ++$count; + } + } catch (\Exception $e) { + continue; + } + + // commit every 50 rows + if (0 === $count % 50) { + $this->dbc->commit(); + $this->dbc->beginTransaction(); + } + } + + // commit the remaining rows + $this->dbc->commit(); + + // close the cursor + $result->closeCursor(); + } catch (\Exception $e) { + error_log('Automatic migration failed: '.$e->getMessage()); + error_log('Please run occ memories:index -f'); + } + } +} diff --git a/src/native.ts b/src/native.ts index 2aace6a0..9cb638a6 100644 --- a/src/native.ts +++ b/src/native.ts @@ -293,13 +293,15 @@ export async function extendDayWithLocal(dayId: number, photos: IPhoto[]) { // Merge local photos into remote photos const localPhotos: IPhoto[] = await res.json(); - const photosSet = new Set(photos.map((p) => p.basename)); - const localOnly = localPhotos.filter((p) => !photosSet.has(p.basename)); + const serverAUIDs = new Set(photos.map((p) => p.auid)); + + // Filter out files that are only available locally + const localOnly = localPhotos.filter((p) => !serverAUIDs.has(p.auid)); localOnly.forEach((p) => (p.islocal = true)); photos.push(...localOnly); - // Sort by datetaken - photos.sort((a, b) => (b.datetaken ?? 0) - (a.datetaken ?? 0)); + // Sort by epoch value + photos.sort((a, b) => (b.epoch ?? 0) - (a.epoch ?? 0)); } /** diff --git a/src/types.ts b/src/types.ts index dc3cb513..8f5f61ff 100644 --- a/src/types.ts +++ b/src/types.ts @@ -78,7 +78,15 @@ export type IPhoto = { isfavorite?: boolean; /** Local file from native */ islocal?: boolean; - /** Optional datetaken epoch */ + + /** AUID of file (optional, NativeX) */ + auid?: number; + /** Epoch of file (optional, NativeX) */ + epoch?: number; + /** File size (optional) */ + size?: number; + + /** Date taken UTC value (lazy fetched) */ datetaken?: number; };