native: implement AUID

Signed-off-by: Varun Patil <radialapps@gmail.com>

lib/Controller/DaysController.php

@@ -166,6 +166,11 @@ class DaysController extends GenericApiController
             $transforms[] = [$this->timelineQuery, 'transformLimit', (int) $limit];
         }
 
+        // Add extra fields for native callers
+        if (Util::callerIsNative()) {
+            $transforms[] = [$this->timelineQuery, 'transformNativeQuery'];
+        }
+
         return $transforms;
     }
 

lib/Db/TimelineQuery.php

@@ -15,11 +15,15 @@ class TimelineQuery
     use TimelineQueryFolders;
     use TimelineQueryLivePhoto;
     use TimelineQueryMap;
+    use TimelineQueryNativeX;
     use TimelineQuerySingleItem;
 
     public const TIMELINE_SELECT = [
-        'm.isvideo', 'm.video_duration', 'm.datetaken', 'm.dayid', 'm.w', 'm.h', 'm.liveid',
+        'm.datetaken', 'm.dayid',
-        'f.etag', 'f.name AS basename', 'mimetypes.mimetype',
+        'm.w', 'm.h', 'm.liveid',
+        'm.isvideo', 'm.video_duration',
+        'f.etag', 'f.name AS basename',
+        'mimetypes.mimetype',
     ];
 
     protected IDBConnection $connection;

lib/Db/TimelineQueryDays.php

@@ -5,7 +5,7 @@ declare(strict_types=1);
 namespace OCA\Memories\Db;
 
 use OCA\Memories\ClustersBackend;
-use OCA\Memories\Util;
+use OCA\Memories\Exif;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\IDBConnection;
 
@@ -221,11 +221,14 @@ trait TimelineQueryDays
         // All cluster transformations
         ClustersBackend\Manager::applyDayPostTransforms($this->request, $row);
 
-        // Remove datetaken unless native (for sorting)
+        // This field is only required due to the GROUP BY clause
-        if (Util::callerIsNative()) {
+        unset($row['datetaken']);
-            $row['datetaken'] = Util::sqlUtcToTimestamp($row['datetaken']);
+
-        } else {
+        // Calculate the AUID if we can
-            unset($row['datetaken']);
+        if (\array_key_exists('epoch', $row) && \array_key_exists('size', $row)
+           && ($epoch = (int) $row['epoch']) && ($size = (int) $row['size'])) {
+            // compute AUID and discard epoch and size
+            $row['auid'] = Exif::getAUID($epoch, $size);
         }
     }
 

lib/Db/TimelineQueryNativeX.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\Memories\Db;
+
+use OCP\DB\QueryBuilder\IQueryBuilder;
+
+trait TimelineQueryNativeX
+{
+    public function transformNativeQuery(IQueryBuilder &$query, bool $aggregate)
+    {
+        if (!$aggregate) {
+            $query->addSelect('m.epoch', 'f.size');
+        }
+    }
+}

lib/Db/TimelineWrite.php

@@ -122,7 +122,7 @@ class TimelineWrite
         $dateTaken = Exif::getDateTaken($file, $exif);
 
         // Store the acutal epoch with the EXIF data
-        $exif['DateTimeEpoch'] = $dateTaken->getTimestamp();
+        $epoch = $exif['DateTimeEpoch'] = $dateTaken->getTimestamp();
 
         // Store the date taken in the database as UTC (local date) only
         // Basically, assume everything happens in Greenwich
@@ -150,6 +150,7 @@ class TimelineWrite
             'objectid' => $query->createNamedParameter((string) $fileId, IQueryBuilder::PARAM_STR),
             'dayid' => $query->createNamedParameter($dayId, IQueryBuilder::PARAM_INT),
             'datetaken' => $query->createNamedParameter($dateTakenStr, IQueryBuilder::PARAM_STR),
+            'epoch' => $query->createNamedParameter($epoch, IQueryBuilder::PARAM_INT),
             'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
             'isvideo' => $query->createNamedParameter($isvideo, IQueryBuilder::PARAM_INT),
             'video_duration' => $query->createNamedParameter($videoDuration, IQueryBuilder::PARAM_INT),

lib/Exif.php

@@ -248,6 +248,17 @@ class Exif
         return [$width, $height];
     }
 
+    /**
+     * Get the Approximate Unique ID (AUID) from parameters.
+     *
+     * @param int $epoch the date taken as a unix timestamp (seconds)
+     * @param int $size  the file size in bytes
+     */
+    public static function getAUID(int $epoch, int $size): int
+    {
+        return crc32($epoch.$size);
+    }
+
     /**
      * Get the list of MIME Types that are allowed to be edited.
      */

lib/Migration/Version505000Date20230821044807.php

@@ -0,0 +1,128 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Varun Patil <radialapps@gmail.com>
+ * @author Varun Patil <radialapps@gmail.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Migration;
+
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\IDBConnection;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version505000Date20230821044807 extends SimpleMigrationStep
+{
+    /** @var IDBConnection */
+    private $dbc;
+
+    public function __construct(IDBConnection $dbc)
+    {
+        $this->dbc = $dbc;
+    }
+
+    /**
+     * @param \Closure(): ISchemaWrapper $schemaClosure
+     */
+    public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+    }
+
+    /**
+     * @param \Closure(): ISchemaWrapper $schemaClosure
+     */
+    public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
+    {
+        /** @var ISchemaWrapper $schema */
+        $schema = $schemaClosure();
+
+        $table = $schema->getTable('memories');
+        $table->addColumn('epoch', Types::BIGINT, [
+            'notnull' => true,
+            'default' => 0,
+        ]);
+
+        return $schema;
+    }
+
+    /**
+     * @param \Closure(): ISchemaWrapper $schemaClosure
+     */
+    public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+        // extracts the epoch value from the EXIF json and stores it in the epoch column
+        try {
+            // get the required records
+            $result = $this->dbc->getQueryBuilder()
+                ->select('m.id', 'm.exif')
+                ->from('memories', 'm')
+                ->executeQuery()
+            ;
+            $count = 0;
+
+            // iterate the memories table and update the epoch column
+            $this->dbc->beginTransaction();
+            while ($row = $result->fetch()) {
+                try {
+                    // get the epoch from the exif data
+                    $exif = json_decode($row['exif'], true);
+                    if (!\is_array($exif) || !\array_key_exists('DateTimeEpoch', $exif)) {
+                        continue;
+                    }
+
+                    // get epoch from exif if available
+                    if ($epoch = (int) $exif['DateTimeEpoch']) {
+                        // update the epoch column
+                        $query = $this->dbc->getQueryBuilder();
+                        $query->update('memories')
+                            ->set('epoch', $query->createNamedParameter($epoch))
+                            ->where($query->expr()->eq('id', $query->createNamedParameter((int) $row['id'], \PDO::PARAM_INT)))
+                            ->executeStatement()
+                        ;
+
+                        // increment the counter
+                        ++$count;
+                    }
+                } catch (\Exception $e) {
+                    continue;
+                }
+
+                // commit every 50 rows
+                if (0 === $count % 50) {
+                    $this->dbc->commit();
+                    $this->dbc->beginTransaction();
+                }
+            }
+
+            // commit the remaining rows
+            $this->dbc->commit();
+
+            // close the cursor
+            $result->closeCursor();
+        } catch (\Exception $e) {
+            error_log('Automatic migration failed: '.$e->getMessage());
+            error_log('Please run occ memories:index -f');
+        }
+    }
+}

src/native.ts

@@ -293,13 +293,15 @@ export async function extendDayWithLocal(dayId: number, photos: IPhoto[]) {
 
   // Merge local photos into remote photos
   const localPhotos: IPhoto[] = await res.json();
-  const photosSet = new Set(photos.map((p) => p.basename));
+  const serverAUIDs = new Set(photos.map((p) => p.auid));
-  const localOnly = localPhotos.filter((p) => !photosSet.has(p.basename));
+
+  // Filter out files that are only available locally
+  const localOnly = localPhotos.filter((p) => !serverAUIDs.has(p.auid));
   localOnly.forEach((p) => (p.islocal = true));
   photos.push(...localOnly);
 
-  // Sort by datetaken
+  // Sort by epoch value
-  photos.sort((a, b) => (b.datetaken ?? 0) - (a.datetaken ?? 0));
+  photos.sort((a, b) => (b.epoch ?? 0) - (a.epoch ?? 0));
 }
 
 /**

src/types.ts

@@ -78,7 +78,15 @@ export type IPhoto = {
   isfavorite?: boolean;
   /** Local file from native */
   islocal?: boolean;
-  /** Optional datetaken epoch */
+
+  /** AUID of file (optional, NativeX) */
+  auid?: number;
+  /** Epoch of file (optional, NativeX) */
+  epoch?: number;
+  /** File size (optional) */
+  size?: number;
+
+  /** Date taken UTC value (lazy fetched) */
   datetaken?: number;
 };