native: implement AUID

Signed-off-by: Varun Patil <radialapps@gmail.com>
pull/783/head
Varun Patil 2023-08-20 23:07:07 -07:00
parent 25761f119b
commit 290c550f2f
9 changed files with 193 additions and 14 deletions

View File

@ -166,6 +166,11 @@ class DaysController extends GenericApiController
$transforms[] = [$this->timelineQuery, 'transformLimit', (int) $limit]; $transforms[] = [$this->timelineQuery, 'transformLimit', (int) $limit];
} }
// Add extra fields for native callers
if (Util::callerIsNative()) {
$transforms[] = [$this->timelineQuery, 'transformNativeQuery'];
}
return $transforms; return $transforms;
} }

View File

@ -15,11 +15,15 @@ class TimelineQuery
use TimelineQueryFolders; use TimelineQueryFolders;
use TimelineQueryLivePhoto; use TimelineQueryLivePhoto;
use TimelineQueryMap; use TimelineQueryMap;
use TimelineQueryNativeX;
use TimelineQuerySingleItem; use TimelineQuerySingleItem;
public const TIMELINE_SELECT = [ public const TIMELINE_SELECT = [
'm.isvideo', 'm.video_duration', 'm.datetaken', 'm.dayid', 'm.w', 'm.h', 'm.liveid', 'm.datetaken', 'm.dayid',
'f.etag', 'f.name AS basename', 'mimetypes.mimetype', 'm.w', 'm.h', 'm.liveid',
'm.isvideo', 'm.video_duration',
'f.etag', 'f.name AS basename',
'mimetypes.mimetype',
]; ];
protected IDBConnection $connection; protected IDBConnection $connection;

View File

@ -5,7 +5,7 @@ declare(strict_types=1);
namespace OCA\Memories\Db; namespace OCA\Memories\Db;
use OCA\Memories\ClustersBackend; use OCA\Memories\ClustersBackend;
use OCA\Memories\Util; use OCA\Memories\Exif;
use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection; use OCP\IDBConnection;
@ -221,11 +221,14 @@ trait TimelineQueryDays
// All cluster transformations // All cluster transformations
ClustersBackend\Manager::applyDayPostTransforms($this->request, $row); ClustersBackend\Manager::applyDayPostTransforms($this->request, $row);
// Remove datetaken unless native (for sorting) // This field is only required due to the GROUP BY clause
if (Util::callerIsNative()) {
$row['datetaken'] = Util::sqlUtcToTimestamp($row['datetaken']);
} else {
unset($row['datetaken']); unset($row['datetaken']);
// Calculate the AUID if we can
if (\array_key_exists('epoch', $row) && \array_key_exists('size', $row)
&& ($epoch = (int) $row['epoch']) && ($size = (int) $row['size'])) {
// compute AUID and discard epoch and size
$row['auid'] = Exif::getAUID($epoch, $size);
} }
} }

View File

@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace OCA\Memories\Db;
use OCP\DB\QueryBuilder\IQueryBuilder;
trait TimelineQueryNativeX
{
public function transformNativeQuery(IQueryBuilder &$query, bool $aggregate)
{
if (!$aggregate) {
$query->addSelect('m.epoch', 'f.size');
}
}
}

View File

@ -122,7 +122,7 @@ class TimelineWrite
$dateTaken = Exif::getDateTaken($file, $exif); $dateTaken = Exif::getDateTaken($file, $exif);
// Store the acutal epoch with the EXIF data // Store the acutal epoch with the EXIF data
$exif['DateTimeEpoch'] = $dateTaken->getTimestamp(); $epoch = $exif['DateTimeEpoch'] = $dateTaken->getTimestamp();
// Store the date taken in the database as UTC (local date) only // Store the date taken in the database as UTC (local date) only
// Basically, assume everything happens in Greenwich // Basically, assume everything happens in Greenwich
@ -150,6 +150,7 @@ class TimelineWrite
'objectid' => $query->createNamedParameter((string) $fileId, IQueryBuilder::PARAM_STR), 'objectid' => $query->createNamedParameter((string) $fileId, IQueryBuilder::PARAM_STR),
'dayid' => $query->createNamedParameter($dayId, IQueryBuilder::PARAM_INT), 'dayid' => $query->createNamedParameter($dayId, IQueryBuilder::PARAM_INT),
'datetaken' => $query->createNamedParameter($dateTakenStr, IQueryBuilder::PARAM_STR), 'datetaken' => $query->createNamedParameter($dateTakenStr, IQueryBuilder::PARAM_STR),
'epoch' => $query->createNamedParameter($epoch, IQueryBuilder::PARAM_INT),
'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT), 'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
'isvideo' => $query->createNamedParameter($isvideo, IQueryBuilder::PARAM_INT), 'isvideo' => $query->createNamedParameter($isvideo, IQueryBuilder::PARAM_INT),
'video_duration' => $query->createNamedParameter($videoDuration, IQueryBuilder::PARAM_INT), 'video_duration' => $query->createNamedParameter($videoDuration, IQueryBuilder::PARAM_INT),

View File

@ -248,6 +248,17 @@ class Exif
return [$width, $height]; return [$width, $height];
} }
/**
* Get the Approximate Unique ID (AUID) from parameters.
*
* @param int $epoch the date taken as a unix timestamp (seconds)
* @param int $size the file size in bytes
*/
public static function getAUID(int $epoch, int $size): int
{
return crc32($epoch.$size);
}
/** /**
* Get the list of MIME Types that are allowed to be edited. * Get the list of MIME Types that are allowed to be edited.
*/ */

View File

@ -0,0 +1,128 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Varun Patil <radialapps@gmail.com>
* @author Varun Patil <radialapps@gmail.com>
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Migration;
use OCP\DB\ISchemaWrapper;
use OCP\DB\Types;
use OCP\IDBConnection;
use OCP\Migration\IOutput;
use OCP\Migration\SimpleMigrationStep;
/**
* Auto-generated migration step: Please modify to your needs!
*/
class Version505000Date20230821044807 extends SimpleMigrationStep
{
/** @var IDBConnection */
private $dbc;
public function __construct(IDBConnection $dbc)
{
$this->dbc = $dbc;
}
/**
* @param \Closure(): ISchemaWrapper $schemaClosure
*/
public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
{
}
/**
* @param \Closure(): ISchemaWrapper $schemaClosure
*/
public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
{
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
$table = $schema->getTable('memories');
$table->addColumn('epoch', Types::BIGINT, [
'notnull' => true,
'default' => 0,
]);
return $schema;
}
/**
* @param \Closure(): ISchemaWrapper $schemaClosure
*/
public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
{
// extracts the epoch value from the EXIF json and stores it in the epoch column
try {
// get the required records
$result = $this->dbc->getQueryBuilder()
->select('m.id', 'm.exif')
->from('memories', 'm')
->executeQuery()
;
$count = 0;
// iterate the memories table and update the epoch column
$this->dbc->beginTransaction();
while ($row = $result->fetch()) {
try {
// get the epoch from the exif data
$exif = json_decode($row['exif'], true);
if (!\is_array($exif) || !\array_key_exists('DateTimeEpoch', $exif)) {
continue;
}
// get epoch from exif if available
if ($epoch = (int) $exif['DateTimeEpoch']) {
// update the epoch column
$query = $this->dbc->getQueryBuilder();
$query->update('memories')
->set('epoch', $query->createNamedParameter($epoch))
->where($query->expr()->eq('id', $query->createNamedParameter((int) $row['id'], \PDO::PARAM_INT)))
->executeStatement()
;
// increment the counter
++$count;
}
} catch (\Exception $e) {
continue;
}
// commit every 50 rows
if (0 === $count % 50) {
$this->dbc->commit();
$this->dbc->beginTransaction();
}
}
// commit the remaining rows
$this->dbc->commit();
// close the cursor
$result->closeCursor();
} catch (\Exception $e) {
error_log('Automatic migration failed: '.$e->getMessage());
error_log('Please run occ memories:index -f');
}
}
}

View File

@ -293,13 +293,15 @@ export async function extendDayWithLocal(dayId: number, photos: IPhoto[]) {
// Merge local photos into remote photos // Merge local photos into remote photos
const localPhotos: IPhoto[] = await res.json(); const localPhotos: IPhoto[] = await res.json();
const photosSet = new Set(photos.map((p) => p.basename)); const serverAUIDs = new Set(photos.map((p) => p.auid));
const localOnly = localPhotos.filter((p) => !photosSet.has(p.basename));
// Filter out files that are only available locally
const localOnly = localPhotos.filter((p) => !serverAUIDs.has(p.auid));
localOnly.forEach((p) => (p.islocal = true)); localOnly.forEach((p) => (p.islocal = true));
photos.push(...localOnly); photos.push(...localOnly);
// Sort by datetaken // Sort by epoch value
photos.sort((a, b) => (b.datetaken ?? 0) - (a.datetaken ?? 0)); photos.sort((a, b) => (b.epoch ?? 0) - (a.epoch ?? 0));
} }
/** /**

View File

@ -78,7 +78,15 @@ export type IPhoto = {
isfavorite?: boolean; isfavorite?: boolean;
/** Local file from native */ /** Local file from native */
islocal?: boolean; islocal?: boolean;
/** Optional datetaken epoch */
/** AUID of file (optional, NativeX) */
auid?: number;
/** Epoch of file (optional, NativeX) */
epoch?: number;
/** File size (optional) */
size?: number;
/** Date taken UTC value (lazy fetched) */
datetaken?: number; datetaken?: number;
}; };