memories/lib/Db/TimelineWrite.php

319 lines
11 KiB
PHP
Raw Normal View History

2022-08-20 02:53:21 +00:00
<?php
2022-10-19 17:10:36 +00:00
2022-08-20 02:53:21 +00:00
declare(strict_types=1);
namespace OCA\Memories\Db;
use OCA\Memories\Exif;
use OCA\Memories\Service\Index;
2022-10-19 17:10:36 +00:00
use OCP\DB\QueryBuilder\IQueryBuilder;
2022-08-20 02:53:21 +00:00
use OCP\Files\File;
use OCP\IDBConnection;
use OCP\Lock\ILockingProvider;
2022-08-20 02:53:21 +00:00
require_once __DIR__.'/../ExifFields.php';
2023-02-06 02:34:54 +00:00
const DELETE_TABLES = ['memories', 'memories_livephoto', 'memories_places'];
2023-02-09 08:35:35 +00:00
const TRUNCATE_TABLES = ['memories_mapclusters'];
2023-02-06 02:34:54 +00:00
2022-10-19 17:10:36 +00:00
class TimelineWrite
{
2023-02-09 05:55:12 +00:00
use TimelineWriteMap;
use TimelineWriteOrphans;
use TimelineWritePlaces;
2022-09-09 07:31:42 +00:00
protected IDBConnection $connection;
2022-11-22 11:10:25 +00:00
protected LivePhoto $livePhoto;
protected ILockingProvider $lockingProvider;
2022-08-20 02:53:21 +00:00
public function __construct(
IDBConnection $connection,
LivePhoto $livePhoto,
ILockingProvider $lockingProvider
) {
2022-09-09 07:31:42 +00:00
$this->connection = $connection;
$this->livePhoto = $livePhoto;
$this->lockingProvider = $lockingProvider;
2022-09-09 07:31:42 +00:00
}
2022-08-20 02:53:21 +00:00
/**
2022-10-19 17:10:36 +00:00
* Process a file to insert Exif data into the database.
*
2023-03-07 20:27:25 +00:00
* @param File $file File node to process
* @param bool $lock Lock the file before processing
* @param bool $force Update the record even if the file has not changed
*
* @return bool True if the file was processed
*
* @throws \OCP\Lock\LockedException If the file is locked
* @throws \OCP\DB\Exception If the database query fails
2022-08-20 02:53:21 +00:00
*/
public function processFile(
File $file,
bool $lock = true,
bool $force = false
): bool {
2022-08-20 02:53:21 +00:00
// Check if we want to process this file
if (!Index::isSupported($file)) {
return false;
2022-08-20 02:53:21 +00:00
}
// Check if we need to lock the file
if ($lock) {
$lockKey = '/memories/'.$file->getId();
$lockType = ILockingProvider::LOCK_EXCLUSIVE;
// Throw directly to caller if we can't get the lock
// This way we don't release someone else's lock
$this->lockingProvider->acquireLock($lockKey, $lockType);
try {
return $this->processFile($file, false, $force);
} finally {
$this->lockingProvider->releaseLock($lockKey, $lockType);
}
}
2022-08-20 02:53:21 +00:00
// Get parameters
$mtime = $file->getMtime();
$fileId = $file->getId();
$isvideo = Index::isVideo($file);
2022-08-20 02:53:21 +00:00
// Get previous row
$prevRow = $this->getCurrentRow($fileId);
2023-03-07 20:20:07 +00:00
// Skip if all of the following:
// - not forced
// - the record exists
// - the file has not changed
// - the record is not an orphan
if (!$force && $prevRow && ((int) $prevRow['mtime'] === $mtime) && (!(bool) $prevRow['orphan'])) {
return false;
2022-08-20 02:53:21 +00:00
}
// Get exif data
$exif = Exif::getExifFromFile($file);
// Check if EXIF is blank, which is probably wrong
if (0 === \count($exif)) {
throw new \Exception('No EXIF data could be read: '.$file->getPath());
2022-10-19 17:10:36 +00:00
}
2022-08-20 02:53:21 +00:00
// Hand off if Live Photo video part
2022-11-22 11:10:25 +00:00
if ($isvideo && $this->livePhoto->isVideoPart($exif)) {
$this->livePhoto->processVideoPart($file, $exif);
return true;
}
// Delete video part if it is no longer valid
if ($prevRow && !\array_key_exists('mapcluster', $prevRow)) {
$this->livePhoto->deleteVideoPart($file);
$prevRow = null;
2022-11-22 11:10:25 +00:00
}
2022-11-10 03:48:03 +00:00
// Video parameters
$videoDuration = round((float) ($isvideo ? ($exif['Duration'] ?? $exif['TrackDuration'] ?? 0) : 0));
2022-11-10 03:48:03 +00:00
// Process location data
// This also modifies the exif array in-place to set the LocationTZID
// and drop the GPS data if it is not valid
[$lat, $lon, $mapCluster] = $this->processExifLocation($fileId, $exif, $prevRow);
2022-11-22 10:36:04 +00:00
// Get date parameters (after setting timezone offset)
$dateTaken = Exif::getDateTaken($file, $exif);
2022-11-07 21:13:16 +00:00
// Store the acutal epoch with the EXIF data
$epoch = $exif['DateTimeEpoch'] = $dateTaken->getTimestamp();
2022-11-07 21:13:16 +00:00
// Store the date taken in the database as UTC (local date) only
// Basically, assume everything happens in Greenwich
$dateLocalUtc = Exif::forgetTimezone($dateTaken)->getTimestamp();
$dateTakenStr = gmdate('Y-m-d H:i:s', $dateLocalUtc);
2022-11-07 21:13:16 +00:00
// We need to use the local time in UTC for the dayId
// This way two photos in different timezones on the same date locally
// end up in the same dayId group
$dayId = floor($dateLocalUtc / 86400);
2023-02-09 08:35:35 +00:00
// Get size of image
[$w, $h] = Exif::getDimensions($exif);
2023-02-09 08:35:35 +00:00
// Get live photo ID of video part
$liveid = $this->livePhoto->getLivePhotoId($file, $exif);
2023-02-05 21:43:25 +00:00
// Get BUID from ImageUniqueId if not present
$buid = $prevRow ? $prevRow['buid'] : '';
if (empty($buid)) {
$imageUniqueId = \array_key_exists('ImageUniqueID', $exif) ? $exif['ImageUniqueID'] : null;
$buid = Exif::getBUID($file->getName(), $imageUniqueId, $file->getSize());
}
// Get exif json
$exifJson = $this->getExifJson($exif);
2023-02-10 17:36:20 +00:00
2023-02-08 18:45:25 +00:00
// Parameters for insert or update
$query = $this->connection->getQueryBuilder();
2023-02-08 18:45:25 +00:00
$params = [
'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
'objectid' => $query->createNamedParameter((string) $fileId, IQueryBuilder::PARAM_STR),
'dayid' => $query->createNamedParameter($dayId, IQueryBuilder::PARAM_INT),
'datetaken' => $query->createNamedParameter($dateTakenStr, IQueryBuilder::PARAM_STR),
'epoch' => $query->createNamedParameter($epoch, IQueryBuilder::PARAM_INT),
2023-02-08 18:45:25 +00:00
'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
'isvideo' => $query->createNamedParameter($isvideo, IQueryBuilder::PARAM_INT),
'video_duration' => $query->createNamedParameter($videoDuration, IQueryBuilder::PARAM_INT),
'w' => $query->createNamedParameter($w, IQueryBuilder::PARAM_INT),
'h' => $query->createNamedParameter($h, IQueryBuilder::PARAM_INT),
'exif' => $query->createNamedParameter($exifJson, IQueryBuilder::PARAM_STR),
'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
'lat' => $query->createNamedParameter($lat, IQueryBuilder::PARAM_STR),
'lon' => $query->createNamedParameter($lon, IQueryBuilder::PARAM_STR),
2023-02-09 08:35:35 +00:00
'mapcluster' => $query->createNamedParameter($mapCluster, IQueryBuilder::PARAM_INT),
'orphan' => $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL),
'buid' => $query->createNamedParameter($buid, IQueryBuilder::PARAM_STR),
2023-02-08 18:45:25 +00:00
];
// There is no easy way to UPSERT in standard SQL
// https://stackoverflow.com/questions/15252213/sql-standard-upsert-call
if ($prevRow) {
$query->update('memories')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
;
foreach ($params as $key => $value) {
$query->set($key, $value);
2022-08-22 18:42:06 +00:00
}
} else {
$query->insert('memories')->values($params);
}
return $query->executeStatement() > 0;
2022-08-20 02:53:21 +00:00
}
/**
2022-10-19 17:10:36 +00:00
* Remove a file from the exif database.
2022-08-20 02:53:21 +00:00
*/
public function deleteFile(File &$file): void
2022-10-19 17:10:36 +00:00
{
2023-02-09 16:40:26 +00:00
// Get full record
$query = $this->connection->getQueryBuilder();
$query->select('*')
->from('memories')
->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT)))
;
$record = $query->executeQuery()->fetch();
// Begin transaction
$this->connection->beginTransaction();
2023-02-09 16:40:26 +00:00
// Delete all records regardless of existence
2023-02-06 02:47:10 +00:00
foreach (DELETE_TABLES as $table) {
2022-11-23 18:40:41 +00:00
$query = $this->connection->getQueryBuilder();
$query->delete($table)
->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT)))
;
$query->executeStatement();
2023-02-06 02:34:54 +00:00
}
2023-02-09 16:40:26 +00:00
// Delete from map cluster
if ($record && ($cid = (int) $record['mapcluster']) > 0) {
2023-02-10 17:29:42 +00:00
$this->mapRemoveFromCluster($cid, (float) $record['lat'], (float) $record['lon']);
2023-02-09 16:40:26 +00:00
}
// Commit transaction
$this->connection->commit();
}
/**
* Clean up the table for entries not present in filecache.
*/
public function cleanupStale(): void
{
// Begin transaction
$this->connection->beginTransaction();
// Delete all stale records
foreach (DELETE_TABLES as $table) {
$query = $this->connection->getQueryBuilder();
$clause = $query
->select($query->expr()->literal('1'))
->from('filecache', 'f')
->where($query->expr()->eq('f.fileid', "*PREFIX*{$table}.fileid"))
->getSQL()
;
$query = $this->connection->getQueryBuilder();
$query->delete($table)
->where($query->createFunction("NOT EXISTS({$clause})"))
->executeStatement()
;
}
// Commit transaction
$this->connection->commit();
2022-08-20 02:53:21 +00:00
}
2022-09-11 00:15:40 +00:00
/**
* Clear the entire index. Does not need confirmation!
*/
public function clear(): void
2022-10-19 17:10:36 +00:00
{
2022-11-23 09:49:00 +00:00
$p = $this->connection->getDatabasePlatform();
2023-02-09 08:35:35 +00:00
foreach (array_merge(DELETE_TABLES, TRUNCATE_TABLES) as $table) {
2023-02-06 02:34:54 +00:00
$this->connection->executeStatement($p->getTruncateTableSQL('*PREFIX*'.$table, false));
}
2022-09-11 00:15:40 +00:00
}
/**
* Get the current row for a file_id, from either table.
*/
private function getCurrentRow(int $fileId): ?array
{
$fetch = function (string $table) use ($fileId) {
$query = $this->connection->getQueryBuilder();
return $query->select('*')
->from($table)
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
->executeQuery()
->fetch()
;
};
return $fetch('memories') ?: $fetch('memories_livephoto') ?: null;
}
/**
* Convert EXIF data to filtered JSON string.
*/
private function getExifJson(array $exif): string
{
// Clean up EXIF to keep only useful metadata
$filteredExif = [];
foreach ($exif as $key => $value) {
// Truncate any fields > 2048 chars
if (\is_string($value) && \strlen($value) > 2048) {
$value = substr($value, 0, 2048);
}
// Only keep fields in the whitelist
if (\array_key_exists($key, EXIF_FIELDS_LIST)) {
$filteredExif[$key] = $value;
}
}
// Store JSON string
$exifJson = json_encode($filteredExif);
// Store error if data > 64kb
if (\is_string($exifJson)) {
if (\strlen($exifJson) > 65535) {
$exifJson = json_encode(['error' => 'Exif data too large']);
}
} else {
$exifJson = json_encode(['error' => 'Exif data encoding error']);
}
return $exifJson;
}
2022-10-19 17:10:36 +00:00
}