From 526559b67250e09397589a88960959a87ed2c1fb Mon Sep 17 00:00:00 2001 From: Varun Patil Date: Sun, 16 Apr 2023 19:53:06 -0700 Subject: [PATCH] place-setup: recalculate places Signed-off-by: Varun Patil --- lib/Command/PlacesSetup.php | 8 ++- lib/Controller/OtherController.php | 4 +- lib/Db/TimelineWriteOrphans.php | 67 +++++++++++++++++++-- lib/Db/TimelineWritePlaces.php | 96 +++++++++++++++--------------- lib/Service/Places.php | 43 ++++++++++++- src/Admin.vue | 20 +++++-- 6 files changed, 173 insertions(+), 65 deletions(-) diff --git a/lib/Command/PlacesSetup.php b/lib/Command/PlacesSetup.php index 225f3715..d96a1707 100644 --- a/lib/Command/PlacesSetup.php +++ b/lib/Command/PlacesSetup.php @@ -88,10 +88,16 @@ class PlacesSetup extends Command } // Download the planet database - $this->output->writeln('Downloading planet database'); $datafile = $this->places->downloadPlanet(); // Import the planet database $this->places->importPlanet($datafile); + + // Recalculate all places + $this->places->recalculateAll(); + + $this->output->writeln('Done'); + + return 0; } } diff --git a/lib/Controller/OtherController.php b/lib/Controller/OtherController.php index 1038c9cc..d7ab42ac 100644 --- a/lib/Controller/OtherController.php +++ b/lib/Controller/OtherController.php @@ -205,11 +205,9 @@ class OtherController extends GenericApiController header('Content-Length: 0'); $places = \OC::$server->get(\OCA\Memories\Service\Places::class); - - echo "Downloading planet file...\n"; - flush(); $datafile = $places->downloadPlanet(); $places->importPlanet($datafile); + $places->recalculateAll(); echo "Done.\n"; } catch (\Exception $e) { diff --git a/lib/Db/TimelineWriteOrphans.php b/lib/Db/TimelineWriteOrphans.php index 69b6ec59..f6828590 100644 --- a/lib/Db/TimelineWriteOrphans.php +++ b/lib/Db/TimelineWriteOrphans.php @@ -12,21 +12,78 @@ trait TimelineWriteOrphans protected IDBConnection $connection; /** - * Mark all files in the table as orphaned. + * Mark all or some files in the table as (un)orphaned. + * + * @param bool $value True to mark as orphaned, false to mark as un-orphaned + * @param int[] $fileIds List of file IDs to mark, or empty to mark all files + * @param bool $onlyMain Only mark the main file, not the live photo * * @return int Number of rows affected */ - public function orphanAll(): int + public function orphanAll(bool $value = true, ?array $fileIds = null, bool $onlyMain = false): int { - $do = function (string $table) { + $do = function (string $table) use ($value, $fileIds) { $query = $this->connection->getQueryBuilder(); $query->update($table) - ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)) + ->set('orphan', $query->createNamedParameter($value, IQueryBuilder::PARAM_BOOL)) ; + if ($fileIds) { + $query->where($query->expr()->in('fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + return $query->executeStatement(); }; - return $do('memories') + $do('memories_livephoto'); + $count = $do('memories'); + + if ($onlyMain) { + return $count; + } + + return $count + $do('memories_livephoto'); + } + + /** + * Orphan and run an update on all files. + * + * @param array $fields list of fields to select + * @param int $txnSize number of rows to process in a single transaction + * @param \Closure $callback will be passed each row + */ + public function orphanAndRun(array $fields, int $txnSize, \Closure $callback) + { + // Orphan all files. This means if we are interrupted, + // it will lead to a re-index of the whole library! + $this->orphanAll(true, null, true); + + while (\count($orphans = $this->getSomeOrphans($txnSize, $fields))) { + $this->connection->beginTransaction(); + + foreach ($orphans as $row) { + $callback($row); + } + + // Mark all files as not orphaned. + $fileIds = array_map(fn ($row) => $row['fileid'], $orphans); + $this->orphanAll(false, $fileIds, true); + + $this->connection->commit(); + } + } + + /** + * Get a list of orphaned files. + */ + protected function getSomeOrphans(int $count, array $fields): array + { + $query = $this->connection->getQueryBuilder(); + $query->select(...$fields) + ->from('memories') + ->where($query->expr()->eq('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))) + ->setMaxResults($count) + ; + + return $query->executeQuery()->fetchAll(); } } diff --git a/lib/Db/TimelineWritePlaces.php b/lib/Db/TimelineWritePlaces.php index a941e2b1..ba0e35f3 100644 --- a/lib/Db/TimelineWritePlaces.php +++ b/lib/Db/TimelineWritePlaces.php @@ -15,53 +15,6 @@ trait TimelineWritePlaces { protected IDBConnection $connection; - /** - * Process the location part of exif data. - * - * Also update the exif data with the tzid from location (LocationTZID) - * Performs an in-place update of the exif data. - * - * @param int $fileId The file ID - * @param array $exif The exif data (will change) - * @param array|bool $prevRow The previous row of data - * - * @return array Update values - */ - protected function processExifLocation(int $fileId, array &$exif, $prevRow): array - { - // Store location data - [$lat, $lon] = self::readCoord($exif); - $oldLat = $prevRow ? (float) $prevRow['lat'] : null; - $oldLon = $prevRow ? (float) $prevRow['lon'] : null; - $mapCluster = $prevRow ? (int) $prevRow['mapcluster'] : -1; - $osmIds = []; - - if ($lat || $lon || $oldLat || $oldLon) { - try { - $mapCluster = $this->mapGetCluster($mapCluster, $lat, $lon, $oldLat, $oldLon); - } catch (\Exception $e) { - $logger = \OC::$server->get(LoggerInterface::class); - $logger->log(3, 'Error updating map cluster data: '.$e->getMessage(), ['app' => 'memories']); - } - - try { - $osmIds = $this->updatePlacesData($fileId, $lat, $lon); - } catch (\Exception $e) { - $logger = \OC::$server->get(LoggerInterface::class); - $logger->log(3, 'Error updating places data: '.$e->getMessage(), ['app' => 'memories']); - } - } - - // NULL if invalid - $mapCluster = $mapCluster <= 0 ? null : $mapCluster; - - // Set tzid from location if not present - $this->setTzidFromLocation($exif, $osmIds); - - // Return update values - return [$lat, $lon, $mapCluster, $osmIds]; - } - /** * Add places data for a file. * @@ -71,7 +24,7 @@ trait TimelineWritePlaces * * @return array The list of osm_id of the places */ - protected function updatePlacesData(int $fileId, $lat, $lon): array + public function updatePlacesData(int $fileId, $lat, $lon): array { // Get GIS type $gisType = \OCA\Memories\Util::placesGISType(); @@ -140,6 +93,53 @@ trait TimelineWritePlaces return array_map(fn ($row) => $row['osm_id'], $rows); } + /** + * Process the location part of exif data. + * + * Also update the exif data with the tzid from location (LocationTZID) + * Performs an in-place update of the exif data. + * + * @param int $fileId The file ID + * @param array $exif The exif data (will change) + * @param array|bool $prevRow The previous row of data + * + * @return array Update values + */ + protected function processExifLocation(int $fileId, array &$exif, $prevRow): array + { + // Store location data + [$lat, $lon] = self::readCoord($exif); + $oldLat = $prevRow ? (float) $prevRow['lat'] : null; + $oldLon = $prevRow ? (float) $prevRow['lon'] : null; + $mapCluster = $prevRow ? (int) $prevRow['mapcluster'] : -1; + $osmIds = []; + + if ($lat || $lon || $oldLat || $oldLon) { + try { + $mapCluster = $this->mapGetCluster($mapCluster, $lat, $lon, $oldLat, $oldLon); + } catch (\Exception $e) { + $logger = \OC::$server->get(LoggerInterface::class); + $logger->log(3, 'Error updating map cluster data: '.$e->getMessage(), ['app' => 'memories']); + } + + try { + $osmIds = $this->updatePlacesData($fileId, $lat, $lon); + } catch (\Exception $e) { + $logger = \OC::$server->get(LoggerInterface::class); + $logger->log(3, 'Error updating places data: '.$e->getMessage(), ['app' => 'memories']); + } + } + + // NULL if invalid + $mapCluster = $mapCluster <= 0 ? null : $mapCluster; + + // Set tzid from location if not present + $this->setTzidFromLocation($exif, $osmIds); + + // Return update values + return [$lat, $lon, $mapCluster, $osmIds]; + } + /** * Set timezone offset from location if not present. * diff --git a/lib/Service/Places.php b/lib/Service/Places.php index a097f39d..48478341 100644 --- a/lib/Service/Places.php +++ b/lib/Service/Places.php @@ -2,6 +2,7 @@ namespace OCA\Memories\Service; +use OCA\Memories\Db\TimelineWrite; use OCP\IConfig; use OCP\IDBConnection; @@ -18,13 +19,16 @@ class Places protected IConfig $config; protected IDBConnection $connection; + protected TimelineWrite $timelineWrite; public function __construct( IConfig $config, - IDBConnection $connection + IDBConnection $connection, + TimelineWrite $timelineWrite ) { $this->config = $config; $this->connection = $connection; + $this->timelineWrite = $timelineWrite; } /** @@ -89,6 +93,9 @@ class Places */ public function downloadPlanet(): string { + echo "Download planet data to temporary file...\n"; + flush(); + $filename = sys_get_temp_dir().'/planet_coarse_boundaries.zip'; unlink($filename); @@ -134,6 +141,7 @@ class Places public function importPlanet(string $datafile): void { echo "Inserting planet data into database...\n"; + flush(); // Detect the GIS type $gis = $this->detectGisType(); @@ -317,16 +325,47 @@ class Places // Mark success echo "Planet database imported successfully!\n"; echo "You should re-index your library now.\n"; + flush(); $this->config->setSystemValue('memories.gis_type', $gis); // Delete data file unlink($datafile); } + /** + * Recalculate all places for all users. + */ + public function recalculateAll() + { + echo "Recalculating places for all files (do not interrupt this process)...\n"; + flush(); + + $count = 0; + $this->timelineWrite->orphanAndRun(['fileid', 'lat', 'lon'], 20, function (array $row) use (&$count) { + ++$count; + + // Only proceed if we have a valid location + $fileid = $row['fileid']; + $lat = (float) $row['lat']; + $lon = (float) $row['lon']; + + // Update places + if ($lat || $lon) { + $this->timelineWrite->updatePlacesData($fileid, $lat, $lon); + } + + // Print every 500 files + if (0 === $count % 500) { + echo "Updated places data for {$count} files\n"; + flush(); + } + }); + } + /** * Create database tables and indices. */ - private function setupDatabase(): void + protected function setupDatabase(): void { try { // Get Gis type diff --git a/src/Admin.vue b/src/Admin.vue index b7ef517d..7e60058f 100644 --- a/src/Admin.vue +++ b/src/Admin.vue @@ -697,16 +697,24 @@ export default defineComponent({ }, placesSetup(event: Event) { + const warnSetup = this.t( + "memories", + "Looks like the database is already setup. Are you sure you want to redownload planet data?" + ); + const warnLong = this.t( + "memories", + "You are about to download the planet database. This may take a while." + ); + const warnReindex = this.t( + "memories", + "This may also cause all photos to be re-indexed!" + ); const msg = - "Looks like the database is already setup. Are you sure you want to drop the table and redownload OSM data?"; - if (this.status.gis_count && !confirm(msg)) { + (this.status.gis_count ? warnSetup : warnLong) + " " + warnReindex; + if (!confirm(msg)) { event.preventDefault(); event.stopPropagation(); return; - } else { - alert( - "Please wait for the download and insertion to complete. This may take a while." - ); } },