place-setup: recalculate places

Signed-off-by: Varun Patil <varunpatil@ucla.edu>

lib/Command/PlacesSetup.php

@@ -88,10 +88,16 @@ class PlacesSetup extends Command
         }
 
         // Download the planet database
-        $this->output->writeln('Downloading planet database');
         $datafile = $this->places->downloadPlanet();
 
         // Import the planet database
         $this->places->importPlanet($datafile);
+
+        // Recalculate all places
+        $this->places->recalculateAll();
+
+        $this->output->writeln('Done');
+
+        return 0;
     }
 }

lib/Controller/OtherController.php

@@ -205,11 +205,9 @@ class OtherController extends GenericApiController
             header('Content-Length: 0');
 
             $places = \OC::$server->get(\OCA\Memories\Service\Places::class);
-
-            echo "Downloading planet file...\n";
-            flush();
             $datafile = $places->downloadPlanet();
             $places->importPlanet($datafile);
+            $places->recalculateAll();
 
             echo "Done.\n";
         } catch (\Exception $e) {

lib/Db/TimelineWriteOrphans.php

@@ -12,21 +12,78 @@ trait TimelineWriteOrphans
     protected IDBConnection $connection;
 
     /**
-     * Mark all files in the table as orphaned.
+     * Mark all or some files in the table as (un)orphaned.
+     *
+     * @param bool  $value    True to mark as orphaned, false to mark as un-orphaned
+     * @param int[] $fileIds  List of file IDs to mark, or empty to mark all files
+     * @param bool  $onlyMain Only mark the main file, not the live photo
      *
      * @return int Number of rows affected
      */
-    public function orphanAll(): int
+    public function orphanAll(bool $value = true, ?array $fileIds = null, bool $onlyMain = false): int
     {
-        $do = function (string $table) {
+        $do = function (string $table) use ($value, $fileIds) {
             $query = $this->connection->getQueryBuilder();
             $query->update($table)
-                ->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
+                ->set('orphan', $query->createNamedParameter($value, IQueryBuilder::PARAM_BOOL))
             ;
 
+            if ($fileIds) {
+                $query->where($query->expr()->in('fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
+            }
+
             return $query->executeStatement();
         };
 
-        return $do('memories') + $do('memories_livephoto');
+        $count = $do('memories');
+
+        if ($onlyMain) {
+            return $count;
+        }
+
+        return $count + $do('memories_livephoto');
+    }
+
+    /**
+     * Orphan and run an update on all files.
+     *
+     * @param array    $fields   list of fields to select
+     * @param int      $txnSize  number of rows to process in a single transaction
+     * @param \Closure $callback will be passed each row
+     */
+    public function orphanAndRun(array $fields, int $txnSize, \Closure $callback)
+    {
+        // Orphan all files. This means if we are interrupted,
+        // it will lead to a re-index of the whole library!
+        $this->orphanAll(true, null, true);
+
+        while (\count($orphans = $this->getSomeOrphans($txnSize, $fields))) {
+            $this->connection->beginTransaction();
+
+            foreach ($orphans as $row) {
+                $callback($row);
+            }
+
+            // Mark all files as not orphaned.
+            $fileIds = array_map(fn ($row) => $row['fileid'], $orphans);
+            $this->orphanAll(false, $fileIds, true);
+
+            $this->connection->commit();
+        }
+    }
+
+    /**
+     * Get a list of orphaned files.
+     */
+    protected function getSomeOrphans(int $count, array $fields): array
+    {
+        $query = $this->connection->getQueryBuilder();
+        $query->select(...$fields)
+            ->from('memories')
+            ->where($query->expr()->eq('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)))
+            ->setMaxResults($count)
+        ;
+
+        return $query->executeQuery()->fetchAll();
     }
 }

lib/Db/TimelineWritePlaces.php

@@ -15,53 +15,6 @@ trait TimelineWritePlaces
 {
     protected IDBConnection $connection;
 
-    /**
-     * Process the location part of exif data.
-     *
-     * Also update the exif data with the tzid from location (LocationTZID)
-     * Performs an in-place update of the exif data.
-     *
-     * @param int        $fileId  The file ID
-     * @param array      $exif    The exif data (will change)
-     * @param array|bool $prevRow The previous row of data
-     *
-     * @return array Update values
-     */
-    protected function processExifLocation(int $fileId, array &$exif, $prevRow): array
-    {
-        // Store location data
-        [$lat, $lon] = self::readCoord($exif);
-        $oldLat = $prevRow ? (float) $prevRow['lat'] : null;
-        $oldLon = $prevRow ? (float) $prevRow['lon'] : null;
-        $mapCluster = $prevRow ? (int) $prevRow['mapcluster'] : -1;
-        $osmIds = [];
-
-        if ($lat || $lon || $oldLat || $oldLon) {
-            try {
-                $mapCluster = $this->mapGetCluster($mapCluster, $lat, $lon, $oldLat, $oldLon);
-            } catch (\Exception $e) {
-                $logger = \OC::$server->get(LoggerInterface::class);
-                $logger->log(3, 'Error updating map cluster data: '.$e->getMessage(), ['app' => 'memories']);
-            }
-
-            try {
-                $osmIds = $this->updatePlacesData($fileId, $lat, $lon);
-            } catch (\Exception $e) {
-                $logger = \OC::$server->get(LoggerInterface::class);
-                $logger->log(3, 'Error updating places data: '.$e->getMessage(), ['app' => 'memories']);
-            }
-        }
-
-        // NULL if invalid
-        $mapCluster = $mapCluster <= 0 ? null : $mapCluster;
-
-        // Set tzid from location if not present
-        $this->setTzidFromLocation($exif, $osmIds);
-
-        // Return update values
-        return [$lat, $lon, $mapCluster, $osmIds];
-    }
-
     /**
      * Add places data for a file.
      *
@@ -71,7 +24,7 @@ trait TimelineWritePlaces
      *
      * @return array The list of osm_id of the places
      */
-    protected function updatePlacesData(int $fileId, $lat, $lon): array
+    public function updatePlacesData(int $fileId, $lat, $lon): array
     {
         // Get GIS type
         $gisType = \OCA\Memories\Util::placesGISType();
@@ -140,6 +93,53 @@ trait TimelineWritePlaces
         return array_map(fn ($row) => $row['osm_id'], $rows);
     }
 
+    /**
+     * Process the location part of exif data.
+     *
+     * Also update the exif data with the tzid from location (LocationTZID)
+     * Performs an in-place update of the exif data.
+     *
+     * @param int        $fileId  The file ID
+     * @param array      $exif    The exif data (will change)
+     * @param array|bool $prevRow The previous row of data
+     *
+     * @return array Update values
+     */
+    protected function processExifLocation(int $fileId, array &$exif, $prevRow): array
+    {
+        // Store location data
+        [$lat, $lon] = self::readCoord($exif);
+        $oldLat = $prevRow ? (float) $prevRow['lat'] : null;
+        $oldLon = $prevRow ? (float) $prevRow['lon'] : null;
+        $mapCluster = $prevRow ? (int) $prevRow['mapcluster'] : -1;
+        $osmIds = [];
+
+        if ($lat || $lon || $oldLat || $oldLon) {
+            try {
+                $mapCluster = $this->mapGetCluster($mapCluster, $lat, $lon, $oldLat, $oldLon);
+            } catch (\Exception $e) {
+                $logger = \OC::$server->get(LoggerInterface::class);
+                $logger->log(3, 'Error updating map cluster data: '.$e->getMessage(), ['app' => 'memories']);
+            }
+
+            try {
+                $osmIds = $this->updatePlacesData($fileId, $lat, $lon);
+            } catch (\Exception $e) {
+                $logger = \OC::$server->get(LoggerInterface::class);
+                $logger->log(3, 'Error updating places data: '.$e->getMessage(), ['app' => 'memories']);
+            }
+        }
+
+        // NULL if invalid
+        $mapCluster = $mapCluster <= 0 ? null : $mapCluster;
+
+        // Set tzid from location if not present
+        $this->setTzidFromLocation($exif, $osmIds);
+
+        // Return update values
+        return [$lat, $lon, $mapCluster, $osmIds];
+    }
+
     /**
      * Set timezone offset from location if not present.
      *

lib/Service/Places.php

@@ -2,6 +2,7 @@
 
 namespace OCA\Memories\Service;
 
+use OCA\Memories\Db\TimelineWrite;
 use OCP\IConfig;
 use OCP\IDBConnection;
 
@@ -18,13 +19,16 @@ class Places
 
     protected IConfig $config;
     protected IDBConnection $connection;
+    protected TimelineWrite $timelineWrite;
 
     public function __construct(
         IConfig $config,
-        IDBConnection $connection
+        IDBConnection $connection,
+        TimelineWrite $timelineWrite
     ) {
         $this->config = $config;
         $this->connection = $connection;
+        $this->timelineWrite = $timelineWrite;
     }
 
     /**
@@ -89,6 +93,9 @@ class Places
      */
     public function downloadPlanet(): string
     {
+        echo "Download planet data to temporary file...\n";
+        flush();
+
         $filename = sys_get_temp_dir().'/planet_coarse_boundaries.zip';
         unlink($filename);
 
@@ -134,6 +141,7 @@ class Places
     public function importPlanet(string $datafile): void
     {
         echo "Inserting planet data into database...\n";
+        flush();
 
         // Detect the GIS type
         $gis = $this->detectGisType();
@@ -317,16 +325,47 @@ class Places
         // Mark success
         echo "Planet database imported successfully!\n";
         echo "You should re-index your library now.\n";
+        flush();
         $this->config->setSystemValue('memories.gis_type', $gis);
 
         // Delete data file
         unlink($datafile);
     }
 
+    /**
+     * Recalculate all places for all users.
+     */
+    public function recalculateAll()
+    {
+        echo "Recalculating places for all files (do not interrupt this process)...\n";
+        flush();
+
+        $count = 0;
+        $this->timelineWrite->orphanAndRun(['fileid', 'lat', 'lon'], 20, function (array $row) use (&$count) {
+            ++$count;
+
+            // Only proceed if we have a valid location
+            $fileid = $row['fileid'];
+            $lat = (float) $row['lat'];
+            $lon = (float) $row['lon'];
+
+            // Update places
+            if ($lat || $lon) {
+                $this->timelineWrite->updatePlacesData($fileid, $lat, $lon);
+            }
+
+            // Print every 500 files
+            if (0 === $count % 500) {
+                echo "Updated places data for {$count} files\n";
+                flush();
+            }
+        });
+    }
+
     /**
      * Create database tables and indices.
      */
-    private function setupDatabase(): void
+    protected function setupDatabase(): void
     {
         try {
             // Get Gis type

src/Admin.vue

@@ -697,16 +697,24 @@ export default defineComponent({
     },
 
     placesSetup(event: Event) {
+      const warnSetup = this.t(
+        "memories",
+        "Looks like the database is already setup. Are you sure you want to redownload planet data?"
+      );
+      const warnLong = this.t(
+        "memories",
+        "You are about to download the planet database. This may take a while."
+      );
+      const warnReindex = this.t(
+        "memories",
+        "This may also cause all photos to be re-indexed!"
+      );
       const msg =
-        "Looks like the database is already setup. Are you sure you want to drop the table and redownload OSM data?";
+        (this.status.gis_count ? warnSetup : warnLong) + " " + warnReindex;
-      if (this.status.gis_count && !confirm(msg)) {
+      if (!confirm(msg)) {
         event.preventDefault();
         event.stopPropagation();
         return;
-      } else {
-        alert(
-          "Please wait for the download and insertion to complete. This may take a while."
-        );
       }
     },