From 7b647dc5c5f361870c22a0240ccf8a1793ea60de Mon Sep 17 00:00:00 2001
From: Varun Patil <radialapps@gmail.com>
Date: Tue, 10 Oct 2023 04:48:20 -0700
Subject: [PATCH] places-setup: improve transactions

Signed-off-by: Varun Patil <radialapps@gmail.com>
---
 lib/Service/Places.php | 47 ++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 25 deletions(-)

diff --git a/lib/Service/Places.php b/lib/Service/Places.php
index 4224a424..b373acf2 100644
--- a/lib/Service/Places.php
+++ b/lib/Service/Places.php
@@ -10,6 +10,7 @@ const GIS_TYPE_NONE = 0;
 const GIS_TYPE_MYSQL = 1;
 const GIS_TYPE_POSTGRES = 2;
 const APPROX_PLACES = 635189;
+const DB_TRANSACTION_SIZE = 50;
 
 const PLANET_URL = 'https://github.com/pulsejet/memories-assets/releases/download/geo-0.0.3/planet_coarse_boundaries.zip';
 
@@ -199,16 +200,11 @@ class Places
             throw new \Exception('No GIS support detected');
         }
 
-        // Drop the table if it exists
-        $p = $this->connection->getDatabasePlatform();
-        if ($this->geomCount() > 0) {
-            $this->connection->executeStatement($p->getDropTableSQL('memories_planet_geometry'));
-        }
-
         // Setup the database
-        $this->setupDatabase();
+        $this->setupDatabase($gis);
 
         // Truncate tables
+        $p = $this->connection->getDatabasePlatform();
         $this->connection->executeStatement($p->getTruncateTableSQL('*PREFIX*memories_planet', false));
         $this->connection->executeStatement($p->getTruncateTableSQL('memories_planet_geometry', false));
 
@@ -247,6 +243,18 @@ class Places
         // The number of places in the current transaction
         $txnCount = 0;
 
+        // Function to commit the current transaction
+        $transact = function () use (&$txnCount) {
+            if (++$txnCount >= DB_TRANSACTION_SIZE) {
+                $this->connection->commit();
+                $this->connection->beginTransaction();
+                $txnCount = 0;
+            }
+        };
+
+        // Start the first transaction
+        $this->connection->beginTransaction();
+
         // Iterate over the data file
         $handle = fopen($datafile, 'r');
         if ($handle) {
@@ -257,10 +265,6 @@ class Places
                     continue;
                 }
 
-                // Begin transaction
-                if (0 === $txnCount++) {
-                    $this->connection->beginTransaction();
-                }
                 ++$count;
 
                 // Decode JSON
@@ -292,6 +296,7 @@ class Places
                 $insertPlace->bindValue('name', $name);
                 $insertPlace->bindValue('other_names', $otherNames);
                 $insertPlace->execute();
+                $transact();
 
                 // Insert polygons into database
                 $idx = 0;
@@ -339,6 +344,7 @@ class Places
                         $insertGeometry->bindValue('osm_id', $osmId);
                         $insertGeometry->bindValue('geometry', $geometry);
                         $insertGeometry->execute();
+                        $transact();
                     } catch (\Exception $e) {
                         echo "ERROR: Failed to insert polygon {$polyid} ({$e->getMessage()} \n";
 
@@ -346,17 +352,10 @@ class Places
                     }
                 }
 
-                // Commit transaction every once in a while
-                if (0 === $count % 100) {
-                    $this->connection->commit();
-                    $txnCount = 0;
-
+                if (0 === $count % 500) {
                     // Print progress
                     $total = APPROX_PLACES;
                     $pct = round($count / $total * 100, 1);
-                }
-
-                if (0 === $count % 500) {
                     echo "Inserted {$count} / {$total} places ({$pct}%), Last: {$name}\n";
                     flush();
                 }
@@ -366,9 +365,7 @@ class Places
         }
 
         // Commit final transaction
-        if ($txnCount > 0) {
-            $this->connection->commit();
-        }
+        $this->connection->commit();
 
         // Mark success
         echo "Planet database imported successfully!\n";
@@ -413,11 +410,11 @@ class Places
     /**
      * Create database tables and indices.
      */
-    protected function setupDatabase(): void
+    protected function setupDatabase(int $gis): void
     {
         try {
-            // Get Gis type
-            $gis = $this->detectGisType();
+            // Drop the table if it exists
+            $this->connection->executeStatement('DROP TABLE IF EXISTS memories_planet_geometry');
 
             // Create table
             $sql = 'CREATE TABLE memories_planet_geometry (