RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

places-setup: speedup by 10x

pull/460/head
Varun Patil 2023-02-24 01:39:49 -08:00
parent fe0de6e9cb
commit 8eaea1cf7a
1 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/Command/PlacesSetup.php
View File

@ -168,6 +168,9 @@ class PlacesSetup extends Command
$sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL()); $sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL());
$insertGeometry = $this->connection->prepare($sql); $insertGeometry = $this->connection->prepare($sql);
// The number of places in the current transaction
$txnCount = 0;
// Iterate over the data file // Iterate over the data file
$handle = fopen($datafile, 'r'); $handle = fopen($datafile, 'r');
if ($handle) { if ($handle) {
@ -177,6 +180,11 @@ class PlacesSetup extends Command
if ('' === trim($line)) { if ('' === trim($line)) {
continue; continue;
} }
// Begin transaction
if (0 === $txnCount++) {
$this->connection->beginTransaction();
}
++$count; ++$count;
// Decode JSON // Decode JSON
@ -260,10 +268,14 @@ class PlacesSetup extends Command
} }
} }
// Commit transaction every once in a while
if ($count % 250 === 0) {
$this->connection->commit();
$txnCount = 0;
// Print progress // Print progress
if (0 === $count % 500) {
$end = time(); $end = time();
$elapsed = $end - $start; $elapsed = ($end - $start) ?: 1;
$rate = $count / $elapsed; $rate = $count / $elapsed;
$remaining = APPROX_PLACES - $count; $remaining = APPROX_PLACES - $count;
$eta = round($remaining / $rate); $eta = round($remaining / $rate);
@ -275,6 +287,11 @@ class PlacesSetup extends Command
fclose($handle); fclose($handle);
} }
// Commit final transaction
if ($txnCount > 0) {
$this->connection->commit();
}
// Delete file // Delete file
unlink($datafile); unlink($datafile);