places-setup: speedup by 10x

pull/460/head
Varun Patil 2023-02-24 01:39:49 -08:00
parent fe0de6e9cb
commit 8eaea1cf7a
1 changed files with 20 additions and 3 deletions

View File

@ -168,6 +168,9 @@ class PlacesSetup extends Command
$sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL());
$insertGeometry = $this->connection->prepare($sql);
// The number of places in the current transaction
$txnCount = 0;
// Iterate over the data file
$handle = fopen($datafile, 'r');
if ($handle) {
@ -177,6 +180,11 @@ class PlacesSetup extends Command
if ('' === trim($line)) {
continue;
}
// Begin transaction
if (0 === $txnCount++) {
$this->connection->beginTransaction();
}
++$count;
// Decode JSON
@ -260,10 +268,14 @@ class PlacesSetup extends Command
}
}
// Commit transaction every once in a while
if ($count % 250 === 0) {
$this->connection->commit();
$txnCount = 0;
// Print progress
if (0 === $count % 500) {
$end = time();
$elapsed = $end - $start;
$elapsed = ($end - $start) ?: 1;
$rate = $count / $elapsed;
$remaining = APPROX_PLACES - $count;
$eta = round($remaining / $rate);
@ -275,6 +287,11 @@ class PlacesSetup extends Command
fclose($handle);
}
// Commit final transaction
if ($txnCount > 0) {
$this->connection->commit();
}
// Delete file
unlink($datafile);