From e495dbfcb86ab6f4cd57a36bc87e4a3144ceb7cc Mon Sep 17 00:00:00 2001 From: Varun Patil Date: Mon, 10 Apr 2023 19:17:16 -0700 Subject: [PATCH] admin: add places-setup route Signed-off-by: Varun Patil --- appinfo/routes.php | 1 + lib/BinExt.php | 7 +- lib/Command/PlacesSetup.php | 403 ++--------------------------- lib/Controller/OtherController.php | 41 +++ lib/Service/Places.php | 357 +++++++++++++++++++++++++ src/Admin.vue | 128 ++++++++- src/services/API.ts | 4 + 7 files changed, 559 insertions(+), 382 deletions(-) create mode 100644 lib/Service/Places.php diff --git a/appinfo/routes.php b/appinfo/routes.php index ce6bc98c..68496841 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -86,6 +86,7 @@ return [ ['name' => 'Other#getSystemStatus', 'url' => '/api/system-status', 'verb' => 'GET'], ['name' => 'Other#getSystemConfig', 'url' => '/api/system-config', 'verb' => 'GET'], ['name' => 'Other#setSystemConfig', 'url' => '/api/system-config/{key}', 'verb' => 'PUT'], + ['name' => 'Other#placesSetup', 'url' => '/api/occ/places-setup', 'verb' => 'POST'], // Service worker ['name' => 'Other#serviceWorker', 'url' => '/service-worker.js', 'verb' => 'GET'], diff --git a/lib/BinExt.php b/lib/BinExt.php index b25475c6..913c8025 100644 --- a/lib/BinExt.php +++ b/lib/BinExt.php @@ -243,7 +243,10 @@ class BinExt try { $client = new \GuzzleHttp\Client(); - $res = $client->request('GET', $url); + $res = $client->request('GET', $url, [ + 'timeout' => 1, + 'connect_timeout' => 1, + ]); } catch (\Exception $e) { throw new \Exception('failed to connect to go-vod: '.$e->getMessage()); } @@ -277,6 +280,8 @@ class BinExt $client = new \GuzzleHttp\Client(); $client->request('POST', $url, [ 'json' => $config, + 'timeout' => 1, + 'connect_timeout' => 1, ]); } catch (\Exception $e) { throw new \Exception('failed to connect to go-vod: '.$e->getMessage()); diff --git a/lib/Command/PlacesSetup.php b/lib/Command/PlacesSetup.php index 84542fa4..0196588a 100644 --- a/lib/Command/PlacesSetup.php +++ b/lib/Command/PlacesSetup.php @@ -23,8 +23,7 @@ declare(strict_types=1); namespace OCA\Memories\Command; -use OCP\IConfig; -use OCP\IDBConnection; +use OCA\Memories\Service\Places; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -38,19 +37,14 @@ const PLANET_URL = 'https://github.com/pulsejet/memories-assets/releases/downloa class PlacesSetup extends Command { - protected IConfig $config; protected OutputInterface $output; - protected IDBConnection $connection; - - protected int $gisType = GIS_TYPE_NONE; + protected Places $places; public function __construct( - IConfig $config, - IDBConnection $connection + Places $places ) { parent::__construct(); - $this->config = $config; - $this->connection = $connection; + $this->places = $places; } protected function configure(): void @@ -68,392 +62,43 @@ class PlacesSetup extends Command $this->output->writeln('Attempting to set up reverse geocoding'); // Detect the GIS type - $this->detectGisType(); - $this->config->setSystemValue('memories.gis_type', $this->gisType); - - // Make sure we support something - if (GIS_TYPE_NONE === $this->gisType) { + if ($this->places->detectGisType() <= 0) { $this->output->writeln('No supported GIS type detected'); return 1; } + $this->output->writeln('Database support was detected'); - // Check if the database is already set up - $hasDb = false; - - try { + // Check if database is already set up + if ($this->places->geomCount() > 0) { $this->output->writeln(''); - $this->connection->executeQuery('SELECT osm_id FROM memories_planet_geometry LIMIT 1')->fetch(); $this->output->writeln('Database is already set up'); $this->output->writeln('This will drop and re-download the planet database'); $this->output->writeln('This is generally not necessary to do frequently '); - $hasDb = true; - } catch (\Exception $e) { - } - // Ask confirmation - $tempdir = sys_get_temp_dir(); - $this->output->writeln(''); - $this->output->writeln('Are you sure you want to download the planet database?'); - $this->output->writeln("This will take a very long time and use some disk space in {$tempdir}"); - $this->output->write('Proceed? [y/N] '); - $handle = fopen('php://stdin', 'r'); - $line = fgets($handle); - if ('y' !== trim($line)) { - $this->output->writeln('Aborting'); - - return 1; - } - - // Drop the table - $p = $this->connection->getDatabasePlatform(); - if ($hasDb) { + // Ask confirmation $this->output->writeln(''); - $this->output->write('Dropping table ... '); - $this->connection->executeStatement($p->getDropTableSQL('memories_planet_geometry')); - $this->output->writeln('OK'); - } + $this->output->writeln('Are you sure you want to download the planet database?'); + $this->output->write('Proceed? [y/N] '); + $handle = fopen('php://stdin', 'r'); + $line = fgets($handle); + if (false === $line) { + $this->output->writeln('You need an interactive terminal to run this command'); - // Setup the database - $this->output->write('Setting up database ... '); - $this->setupDatabase(); - $this->output->writeln('OK'); - - // Truncate tables - $this->output->write('Truncating tables ... '); - $this->connection->executeStatement($p->getTruncateTableSQL('*PREFIX*memories_planet', false)); - $this->connection->executeStatement($p->getTruncateTableSQL('memories_planet_geometry', false)); - $this->output->writeln('OK'); - - // Download the data - $this->output->write('Downloading data ... '); - $datafile = $this->downloadPlanet(); - $this->output->writeln('OK'); - - // Start importing - $this->output->writeln(''); - $this->output->writeln('Importing data (this will take a while) ...'); - - // Start time - $start = time(); - - // Create place insertion statement - $query = $this->connection->getQueryBuilder(); - $query->insert('memories_planet') - ->values([ - 'osm_id' => $query->createParameter('osm_id'), - 'admin_level' => $query->createParameter('admin_level'), - 'name' => $query->createParameter('name'), - 'other_names' => $query->createParameter('other_names'), - ]) - ; - $insertPlace = $this->connection->prepare($query->getSQL()); - - // Create geometry insertion statement - $query = $this->connection->getQueryBuilder(); - $geomParam = $query->createParameter('geometry'); - if (GIS_TYPE_MYSQL === $this->gisType) { - $geomParam = "ST_GeomFromText({$geomParam})"; - } elseif (GIS_TYPE_POSTGRES === $this->gisType) { - $geomParam = "POLYGON({$geomParam}::text)"; - } - $query->insert('memories_planet_geometry') - ->values([ - 'id' => $query->createParameter('id'), - 'poly_id' => $query->createParameter('poly_id'), - 'type_id' => $query->createParameter('type_id'), - 'osm_id' => $query->createParameter('osm_id'), - 'geometry' => $query->createFunction($geomParam), - ]) - ; - $sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL()); - $insertGeometry = $this->connection->prepare($sql); - - // The number of places in the current transaction - $txnCount = 0; - - // Iterate over the data file - $handle = fopen($datafile, 'r'); - if ($handle) { - $count = 0; - while (($line = fgets($handle)) !== false) { - // Skip empty lines - if ('' === trim($line)) { - continue; - } - - // Begin transaction - if (0 === $txnCount++) { - $this->connection->beginTransaction(); - } - ++$count; - - // Decode JSON - $data = json_decode($line, true); - if (null === $data) { - $this->output->writeln('Failed to decode JSON'); - - continue; - } - - // Extract data - $osmId = $data['osm_id']; - $adminLevel = $data['admin_level']; - $name = $data['name']; - $boundaries = $data['geometry']; - $otherNames = json_encode($data['other_names'] ?? []); - - // Skip some places - if ($adminLevel > -2 && ($adminLevel <= 1 || $adminLevel >= 10)) { - // <=1: These are too general, e.g. "Earth"? or invalid - // >=10: These are too specific, e.g. "Community Board" - // <-1: These are special, e.g. "Timezone" = -7 - continue; - } - - // Insert place into database - $insertPlace->bindValue('osm_id', $osmId); - $insertPlace->bindValue('admin_level', $adminLevel); - $insertPlace->bindValue('name', $name); - $insertPlace->bindValue('other_names', $otherNames); - $insertPlace->execute(); - - // Insert polygons into database - $idx = 0; - foreach ($boundaries as &$polygon) { - // $polygon is a struct as - // [ "t" => "e", "c" => [lon, lat], [lon, lat], ... ] ] - - $polyid = $polygon['i']; - $typeid = $polygon['t']; - $pkey = $polygon['k']; - $coords = $polygon['c']; - - // Create parameters - ++$idx; - $geometry = ''; - - if (\count($coords) < 3) { - $this->output->writeln('Invalid polygon'); - - continue; - } - - if (GIS_TYPE_MYSQL === $this->gisType) { - $points = implode(',', array_map(function ($point) { - $x = $point[0]; - $y = $point[1]; - - return "{$x} {$y}"; - }, $coords)); - - $geometry = "POLYGON(({$points}))"; - } elseif (GIS_TYPE_POSTGRES === $this->gisType) { - $geometry = implode(',', array_map(function ($point) { - $x = $point[0]; - $y = $point[1]; - - return "({$x},{$y})"; - }, $coords)); - } - - try { - $insertGeometry->bindValue('id', $pkey); - $insertGeometry->bindValue('poly_id', $polyid); - $insertGeometry->bindValue('type_id', $typeid); - $insertGeometry->bindValue('osm_id', $osmId); - $insertGeometry->bindValue('geometry', $geometry); - $insertGeometry->execute(); - } catch (\Exception $e) { - $this->output->writeln('Failed to insert into database'); - $this->output->writeln($e->getMessage()); - - continue; - } - } - - // Commit transaction every once in a while - if (0 === $count % 100) { - $this->connection->commit(); - $txnCount = 0; - - // Print progress - $end = time(); - $elapsed = ($end - $start) ?: 1; - $rate = $count / $elapsed; - $remaining = APPROX_PLACES - $count; - $eta = round($remaining / $rate); - $rate = round($rate, 1); - $this->output->writeln("Inserted {$count} places, {$rate}/s, ETA: {$eta}s, Last: {$name}"); - } + return 1; } + if ('y' !== trim($line)) { + $this->output->writeln('Aborting'); - fclose($handle); - } - - // Commit final transaction - if ($txnCount > 0) { - $this->connection->commit(); - } - - // Delete file - unlink($datafile); - - // Done - $this->output->writeln(''); - $this->output->writeln('Planet database imported successfully!'); - $this->output->writeln('If this is the first time you did this, you should now run:'); - $this->output->writeln('occ memories:index -f'); - - // Mark success - $this->config->setSystemValue('memories.gis_type', $this->gisType); - - return 0; - } - - protected function detectGisType() - { - // Make sure database prefix is set - $prefix = $this->config->getSystemValue('dbtableprefix', '') ?: ''; - if ('' === $prefix) { - $this->output->writeln('Database table prefix is not set'); - $this->output->writeln('Custom database extensions cannot be used without a prefix'); - $this->output->writeln('Reverse geocoding will not work and is disabled'); - $this->gisType = GIS_TYPE_NONE; - - return; - } - - // Warn the admin about the database prefix not being used - $this->output->writeln(''); - $this->output->writeln("Database table prefix is set to '{$prefix}'"); - $this->output->writeln('If the planet can be imported, it will not use this prefix'); - $this->output->writeln('The table will be named "memories_planet_geometry"'); - $this->output->writeln('This is necessary for using custom database extensions'); - $this->output->writeln(''); - - // Detect database type - $platform = strtolower(\get_class($this->connection->getDatabasePlatform())); - - // Test MySQL-like support in databse - if (str_contains($platform, 'mysql') || str_contains($platform, 'mariadb')) { - try { - $res = $this->connection->executeQuery("SELECT ST_GeomFromText('POINT(1 1)')")->fetch(); - if (0 === \count($res)) { - throw new \Exception('Invalid result'); - } - $this->output->writeln('MySQL-like support detected!'); - $this->gisType = GIS_TYPE_MYSQL; - - return; - } catch (\Exception $e) { - $this->output->writeln('No MySQL-like support detected'); + return 1; } } - // Test Postgres native geometry like support in database - if (str_contains($platform, 'postgres')) { - try { - $res = $this->connection->executeQuery("SELECT POINT('1,1')")->fetch(); - if (0 === \count($res)) { - throw new \Exception('Invalid result'); - } - $this->output->writeln('Postgres native geometry support detected!'); - $this->gisType = GIS_TYPE_POSTGRES; + // Download the planet database + $this->output->writeln('Downloading planet database'); + $datafile = $this->places->downloadPlanet(); - return; - } catch (\Exception $e) { - $this->output->writeln('No Postgres native geometry support detected'); - } - } - } - - protected function setupDatabase(): void - { - try { - $sql = 'CREATE TABLE memories_planet_geometry ( - id varchar(255) NOT NULL PRIMARY KEY, - poly_id varchar(255) NOT NULL, - type_id int NOT NULL, - osm_id int NOT NULL, - geometry polygon NOT NULL - );'; - $this->connection->executeQuery($sql); - - // Add indexes - $this->connection->executeQuery('CREATE INDEX planet_osm_id_idx ON memories_planet_geometry (osm_id);'); - - // Add spatial index - if (GIS_TYPE_MYSQL === $this->gisType) { - $this->connection->executeQuery('CREATE SPATIAL INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry (geometry);'); - } elseif (GIS_TYPE_POSTGRES === $this->gisType) { - $this->connection->executeQuery('CREATE INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry USING GIST (geometry);'); - } - } catch (\Exception $e) { - $this->output->writeln('Failed to create planet table'); - $this->output->writeln($e->getMessage()); - - exit; - } - } - - protected function ensureDeleted(string $filename) - { - if (!file_exists($filename)) { - return; - } - - unlink($filename); - if (file_exists($filename)) { - $this->output->writeln('Failed to delete data file'); - $this->output->writeln("Please delete {$filename} manually"); - - exit; - } - } - - protected function downloadPlanet(): string - { - $filename = sys_get_temp_dir().'/planet_coarse_boundaries.zip'; - $this->ensureDeleted($filename); - - $txtfile = sys_get_temp_dir().'/planet_coarse_boundaries.txt'; - $this->ensureDeleted($txtfile); - - $fp = fopen($filename, 'w+'); - - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, PLANET_URL); - curl_setopt($ch, CURLOPT_FILE, $fp); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_TIMEOUT, 60000); - curl_exec($ch); - curl_close($ch); - - fclose($fp); - - // Unzip - $zip = new \ZipArchive(); - $res = $zip->open($filename); - if (true === $res) { - $zip->extractTo(sys_get_temp_dir()); - $zip->close(); - } else { - $this->output->writeln('Failed to unzip planet file'); - - exit; - } - - // Check if file exists - if (!file_exists($txtfile)) { - $this->output->writeln('Failed to find planet data file after unzip'); - - exit; - } - - // Delete zip file - unlink($filename); - - return $txtfile; + // Import the planet database + $this->places->importPlanet($datafile); } } diff --git a/lib/Controller/OtherController.php b/lib/Controller/OtherController.php index 8539decc..3e3696ed 100644 --- a/lib/Controller/OtherController.php +++ b/lib/Controller/OtherController.php @@ -128,6 +128,16 @@ class OtherController extends GenericApiController // Check for system perl $status['perl'] = $this->getExecutableStatus(exec('which perl')); + // Get GIS status + $places = \OC::$server->get(\OCA\Memories\Service\Places::class); + + try { + $status['gis_type'] = $places->detectGisType(); + $status['gis_count'] = $places->geomCount(); + } catch (\Exception $e) { + $status['gis_type'] = $e->getMessage(); + } + // Check ffmpeg and ffprobe binaries $status['ffmpeg'] = $this->getExecutableStatus(Util::getSystemConfig('memories.vod.ffmpeg')); $status['ffprobe'] = $this->getExecutableStatus(Util::getSystemConfig('memories.vod.ffprobe')); @@ -158,6 +168,37 @@ class OtherController extends GenericApiController }); } + /** + * @AdminRequired + */ + public function placesSetup(): Http\Response + { + try { + // Set PHP timeout to infinite + set_time_limit(0); + + // Send headers for long-running request + header('Content-Type: text/plain'); + header('X-Accel-Buffering: no'); + header('Cache-Control: no-cache'); + header('Connection: keep-alive'); + header('Content-Length: 0'); + + $places = \OC::$server->get(\OCA\Memories\Service\Places::class); + + echo "Downloading planet file...\n"; + flush(); + $datafile = $places->downloadPlanet(); + $places->importPlanet($datafile); + + echo "Done.\n"; + } catch (\Exception $e) { + echo 'Failed: '.$e->getMessage()."\n"; + } + + exit; + } + /** * @NoAdminRequired * diff --git a/lib/Service/Places.php b/lib/Service/Places.php new file mode 100644 index 00000000..e0eb7ee2 --- /dev/null +++ b/lib/Service/Places.php @@ -0,0 +1,357 @@ +config = $config; + $this->connection = $connection; + } + + /** + * Make SQL query to detect GIS type. + */ + public function detectGisType() + { + // Make sure database prefix is set + $prefix = $this->config->getSystemValue('dbtableprefix', '') ?: ''; + if ('' === $prefix) { + throw new \Exception('Database table prefix is not set. Cannot use database exceptions (dbtableprefix).'); + } + + // Detect database type + $platform = strtolower(\get_class($this->connection->getDatabasePlatform())); + + // Test MySQL-like support in databse + if (str_contains($platform, 'mysql') || str_contains($platform, 'mariadb')) { + try { + $res = $this->connection->executeQuery("SELECT ST_GeomFromText('POINT(1 1)')")->fetch(); + if (0 === \count($res)) { + throw new \Exception('Invalid result'); + } + + return GIS_TYPE_MYSQL; + } catch (\Exception $e) { + throw new \Exception('No MySQL-like geometry support detected'); + } + } + + // Test Postgres native geometry like support in database + if (str_contains($platform, 'postgres')) { + try { + $res = $this->connection->executeQuery("SELECT POINT('1,1')")->fetch(); + if (0 === \count($res)) { + throw new \Exception('Invalid result'); + } + + return GIS_TYPE_POSTGRES; + } catch (\Exception $e) { + throw new \Exception('No Postgres native geometry support detected'); + } + } + + return GIS_TYPE_NONE; + } + + /** + * Check if DB is already setup and return number of entries. + */ + public function geomCount(): int + { + try { + return $this->connection->executeQuery('SELECT COUNT(osm_id) as c FROM memories_planet_geometry')->fetchOne(); + } catch (\Exception $e) { + return 0; + } + } + + /** + * Download planet database file and return path to it. + */ + public function downloadPlanet(): string + { + $filename = sys_get_temp_dir().'/planet_coarse_boundaries.zip'; + unlink($filename); + + $txtfile = sys_get_temp_dir().'/planet_coarse_boundaries.txt'; + unlink($txtfile); + + $fp = fopen($filename, 'w+'); + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, PLANET_URL); + curl_setopt($ch, CURLOPT_FILE, $fp); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_TIMEOUT, 3600); + curl_exec($ch); + curl_close($ch); + + fclose($fp); + + // Unzip + $zip = new \ZipArchive(); + $res = $zip->open($filename); + if (true === $res) { + $zip->extractTo(sys_get_temp_dir()); + $zip->close(); + } else { + throw new \Exception('Failed to unzip planet data file'); + } + + // Check if file exists + if (!file_exists($txtfile)) { + throw new \Exception('Failed to find planet data file after unzip'); + } + + // Delete zip file + unlink($filename); + + return $txtfile; + } + + /** + * Insert planet into database from file. + */ + public function importPlanet(string $datafile): void + { + echo "Inserting planet data into database...\n"; + + // Detect the GIS type + $gis = $this->detectGisType(); + + // Make sure we support something + if (GIS_TYPE_NONE === $gis) { + throw new \Exception('No GIS support detected'); + } + + // Drop the table if it exists + $p = $this->connection->getDatabasePlatform(); + if ($this->geomCount() > 0) { + $this->connection->executeStatement($p->getDropTableSQL('memories_planet_geometry')); + } + + // Setup the database + $this->setupDatabase(); + + // Truncate tables + $this->connection->executeStatement($p->getTruncateTableSQL('*PREFIX*memories_planet', false)); + $this->connection->executeStatement($p->getTruncateTableSQL('memories_planet_geometry', false)); + + // Create place insertion statement + $query = $this->connection->getQueryBuilder(); + $query->insert('memories_planet') + ->values([ + 'osm_id' => $query->createParameter('osm_id'), + 'admin_level' => $query->createParameter('admin_level'), + 'name' => $query->createParameter('name'), + 'other_names' => $query->createParameter('other_names'), + ]) + ; + $insertPlace = $this->connection->prepare($query->getSQL()); + + // Create geometry insertion statement + $query = $this->connection->getQueryBuilder(); + $geomParam = $query->createParameter('geometry'); + if (GIS_TYPE_MYSQL === $gis) { + $geomParam = "ST_GeomFromText({$geomParam})"; + } elseif (GIS_TYPE_POSTGRES === $gis) { + $geomParam = "POLYGON({$geomParam}::text)"; + } + $query->insert('memories_planet_geometry') + ->values([ + 'id' => $query->createParameter('id'), + 'poly_id' => $query->createParameter('poly_id'), + 'type_id' => $query->createParameter('type_id'), + 'osm_id' => $query->createParameter('osm_id'), + 'geometry' => $query->createFunction($geomParam), + ]) + ; + $sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL()); + $insertGeometry = $this->connection->prepare($sql); + + // The number of places in the current transaction + $txnCount = 0; + + // Iterate over the data file + $handle = fopen($datafile, 'r'); + if ($handle) { + $count = 0; + while (($line = fgets($handle)) !== false) { + // Skip empty lines + if ('' === trim($line)) { + continue; + } + + // Begin transaction + if (0 === $txnCount++) { + $this->connection->beginTransaction(); + } + ++$count; + + // Decode JSON + $data = json_decode($line, true); + if (null === $data) { + echo "ERROR: Failed to decode JSON\n"; + + continue; + } + + // Extract data + $osmId = $data['osm_id']; + $adminLevel = $data['admin_level']; + $name = $data['name']; + $boundaries = $data['geometry']; + $otherNames = json_encode($data['other_names'] ?? []); + + // Skip some places + if ($adminLevel > -2 && ($adminLevel <= 1 || $adminLevel >= 10)) { + // <=1: These are too general, e.g. "Earth"? or invalid + // >=10: These are too specific, e.g. "Community Board" + // <-1: These are special, e.g. "Timezone" = -7 + continue; + } + + // Insert place into database + $insertPlace->bindValue('osm_id', $osmId); + $insertPlace->bindValue('admin_level', $adminLevel); + $insertPlace->bindValue('name', $name); + $insertPlace->bindValue('other_names', $otherNames); + $insertPlace->execute(); + + // Insert polygons into database + $idx = 0; + foreach ($boundaries as &$polygon) { + // $polygon is a struct as + // [ "t" => "e", "c" => [lon, lat], [lon, lat], ... ] ] + + $polyid = $polygon['i']; + $typeid = $polygon['t']; + $pkey = $polygon['k']; + $coords = $polygon['c']; + + // Create parameters + ++$idx; + $geometry = ''; + + if (\count($coords) < 3) { + echo "ERROR: Invalid polygon {$polyid}\n"; + + continue; + } + + if (GIS_TYPE_MYSQL === $gis) { + $points = implode(',', array_map(function ($point) { + $x = $point[0]; + $y = $point[1]; + + return "{$x} {$y}"; + }, $coords)); + + $geometry = "POLYGON(({$points}))"; + } elseif (GIS_TYPE_POSTGRES === $gis) { + $geometry = implode(',', array_map(function ($point) { + $x = $point[0]; + $y = $point[1]; + + return "({$x},{$y})"; + }, $coords)); + } + + try { + $insertGeometry->bindValue('id', $pkey); + $insertGeometry->bindValue('poly_id', $polyid); + $insertGeometry->bindValue('type_id', $typeid); + $insertGeometry->bindValue('osm_id', $osmId); + $insertGeometry->bindValue('geometry', $geometry); + $insertGeometry->execute(); + } catch (\Exception $e) { + echo "ERROR: Failed to insert polygon {$polyid} ({$e->getMessage()} \n"; + + continue; + } + } + + // Commit transaction every once in a while + if (0 === $count % 100) { + $this->connection->commit(); + $txnCount = 0; + + // Print progress + $total = APPROX_PLACES; + $pct = round($count / $total * 100, 1); + } + + if (0 === $count % 500) { + echo "Inserted {$count} / {$total} places ({$pct}%), Last: {$name}\n"; + flush(); + } + } + + fclose($handle); + } + + // Commit final transaction + if ($txnCount > 0) { + $this->connection->commit(); + } + + // Mark success + echo "Planet database imported successfully!\n"; + $this->config->setSystemValue('memories.gis_type', $gis); + + // Delete data file + unlink($datafile); + } + + /** + * Create database tables and indices. + */ + private function setupDatabase(): void + { + try { + // Get Gis type + $gis = $this->detectGisType(); + + // Create table + $sql = 'CREATE TABLE memories_planet_geometry ( + id varchar(255) NOT NULL PRIMARY KEY, + poly_id varchar(255) NOT NULL, + type_id int NOT NULL, + osm_id int NOT NULL, + geometry polygon NOT NULL + );'; + $this->connection->executeQuery($sql); + + // Add indexes + $this->connection->executeQuery('CREATE INDEX planet_osm_id_idx ON memories_planet_geometry (osm_id);'); + + // Add spatial index + if (GIS_TYPE_MYSQL === $gis) { + $this->connection->executeQuery('CREATE SPATIAL INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry (geometry);'); + } elseif (GIS_TYPE_POSTGRES === $gis) { + $this->connection->executeQuery('CREATE INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry USING GIST (geometry);'); + } + } catch (\Exception $e) { + throw new \Exception('Failed to create database tables: '.$e->getMessage()); + } + } +} diff --git a/src/Admin.vue b/src/Admin.vue index b296ce8a..82ffadb4 100644 --- a/src/Admin.vue +++ b/src/Admin.vue @@ -2,6 +2,7 @@
+

{{ t("memories", "EXIF Extraction") }}