RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

admin: add places-setup route 

Signed-off-by: Varun Patil <varunpatil@ucla.edu>
pull/563/head
Varun Patil 2023-04-10 19:17:16 -07:00
parent 4871c2c2bc
commit e495dbfcb8
7 changed files with 559 additions and 382 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
appinfo/routes.php
View File

@ -86,6 +86,7 @@ return [
['name' => 'Other#getSystemStatus', 'url' => '/api/system-status', 'verb' => 'GET'],
['name' => 'Other#getSystemConfig', 'url' => '/api/system-config', 'verb' => 'GET'],
['name' => 'Other#setSystemConfig', 'url' => '/api/system-config/{key}', 'verb' => 'PUT'],
['name' => 'Other#placesSetup', 'url' => '/api/occ/places-setup', 'verb' => 'POST'],
// Service worker
['name' => 'Other#serviceWorker', 'url' => '/service-worker.js', 'verb' => 'GET'],

7
lib/BinExt.php
View File

@ -243,7 +243,10 @@ class BinExt
try {
$client = new \GuzzleHttp\Client();
$res = $client->request('GET', $url);
$res = $client->request('GET', $url, [
'timeout' => 1,
'connect_timeout' => 1,
]);
} catch (\Exception $e) {
throw new \Exception('failed to connect to go-vod: '.$e->getMessage());
}
@ -277,6 +280,8 @@ class BinExt
$client = new \GuzzleHttp\Client();
$client->request('POST', $url, [
'json' => $config,
'timeout' => 1,
'connect_timeout' => 1,
]);
} catch (\Exception $e) {
throw new \Exception('failed to connect to go-vod: '.$e->getMessage());

403
lib/Command/PlacesSetup.php
View File

@ -23,8 +23,7 @@ declare(strict_types=1);
namespace OCA\Memories\Command;
use OCP\IConfig;
use OCP\IDBConnection;
use OCA\Memories\Service\Places;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
@ -38,19 +37,14 @@ const PLANET_URL = 'https://github.com/pulsejet/memories-assets/releases/downloa
class PlacesSetup extends Command
{
protected IConfig $config;
protected OutputInterface $output;
protected IDBConnection $connection;
protected int $gisType = GIS_TYPE_NONE;
protected Places $places;
public function __construct(
IConfig $config,
IDBConnection $connection
Places $places
) {
parent::__construct();
$this->config = $config;
$this->connection = $connection;
$this->places = $places;
}
protected function configure(): void
@ -68,392 +62,43 @@ class PlacesSetup extends Command
$this->output->writeln('Attempting to set up reverse geocoding');
// Detect the GIS type
$this->detectGisType();
$this->config->setSystemValue('memories.gis_type', $this->gisType);
// Make sure we support something
if (GIS_TYPE_NONE === $this->gisType) {
if ($this->places->detectGisType() <= 0) {
$this->output->writeln('<error>No supported GIS type detected</error>');
return 1;
}
$this->output->writeln('Database support was detected');
// Check if the database is already set up
$hasDb = false;
try {
// Check if database is already set up
if ($this->places->geomCount() > 0) {
$this->output->writeln('');
$this->connection->executeQuery('SELECT osm_id FROM memories_planet_geometry LIMIT 1')->fetch();
$this->output->writeln('<error>Database is already set up</error>');
$this->output->writeln('<error>This will drop and re-download the planet database</error>');
$this->output->writeln('<error>This is generally not necessary to do frequently </error>');
$hasDb = true;
} catch (\Exception $e) {
}
// Ask confirmation
$tempdir = sys_get_temp_dir();
$this->output->writeln('');
$this->output->writeln('Are you sure you want to download the planet database?');
$this->output->writeln("This will take a very long time and use some disk space in {$tempdir}");
$this->output->write('Proceed? [y/N] ');
$handle = fopen('php://stdin', 'r');
$line = fgets($handle);
if ('y' !== trim($line)) {
$this->output->writeln('Aborting');
return 1;
}
// Drop the table
$p = $this->connection->getDatabasePlatform();
if ($hasDb) {
// Ask confirmation
$this->output->writeln('');
$this->output->write('Dropping table ... ');
$this->connection->executeStatement($p->getDropTableSQL('memories_planet_geometry'));
$this->output->writeln('OK');
}
$this->output->writeln('Are you sure you want to download the planet database?');
$this->output->write('Proceed? [y/N] ');
$handle = fopen('php://stdin', 'r');
$line = fgets($handle);
if (false === $line) {
$this->output->writeln('<error>You need an interactive terminal to run this command</error>');
// Setup the database
$this->output->write('Setting up database ... ');
$this->setupDatabase();
$this->output->writeln('OK');
// Truncate tables
$this->output->write('Truncating tables ... ');
$this->connection->executeStatement($p->getTruncateTableSQL('*PREFIX*memories_planet', false));
$this->connection->executeStatement($p->getTruncateTableSQL('memories_planet_geometry', false));
$this->output->writeln('OK');
// Download the data
$this->output->write('Downloading data ... ');
$datafile = $this->downloadPlanet();
$this->output->writeln('OK');
// Start importing
$this->output->writeln('');
$this->output->writeln('Importing data (this will take a while) ...');
// Start time
$start = time();
// Create place insertion statement
$query = $this->connection->getQueryBuilder();
$query->insert('memories_planet')
->values([
'osm_id' => $query->createParameter('osm_id'),
'admin_level' => $query->createParameter('admin_level'),
'name' => $query->createParameter('name'),
'other_names' => $query->createParameter('other_names'),
])
;
$insertPlace = $this->connection->prepare($query->getSQL());
// Create geometry insertion statement
$query = $this->connection->getQueryBuilder();
$geomParam = $query->createParameter('geometry');
if (GIS_TYPE_MYSQL === $this->gisType) {
$geomParam = "ST_GeomFromText({$geomParam})";
} elseif (GIS_TYPE_POSTGRES === $this->gisType) {
$geomParam = "POLYGON({$geomParam}::text)";
}
$query->insert('memories_planet_geometry')
->values([
'id' => $query->createParameter('id'),
'poly_id' => $query->createParameter('poly_id'),
'type_id' => $query->createParameter('type_id'),
'osm_id' => $query->createParameter('osm_id'),
'geometry' => $query->createFunction($geomParam),
])
;
$sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL());
$insertGeometry = $this->connection->prepare($sql);
// The number of places in the current transaction
$txnCount = 0;
// Iterate over the data file
$handle = fopen($datafile, 'r');
if ($handle) {
$count = 0;
while (($line = fgets($handle)) !== false) {
// Skip empty lines
if ('' === trim($line)) {
continue;
}
// Begin transaction
if (0 === $txnCount++) {
$this->connection->beginTransaction();
}
++$count;
// Decode JSON
$data = json_decode($line, true);
if (null === $data) {
$this->output->writeln('<error>Failed to decode JSON</error>');
continue;
}
// Extract data
$osmId = $data['osm_id'];
$adminLevel = $data['admin_level'];
$name = $data['name'];
$boundaries = $data['geometry'];
$otherNames = json_encode($data['other_names'] ?? []);
// Skip some places
if ($adminLevel > -2 && ($adminLevel <= 1 || $adminLevel >= 10)) {
// <=1: These are too general, e.g. "Earth"? or invalid
// >=10: These are too specific, e.g. "Community Board"
// <-1: These are special, e.g. "Timezone" = -7
continue;
}
// Insert place into database
$insertPlace->bindValue('osm_id', $osmId);
$insertPlace->bindValue('admin_level', $adminLevel);
$insertPlace->bindValue('name', $name);
$insertPlace->bindValue('other_names', $otherNames);
$insertPlace->execute();
// Insert polygons into database
$idx = 0;
foreach ($boundaries as &$polygon) {
// $polygon is a struct as
// [ "t" => "e", "c" => [lon, lat], [lon, lat], ... ] ]
$polyid = $polygon['i'];
$typeid = $polygon['t'];
$pkey = $polygon['k'];
$coords = $polygon['c'];
// Create parameters
++$idx;
$geometry = '';
if (\count($coords) < 3) {
$this->output->writeln('<error>Invalid polygon</error>');
continue;
}
if (GIS_TYPE_MYSQL === $this->gisType) {
$points = implode(',', array_map(function ($point) {
$x = $point[0];
$y = $point[1];
return "{$x} {$y}";
}, $coords));
$geometry = "POLYGON(({$points}))";
} elseif (GIS_TYPE_POSTGRES === $this->gisType) {
$geometry = implode(',', array_map(function ($point) {
$x = $point[0];
$y = $point[1];
return "({$x},{$y})";
}, $coords));
}
try {
$insertGeometry->bindValue('id', $pkey);
$insertGeometry->bindValue('poly_id', $polyid);
$insertGeometry->bindValue('type_id', $typeid);
$insertGeometry->bindValue('osm_id', $osmId);
$insertGeometry->bindValue('geometry', $geometry);
$insertGeometry->execute();
} catch (\Exception $e) {
$this->output->writeln('<error>Failed to insert into database</error>');
$this->output->writeln($e->getMessage());
continue;
}
}
// Commit transaction every once in a while
if (0 === $count % 100) {
$this->connection->commit();
$txnCount = 0;
// Print progress
$end = time();
$elapsed = ($end - $start) ?: 1;
$rate = $count / $elapsed;
$remaining = APPROX_PLACES - $count;
$eta = round($remaining / $rate);
$rate = round($rate, 1);
$this->output->writeln("Inserted {$count} places, {$rate}/s, ETA: {$eta}s, Last: {$name}");
}
return 1;
}
if ('y' !== trim($line)) {
$this->output->writeln('Aborting');
fclose($handle);
}
// Commit final transaction
if ($txnCount > 0) {
$this->connection->commit();
}
// Delete file
unlink($datafile);
// Done
$this->output->writeln('');
$this->output->writeln('Planet database imported successfully!');
$this->output->writeln('If this is the first time you did this, you should now run:');
$this->output->writeln('occ memories:index -f');
// Mark success
$this->config->setSystemValue('memories.gis_type', $this->gisType);
return 0;
}
protected function detectGisType()
{
// Make sure database prefix is set
$prefix = $this->config->getSystemValue('dbtableprefix', '') ?: '';
if ('' === $prefix) {
$this->output->writeln('<error>Database table prefix is not set</error>');
$this->output->writeln('Custom database extensions cannot be used without a prefix');
$this->output->writeln('Reverse geocoding will not work and is disabled');
$this->gisType = GIS_TYPE_NONE;
return;
}
// Warn the admin about the database prefix not being used
$this->output->writeln('');
$this->output->writeln("Database table prefix is set to '{$prefix}'");
$this->output->writeln('If the planet can be imported, it will not use this prefix');
$this->output->writeln('The table will be named "memories_planet_geometry"');
$this->output->writeln('This is necessary for using custom database extensions');
$this->output->writeln('');
// Detect database type
$platform = strtolower(\get_class($this->connection->getDatabasePlatform()));
// Test MySQL-like support in databse
if (str_contains($platform, 'mysql') || str_contains($platform, 'mariadb')) {
try {
$res = $this->connection->executeQuery("SELECT ST_GeomFromText('POINT(1 1)')")->fetch();
if (0 === \count($res)) {
throw new \Exception('Invalid result');
}
$this->output->writeln('MySQL-like support detected!');
$this->gisType = GIS_TYPE_MYSQL;
return;
} catch (\Exception $e) {
$this->output->writeln('No MySQL-like support detected');
return 1;
}
}
// Test Postgres native geometry like support in database
if (str_contains($platform, 'postgres')) {
try {
$res = $this->connection->executeQuery("SELECT POINT('1,1')")->fetch();
if (0 === \count($res)) {
throw new \Exception('Invalid result');
}
$this->output->writeln('Postgres native geometry support detected!');
$this->gisType = GIS_TYPE_POSTGRES;
// Download the planet database
$this->output->writeln('Downloading planet database');
$datafile = $this->places->downloadPlanet();
return;
} catch (\Exception $e) {
$this->output->writeln('No Postgres native geometry support detected');
}
}
}
protected function setupDatabase(): void
{
try {
$sql = 'CREATE TABLE memories_planet_geometry (
id varchar(255) NOT NULL PRIMARY KEY,
poly_id varchar(255) NOT NULL,
type_id int NOT NULL,
osm_id int NOT NULL,
geometry polygon NOT NULL
);';
$this->connection->executeQuery($sql);
// Add indexes
$this->connection->executeQuery('CREATE INDEX planet_osm_id_idx ON memories_planet_geometry (osm_id);');
// Add spatial index
if (GIS_TYPE_MYSQL === $this->gisType) {
$this->connection->executeQuery('CREATE SPATIAL INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry (geometry);');
} elseif (GIS_TYPE_POSTGRES === $this->gisType) {
$this->connection->executeQuery('CREATE INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry USING GIST (geometry);');
}
} catch (\Exception $e) {
$this->output->writeln('Failed to create planet table');
$this->output->writeln($e->getMessage());
exit;
}
}
protected function ensureDeleted(string $filename)
{
if (!file_exists($filename)) {
return;
}
unlink($filename);
if (file_exists($filename)) {
$this->output->writeln('<error>Failed to delete data file</error>');
$this->output->writeln("Please delete {$filename} manually");
exit;
}
}
protected function downloadPlanet(): string
{
$filename = sys_get_temp_dir().'/planet_coarse_boundaries.zip';
$this->ensureDeleted($filename);
$txtfile = sys_get_temp_dir().'/planet_coarse_boundaries.txt';
$this->ensureDeleted($txtfile);
$fp = fopen($filename, 'w+');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, PLANET_URL);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_TIMEOUT, 60000);
curl_exec($ch);
curl_close($ch);
fclose($fp);
// Unzip
$zip = new \ZipArchive();
$res = $zip->open($filename);
if (true === $res) {
$zip->extractTo(sys_get_temp_dir());
$zip->close();
} else {
$this->output->writeln('Failed to unzip planet file');
exit;
}
// Check if file exists
if (!file_exists($txtfile)) {
$this->output->writeln('Failed to find planet data file after unzip');
exit;
}
// Delete zip file
unlink($filename);
return $txtfile;
// Import the planet database
$this->places->importPlanet($datafile);
}
}

41
lib/Controller/OtherController.php
View File

@ -128,6 +128,16 @@ class OtherController extends GenericApiController
// Check for system perl
$status['perl'] = $this->getExecutableStatus(exec('which perl'));
// Get GIS status
$places = \OC::$server->get(\OCA\Memories\Service\Places::class);
try {
$status['gis_type'] = $places->detectGisType();
$status['gis_count'] = $places->geomCount();
} catch (\Exception $e) {
$status['gis_type'] = $e->getMessage();
}
// Check ffmpeg and ffprobe binaries
$status['ffmpeg'] = $this->getExecutableStatus(Util::getSystemConfig('memories.vod.ffmpeg'));
$status['ffprobe'] = $this->getExecutableStatus(Util::getSystemConfig('memories.vod.ffprobe'));
@ -158,6 +168,37 @@ class OtherController extends GenericApiController
});
}
/**
* @AdminRequired
*/
public function placesSetup(): Http\Response
{
try {
// Set PHP timeout to infinite
set_time_limit(0);
// Send headers for long-running request
header('Content-Type: text/plain');
header('X-Accel-Buffering: no');
header('Cache-Control: no-cache');
header('Connection: keep-alive');
header('Content-Length: 0');
$places = \OC::$server->get(\OCA\Memories\Service\Places::class);
echo "Downloading planet file...\n";
flush();
$datafile = $places->downloadPlanet();
$places->importPlanet($datafile);
echo "Done.\n";
} catch (\Exception $e) {
echo 'Failed: '.$e->getMessage()."\n";
}
exit;
}
/**
* @NoAdminRequired
*

357
lib/Service/Places.php 100644
View File

@ -0,0 +1,357 @@
<?php
namespace OCA\Memories\Service;
use OCP\IConfig;
use OCP\IDBConnection;
const GIS_TYPE_NONE = 0;
const GIS_TYPE_MYSQL = 1;
const GIS_TYPE_POSTGRES = 2;
const APPROX_PLACES = 635189;
const PLANET_URL = 'https://github.com/pulsejet/memories-assets/releases/download/geo-0.0.3/planet_coarse_boundaries.zip';
class Places
{
protected IDBConnection $db;
protected IConfig $config;
protected IDBConnection $connection;
public function __construct(
IConfig $config,
IDBConnection $connection
) {
$this->config = $config;
$this->connection = $connection;
}
/**
* Make SQL query to detect GIS type.
*/
public function detectGisType()
{
// Make sure database prefix is set
$prefix = $this->config->getSystemValue('dbtableprefix', '') ?: '';
if ('' === $prefix) {
throw new \Exception('Database table prefix is not set. Cannot use database exceptions (dbtableprefix).');
}
// Detect database type
$platform = strtolower(\get_class($this->connection->getDatabasePlatform()));
// Test MySQL-like support in databse
if (str_contains($platform, 'mysql') || str_contains($platform, 'mariadb')) {
try {
$res = $this->connection->executeQuery("SELECT ST_GeomFromText('POINT(1 1)')")->fetch();
if (0 === \count($res)) {
throw new \Exception('Invalid result');
}
return GIS_TYPE_MYSQL;
} catch (\Exception $e) {
throw new \Exception('No MySQL-like geometry support detected');
}
}
// Test Postgres native geometry like support in database
if (str_contains($platform, 'postgres')) {
try {
$res = $this->connection->executeQuery("SELECT POINT('1,1')")->fetch();
if (0 === \count($res)) {
throw new \Exception('Invalid result');
}
return GIS_TYPE_POSTGRES;
} catch (\Exception $e) {
throw new \Exception('No Postgres native geometry support detected');
}
}
return GIS_TYPE_NONE;
}
/**
* Check if DB is already setup and return number of entries.
*/
public function geomCount(): int
{
try {
return $this->connection->executeQuery('SELECT COUNT(osm_id) as c FROM memories_planet_geometry')->fetchOne();
} catch (\Exception $e) {
return 0;
}
}
/**
* Download planet database file and return path to it.
*/
public function downloadPlanet(): string
{
$filename = sys_get_temp_dir().'/planet_coarse_boundaries.zip';
unlink($filename);
$txtfile = sys_get_temp_dir().'/planet_coarse_boundaries.txt';
unlink($txtfile);
$fp = fopen($filename, 'w+');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, PLANET_URL);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_TIMEOUT, 3600);
curl_exec($ch);
curl_close($ch);
fclose($fp);
// Unzip
$zip = new \ZipArchive();
$res = $zip->open($filename);
if (true === $res) {
$zip->extractTo(sys_get_temp_dir());
$zip->close();
} else {
throw new \Exception('Failed to unzip planet data file');
}
// Check if file exists
if (!file_exists($txtfile)) {
throw new \Exception('Failed to find planet data file after unzip');
}
// Delete zip file
unlink($filename);
return $txtfile;
}
/**
* Insert planet into database from file.
*/
public function importPlanet(string $datafile): void
{
echo "Inserting planet data into database...\n";
// Detect the GIS type
$gis = $this->detectGisType();
// Make sure we support something
if (GIS_TYPE_NONE === $gis) {
throw new \Exception('No GIS support detected');
}
// Drop the table if it exists
$p = $this->connection->getDatabasePlatform();
if ($this->geomCount() > 0) {
$this->connection->executeStatement($p->getDropTableSQL('memories_planet_geometry'));
}
// Setup the database
$this->setupDatabase();
// Truncate tables
$this->connection->executeStatement($p->getTruncateTableSQL('*PREFIX*memories_planet', false));
$this->connection->executeStatement($p->getTruncateTableSQL('memories_planet_geometry', false));
// Create place insertion statement
$query = $this->connection->getQueryBuilder();
$query->insert('memories_planet')
->values([
'osm_id' => $query->createParameter('osm_id'),
'admin_level' => $query->createParameter('admin_level'),
'name' => $query->createParameter('name'),
'other_names' => $query->createParameter('other_names'),
])
;
$insertPlace = $this->connection->prepare($query->getSQL());
// Create geometry insertion statement
$query = $this->connection->getQueryBuilder();
$geomParam = $query->createParameter('geometry');
if (GIS_TYPE_MYSQL === $gis) {
$geomParam = "ST_GeomFromText({$geomParam})";
} elseif (GIS_TYPE_POSTGRES === $gis) {
$geomParam = "POLYGON({$geomParam}::text)";
}
$query->insert('memories_planet_geometry')
->values([
'id' => $query->createParameter('id'),
'poly_id' => $query->createParameter('poly_id'),
'type_id' => $query->createParameter('type_id'),
'osm_id' => $query->createParameter('osm_id'),
'geometry' => $query->createFunction($geomParam),
])
;
$sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL());
$insertGeometry = $this->connection->prepare($sql);
// The number of places in the current transaction
$txnCount = 0;
// Iterate over the data file
$handle = fopen($datafile, 'r');
if ($handle) {
$count = 0;
while (($line = fgets($handle)) !== false) {
// Skip empty lines
if ('' === trim($line)) {
continue;
}
// Begin transaction
if (0 === $txnCount++) {
$this->connection->beginTransaction();
}
++$count;
// Decode JSON
$data = json_decode($line, true);
if (null === $data) {
echo "ERROR: Failed to decode JSON\n";
continue;
}
// Extract data
$osmId = $data['osm_id'];
$adminLevel = $data['admin_level'];
$name = $data['name'];
$boundaries = $data['geometry'];
$otherNames = json_encode($data['other_names'] ?? []);
// Skip some places
if ($adminLevel > -2 && ($adminLevel <= 1 || $adminLevel >= 10)) {
// <=1: These are too general, e.g. "Earth"? or invalid
// >=10: These are too specific, e.g. "Community Board"
// <-1: These are special, e.g. "Timezone" = -7
continue;
}
// Insert place into database
$insertPlace->bindValue('osm_id', $osmId);
$insertPlace->bindValue('admin_level', $adminLevel);
$insertPlace->bindValue('name', $name);
$insertPlace->bindValue('other_names', $otherNames);
$insertPlace->execute();
// Insert polygons into database
$idx = 0;
foreach ($boundaries as &$polygon) {
// $polygon is a struct as
// [ "t" => "e", "c" => [lon, lat], [lon, lat], ... ] ]
$polyid = $polygon['i'];
$typeid = $polygon['t'];
$pkey = $polygon['k'];
$coords = $polygon['c'];
// Create parameters
++$idx;
$geometry = '';
if (\count($coords) < 3) {
echo "ERROR: Invalid polygon {$polyid}\n";
continue;
}
if (GIS_TYPE_MYSQL === $gis) {
$points = implode(',', array_map(function ($point) {
$x = $point[0];
$y = $point[1];
return "{$x} {$y}";
}, $coords));
$geometry = "POLYGON(({$points}))";
} elseif (GIS_TYPE_POSTGRES === $gis) {
$geometry = implode(',', array_map(function ($point) {
$x = $point[0];
$y = $point[1];
return "({$x},{$y})";
}, $coords));
}
try {
$insertGeometry->bindValue('id', $pkey);
$insertGeometry->bindValue('poly_id', $polyid);
$insertGeometry->bindValue('type_id', $typeid);
$insertGeometry->bindValue('osm_id', $osmId);
$insertGeometry->bindValue('geometry', $geometry);
$insertGeometry->execute();
} catch (\Exception $e) {
echo "ERROR: Failed to insert polygon {$polyid} ({$e->getMessage()} \n";
continue;
}
}
// Commit transaction every once in a while
if (0 === $count % 100) {
$this->connection->commit();
$txnCount = 0;
// Print progress
$total = APPROX_PLACES;
$pct = round($count / $total * 100, 1);
}
if (0 === $count % 500) {
echo "Inserted {$count} / {$total} places ({$pct}%), Last: {$name}\n";
flush();
}
}
fclose($handle);
}
// Commit final transaction
if ($txnCount > 0) {
$this->connection->commit();
}
// Mark success
echo "Planet database imported successfully!\n";
$this->config->setSystemValue('memories.gis_type', $gis);
// Delete data file
unlink($datafile);
}
/**
* Create database tables and indices.
*/
private function setupDatabase(): void
{
try {
// Get Gis type
$gis = $this->detectGisType();
// Create table
$sql = 'CREATE TABLE memories_planet_geometry (
id varchar(255) NOT NULL PRIMARY KEY,
poly_id varchar(255) NOT NULL,
type_id int NOT NULL,
osm_id int NOT NULL,
geometry polygon NOT NULL
);';
$this->connection->executeQuery($sql);
// Add indexes
$this->connection->executeQuery('CREATE INDEX planet_osm_id_idx ON memories_planet_geometry (osm_id);');
// Add spatial index
if (GIS_TYPE_MYSQL === $gis) {
$this->connection->executeQuery('CREATE SPATIAL INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry (geometry);');
} elseif (GIS_TYPE_POSTGRES === $gis) {
$this->connection->executeQuery('CREATE INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry USING GIST (geometry);');
}
} catch (\Exception $e) {
throw new \Exception('Failed to create database tables: '.$e->getMessage());
}
}
}

128
src/Admin.vue
View File

@ -2,6 +2,7 @@
<div class="outer" v-if="loaded">
<NcLoadingIcon class="loading-icon" v-show="loading" />
<!----------------------------- General Settings ----------------------------->
<h2>{{ t("memories", "EXIF Extraction") }}</h2>
<template v-if="status">
@ -34,6 +35,70 @@
}}
</NcCheckboxRadioSwitch>
<!----------------------------- Places ----------------------------->
<h2>{{ t("memories", "Reverse Geocoding") }}</h2>
<p>
<template v-if="status">
<NcNoteCard :type="gisStatusType">
{{ gisStatus }}
</NcNoteCard>
<NcNoteCard
v-if="status.gis_count !== undefined"
:type="status.gis_count > 0 ? 'success' : 'warning'"
>
{{
status.gis_count > 0
? t("memories", "Database is populated with {n} geometries", {
n: status.gis_count,
})
: t("memories", "Geometry table has not been created")
}}
</NcNoteCard>
</template>
{{
t(
"memories",
"Memories supports offline reverse geocoding using the OpenStreetMaps data on MySQL and Postgres."
)
}}
<br />
{{
t(
"memories",
"You need to download the planet data into your database. This is highly recommended and has low overhead."
)
}}
<br />
{{
t(
"memories",
"If the button below does not work for importing the planet data, use 'occ memories:places-setup'."
)
}}
<br />
{{
t(
"memories",
"Note: the geometry data is stored in the 'memories_planet_geometry' table, with no prefix."
)
}}
</p>
<form
:action="placesSetupUrl"
method="post"
@submit="placesSetup"
target="_blank"
>
<input name="requesttoken" type="hidden" :value="requestToken" />
<NcButton nativeType="submit" type="warning">
{{ t("memories", "Download planet database") }}
</NcButton>
</form>
<!----------------------------- Video Streaming ----------------------------->
<h2>{{ t("memories", "Video Streaming") }}</h2>
<p>
@ -207,8 +272,8 @@
VA-API configuration
</a>
<NcNoteCard :type="vaapiStatusType()" v-if="status">
{{ vaapiStatusText() }}
<NcNoteCard :type="vaapiStatusType" v-if="status">
{{ vaapiStatusText }}
</NcNoteCard>
<NcCheckboxRadioSwitch
@ -299,6 +364,7 @@ const NcCheckboxRadioSwitch = () =>
const NcNoteCard = () => import("@nextcloud/vue/dist/Components/NcNoteCard");
const NcTextField = () => import("@nextcloud/vue/dist/Components/NcTextField");
import NcLoadingIcon from "@nextcloud/vue/dist/Components/NcLoadingIcon";
import NcButton from "@nextcloud/vue/dist/Components/NcButton";
/** Map from UI to backend settings */
const settings = {
@ -328,6 +394,8 @@ const invertedBooleans = ["enableTranscoding"];
type BinaryStatus = "ok" | "not_found" | "not_executable" | "test_ok" | string;
type IStatus = {
gis_type: number;
gis_count?: number;
exiftool: BinaryStatus;
perl: BinaryStatus;
ffmpeg: BinaryStatus;
@ -343,6 +411,7 @@ export default defineComponent({
NcNoteCard,
NcTextField,
NcLoadingIcon,
NcButton,
},
data: () => ({
@ -441,6 +510,20 @@ export default defineComponent({
});
},
placesSetup(event: Event) {
const msg =
"Looks like the database is already setup. Are you sure you want to drop the table and redownload OSM data?";
if (this.status.gis_count && !confirm(msg)) {
event.preventDefault();
event.stopPropagation();
return;
} else {
alert(
"Please wait for the download and insertion to complete. This may take a while."
);
}
},
binaryStatus(name: string, status: BinaryStatus): string {
if (status === "ok") {
return this.t("memories", "{name} binary exists and is executable", {
@ -491,6 +574,43 @@ export default defineComponent({
return "warning";
}
},
},
computed: {
requestToken() {
return (<any>axios.defaults.headers).requesttoken;
},
gisStatus() {
if (typeof this.status.gis_type !== "number") {
return this.status.gis_type;
}
if (this.status.gis_type <= 0) {
return this.t(
"memories",
"Geometry support was not detected in your database"
);
} else if (this.status.gis_type === 1) {
return this.t("memories", "MySQL-like geometry support was detected ");
} else if (this.status.gis_type === 2) {
return this.t(
"memories",
"Postgres native geometry support was detected"
);
}
},
gisStatusType() {
return typeof this.status.gis_type !== "number" ||
this.status.gis_type <= 0
? "error"
: "success";
},
placesSetupUrl() {
return API.OCC_PLACES_SETUP();
},
vaapiStatusText(): string {
const dev = "/dev/dri/renderD128";
@ -536,6 +656,10 @@ export default defineComponent({
}
}
form {
margin-top: 1em;
}
.checkbox-radio-switch {
margin: 2px 8px;
}

4
src/services/API.ts
View File

@ -194,6 +194,10 @@ export class API {
return gen(`${BASE}/system-status`);
}
static OCC_PLACES_SETUP() {
return gen(`${BASE}/occ/places-setup`);
}
static MAP_CLUSTERS() {
return tok(gen(`${BASE}/map/clusters`));
}