Add WIP table creation
Varun Patil
parent
62579b1b89
commit
24e70a7e06
|
|
@ -0,0 +1,282 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* @copyright Copyright (c) 2022, Varun Patil <radialapps@gmail.com>
|
||||
* @author Varun Patil <radialapps@gmail.com>
|
||||
* @license AGPL-3.0-or-later
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
namespace OCA\Memories\Command;
|
||||
|
||||
use OCP\IConfig;
|
||||
use OCP\IDBConnection;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
const GIS_TYPE_NONE = 0;
|
||||
const GIS_TYPE_MYSQL = 1;
|
||||
const GIS_TYPE_POSTGRES = 2;
|
||||
const APPROX_PLACES = 600000;
|
||||
|
||||
class GeoSetup extends Command
|
||||
{
|
||||
protected IConfig $config;
|
||||
protected OutputInterface $output;
|
||||
protected IDBConnection $connection;
|
||||
|
||||
protected int $gisType = GIS_TYPE_NONE;
|
||||
|
||||
public function __construct(
|
||||
IConfig $config,
|
||||
IDBConnection $connection
|
||||
) {
|
||||
parent::__construct();
|
||||
$this->config = $config;
|
||||
$this->connection = $connection;
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->setName('memories:geo-setup')
|
||||
->setDescription('Setup reverse geocoding')
|
||||
;
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$this->output = $output;
|
||||
|
||||
$this->output->writeln('Attempting to set up reverse geocoding');
|
||||
|
||||
// Detect the GIS type
|
||||
$this->detectGisType();
|
||||
$this->config->setSystemValue('memories.gis_type', $this->gisType);
|
||||
|
||||
// Make sure we support something
|
||||
if (GIS_TYPE_NONE === $this->gisType) {
|
||||
$this->output->writeln('<error>No supported GIS type detected</error>');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Check if the database is already set up
|
||||
try {
|
||||
$this->connection->executeQuery('SELECT osm_id FROM memories_planet_geometry LIMIT 1')->fetch();
|
||||
$this->output->writeln('Database already set up ... skipping');
|
||||
} catch (\Exception $e) {
|
||||
$this->output->writeln('Setting up database ...');
|
||||
$this->setupDatabase();
|
||||
$this->output->writeln('Database set up');
|
||||
}
|
||||
|
||||
// TODO: Download the data
|
||||
// TODO: Warn user and truncate all tables
|
||||
$datafile = '/tmp/planet_coarse_boundaries.txt';
|
||||
|
||||
// Truncate tables
|
||||
$this->output->writeln('Truncating tables ...');
|
||||
$p = $this->connection->getDatabasePlatform();
|
||||
$t1 = $p->getTruncateTableSQL('*PREFIX*memories_planet', false);
|
||||
$t2 = $p->getTruncateTableSQL('memories_planet_geometry', false);
|
||||
$this->connection->executeStatement("{$t1}; {$t2}");
|
||||
|
||||
// Start time
|
||||
$start = time();
|
||||
|
||||
$handle = fopen($datafile, 'r');
|
||||
if ($handle) {
|
||||
$count = 0;
|
||||
while (($line = fgets($handle)) !== false) {
|
||||
// Skip empty lines
|
||||
if ('' === trim($line)) {
|
||||
continue;
|
||||
}
|
||||
++$count;
|
||||
|
||||
// Decode JSON
|
||||
$data = json_decode($line, true);
|
||||
if (null === $data) {
|
||||
$this->output->writeln('<error>Failed to decode JSON</error>');
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract data
|
||||
$osmId = $data['osm_id'];
|
||||
$adminLevel = $data['admin_level'];
|
||||
$name = $data['name'];
|
||||
$boundaries = $data['geometry'];
|
||||
|
||||
// Insert place into database
|
||||
$query = $this->connection->getQueryBuilder();
|
||||
$query->insert('memories_planet')
|
||||
->values([
|
||||
'osm_id' => $query->createNamedParameter($osmId),
|
||||
'admin_level' => $query->createNamedParameter($adminLevel),
|
||||
'name' => $query->createNamedParameter($name),
|
||||
])
|
||||
;
|
||||
$query->executeStatement();
|
||||
|
||||
// Insert polygons into database
|
||||
$idx = 0;
|
||||
foreach ($boundaries as &$polygon) {
|
||||
// $boundary is a list of points
|
||||
// [ [lon, lat], [lon, lat], ... ]
|
||||
++$idx;
|
||||
$pkey = $osmId.'-'.$idx;
|
||||
$geometry = '';
|
||||
|
||||
if (\count($polygon) < 3) {
|
||||
$this->output->writeln('<error>Invalid polygon</error>');
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$query = $this->connection->getQueryBuilder();
|
||||
|
||||
if (GIS_TYPE_MYSQL === $this->gisType) {
|
||||
$points = array_map(function ($point) {
|
||||
return $point[0].' '.$point[1];
|
||||
}, $polygon);
|
||||
$geometry = implode(',', $points);
|
||||
|
||||
$geometry = 'POLYGON(('.$geometry.'))';
|
||||
$geometry = 'ST_GeomFromText(\''.$geometry.'\')';
|
||||
} elseif (GIS_TYPE_POSTGRES === $this->gisType) {
|
||||
$points = array_map(function ($point) {
|
||||
return '('.$point[0].','.$point[1].')';
|
||||
}, $polygon);
|
||||
$geometry = implode(',', $points);
|
||||
$geometry = 'POLYGON(\''.$geometry.'\')';
|
||||
}
|
||||
|
||||
try {
|
||||
$query->insert('memories_planet_geometry')
|
||||
->values([
|
||||
'id' => $query->createNamedParameter($pkey),
|
||||
'osm_id' => $query->createNamedParameter($osmId),
|
||||
'geometry' => $query->createFunction($geometry),
|
||||
])
|
||||
;
|
||||
$sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $query->getSQL());
|
||||
$this->connection->executeQuery($sql, $query->getParameters());
|
||||
} catch (\Exception $e) {
|
||||
$this->output->writeln('<error>Failed to insert into database</error>');
|
||||
$this->output->writeln($e->getMessage());
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Print progress
|
||||
if (0 === $count % 1000) {
|
||||
$end = time();
|
||||
$elapsed = $end - $start;
|
||||
$rate = $count / $elapsed;
|
||||
$remaining = APPROX_PLACES - $count;
|
||||
$eta = $remaining / $rate;
|
||||
$rate = round($rate, 2);
|
||||
$this->output->writeln("Inserted {$count} places, {$rate} per second, ETA: {$eta} seconds");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected function detectGisType()
|
||||
{
|
||||
// Test MySQL-like support in databse
|
||||
try {
|
||||
$res = $this->connection->executeQuery("SELECT ST_GeomFromText('POINT(1 1)')")->fetch();
|
||||
if (0 === \count($res)) {
|
||||
throw new \Exception('Invalid result');
|
||||
}
|
||||
$this->output->writeln('MySQL-like support detected!');
|
||||
|
||||
// Make sure this is actually MySQL
|
||||
$res = $this->connection->executeQuery('SELECT VERSION()')->fetch();
|
||||
if (0 === \count($res)) {
|
||||
throw new \Exception('Invalid result');
|
||||
}
|
||||
if (false === strpos($res['VERSION()'], 'MariaDB') && false === strpos($res['VERSION()'], 'MySQL')) {
|
||||
throw new \Exception('MySQL not detected');
|
||||
}
|
||||
|
||||
$this->gisType = GIS_TYPE_MYSQL;
|
||||
} catch (\Exception $e) {
|
||||
$this->output->writeln('No MySQL-like support detected');
|
||||
}
|
||||
|
||||
// Test Postgres native geometry like support in database
|
||||
if (GIS_TYPE_NONE === $this->gisType) {
|
||||
try {
|
||||
$res = $this->connection->executeQuery("SELECT POINT('1,1')")->fetch();
|
||||
if (0 === \count($res)) {
|
||||
throw new \Exception('Invalid result');
|
||||
}
|
||||
$this->output->writeln('Postgres native geometry support detected!');
|
||||
$this->gisType = GIS_TYPE_POSTGRES;
|
||||
} catch (\Exception $e) {
|
||||
$this->output->writeln('No Postgres native geometry support detected');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected function setupDatabase(): void
|
||||
{
|
||||
try {
|
||||
$sql = 'CREATE TABLE memories_planet_geometry (
|
||||
id varchar(255) NOT NULL PRIMARY KEY,
|
||||
osm_id int NOT NULL,
|
||||
geometry polygon NOT NULL
|
||||
);';
|
||||
$this->connection->executeQuery($sql);
|
||||
|
||||
// Add indexes
|
||||
$this->connection->executeQuery('CREATE INDEX planet_osm_id_idx ON memories_planet_geometry (osm_id);');
|
||||
|
||||
// Add spatial index
|
||||
if (GIS_TYPE_MYSQL === $this->gisType) {
|
||||
$this->connection->executeQuery('CREATE SPATIAL INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry (geometry);');
|
||||
} elseif (GIS_TYPE_POSTGRES === $this->gisType) {
|
||||
$this->connection->executeQuery('CREATE INDEX planet_osm_polygon_geometry_idx ON memories_planet_geometry USING GIST (geometry);');
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$this->output->writeln('Failed to create planet table');
|
||||
$this->output->writeln($e->getMessage());
|
||||
|
||||
exit;
|
||||
}
|
||||
}
|
||||
|
||||
protected function runSQL(string &$line)
|
||||
{
|
||||
try {
|
||||
$this->connection->executeStatement($line);
|
||||
} catch (\Exception $e) {
|
||||
$this->output->writeln(substr($line, 0, 100));
|
||||
$this->output->writeln($e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,9 +14,9 @@ class TimelineQuery
|
|||
use TimelineQueryFilters;
|
||||
use TimelineQueryFolders;
|
||||
use TimelineQueryLivePhoto;
|
||||
use TimelineQueryPlaces;
|
||||
use TimelineQueryPeopleFaceRecognition;
|
||||
use TimelineQueryPeopleRecognize;
|
||||
use TimelineQueryPlaces;
|
||||
use TimelineQueryTags;
|
||||
|
||||
protected IDBConnection $connection;
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ trait TimelineQueryPlaces
|
|||
|
||||
public function transformPlaceFilter(IQueryBuilder &$query, string $userId, int $locationId)
|
||||
{
|
||||
$query->innerJoin('m', 'memories_geo', 'mg', $query->expr()->andX(
|
||||
$query->expr()->eq('mg.fileid', 'm.fileid'),
|
||||
$query->expr()->eq('mg.osm_id', $query->createNamedParameter($locationId)),
|
||||
$query->innerJoin('m', 'memories_places', 'mp', $query->expr()->andX(
|
||||
$query->expr()->eq('mp.fileid', 'm.fileid'),
|
||||
$query->expr()->eq('mp.osm_id', $query->createNamedParameter($locationId)),
|
||||
));
|
||||
}
|
||||
|
||||
|
|
@ -25,25 +25,24 @@ trait TimelineQueryPlaces
|
|||
|
||||
// SELECT location name and count of photos
|
||||
$count = $query->func()->count($query->createFunction('DISTINCT m.fileid'), 'count');
|
||||
$query->select('p.osm_id', 'p.name', $count)->from('memories_planet', 'p');
|
||||
$query->select('e.osm_id', 'e.name', $count)->from('memories_planet', 'e');
|
||||
|
||||
// WHERE there are items with this osm_id
|
||||
$query->innerJoin('p', 'memories_geo', 'mg', $query->expr()->eq('mg.osm_id', 'p.osm_id'));
|
||||
$query->innerJoin('e', 'memories_places', 'mp', $query->expr()->eq('mp.osm_id', 'e.osm_id'));
|
||||
|
||||
// WHERE these items are memories indexed photos
|
||||
$query->innerJoin('mg', 'memories', 'm', $query->expr()->eq('m.fileid', 'mg.fileid'));
|
||||
$query->innerJoin('mp', 'memories', 'm', $query->expr()->eq('m.fileid', 'mp.fileid'));
|
||||
|
||||
// WHERE these photos are in the user's requested folder recursively
|
||||
$query = $this->joinFilecache($query, $root, true, false);
|
||||
|
||||
// GROUP and ORDER by tag name
|
||||
$query->groupBy('p.osm_id');
|
||||
$query->orderBy($query->createFunction('LOWER(p.name)'), 'ASC');
|
||||
$query->addOrderBy('p.osm_id'); // tie-breaker
|
||||
$query->groupBy('e.osm_id');
|
||||
$query->orderBy($query->createFunction('LOWER(e.name)'), 'ASC');
|
||||
$query->addOrderBy('e.osm_id'); // tie-breaker
|
||||
|
||||
// FETCH all tags
|
||||
$sql = str_replace('*PREFIX*memories_planet', 'memories_planet', $query->getSQL());
|
||||
$cursor = $this->executeQueryWithCTEs($query, $sql);
|
||||
$cursor = $this->executeQueryWithCTEs($query);
|
||||
$places = $cursor->fetchAll();
|
||||
|
||||
// Post process
|
||||
|
|
@ -60,12 +59,12 @@ trait TimelineQueryPlaces
|
|||
$query = $this->connection->getQueryBuilder();
|
||||
|
||||
// SELECT all photos with this tag
|
||||
$query->select('f.fileid', 'f.etag')->from('memories_geo', 'mg')
|
||||
->where($query->expr()->eq('mg.osm_id', $query->createNamedParameter($id)))
|
||||
$query->select('f.fileid', 'f.etag')->from('memories_places', 'mp')
|
||||
->where($query->expr()->eq('mp.osm_id', $query->createNamedParameter($id)))
|
||||
;
|
||||
|
||||
// WHERE these items are memories indexed photos
|
||||
$query->innerJoin('mg', 'memories', 'm', $query->expr()->eq('m.fileid', 'mg.fileid'));
|
||||
$query->innerJoin('mp', 'memories', 'm', $query->expr()->eq('m.fileid', 'mp.fileid'));
|
||||
|
||||
// WHERE these photos are in the user's requested folder recursively
|
||||
$query = $this->joinFilecache($query, $root, true, false);
|
||||
|
|
|
|||
|
|
@ -228,8 +228,8 @@ class TimelineWrite
|
|||
public function clear()
|
||||
{
|
||||
$p = $this->connection->getDatabasePlatform();
|
||||
$t1 = $p->getTruncateTableSQL('`*PREFIX*memories`', false);
|
||||
$t2 = $p->getTruncateTableSQL('`*PREFIX*memories_livephoto`', false);
|
||||
$t1 = $p->getTruncateTableSQL('*PREFIX*memories', false);
|
||||
$t2 = $p->getTruncateTableSQL('*PREFIX*memories_livephoto', false);
|
||||
$this->connection->executeStatement("{$t1}; {$t2}");
|
||||
}
|
||||
|
||||
|
|
@ -283,14 +283,14 @@ class TimelineWrite
|
|||
{
|
||||
// Make query to memories_planet table
|
||||
$query = $this->connection->getQueryBuilder();
|
||||
$query->select('osm_id')
|
||||
->from('memories_planet')
|
||||
->where($query->createFunction('ST_Contains(`geometry`, ST_GeomFromText(\'POINT('.$lon.' '.$lat.')\', 4326))'))
|
||||
$query->select($query->createFunction('DISTINCT(osm_id)'))
|
||||
->from('memories_planet_geometry')
|
||||
->where($query->createFunction('ST_Contains(`geometry`, ST_GeomFromText(\'POINT('.$lon.' '.$lat.')\'))'))
|
||||
;
|
||||
|
||||
// Remove memories_planet has no *PREFIX*
|
||||
$sql = $query->getSQL();
|
||||
$sql = str_replace('*PREFIX*memories_planet', 'memories_planet', $sql);
|
||||
$sql = str_replace('*PREFIX*memories_planet_geometry', 'memories_planet_geometry', $sql);
|
||||
|
||||
// Run query
|
||||
$result = $this->connection->executeQuery($sql);
|
||||
|
|
@ -298,14 +298,14 @@ class TimelineWrite
|
|||
|
||||
// Delete previous records
|
||||
$query = $this->connection->getQueryBuilder();
|
||||
$query->delete('memories_geo')
|
||||
$query->delete('memories_places')
|
||||
->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT)))
|
||||
;
|
||||
|
||||
// Insert records
|
||||
foreach ($rows as $row) {
|
||||
$query = $this->connection->getQueryBuilder();
|
||||
$query->insert('memories_geo')
|
||||
$query->insert('memories_places')
|
||||
->values([
|
||||
'fileid' => $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT),
|
||||
'osm_id' => $query->createNamedParameter($row['osm_id'], IQueryBuilder::PARAM_INT),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* @copyright Copyright (c) 2023 Your name <your@email.com>
|
||||
* @author Your name <your@email.com>
|
||||
* @license GNU AGPL version 3 or any later version
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
namespace OCA\Memories\Migration;
|
||||
|
||||
use OCP\DB\ISchemaWrapper;
|
||||
use OCP\DB\Types;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\SimpleMigrationStep;
|
||||
|
||||
/**
|
||||
* Auto-generated migration step: Please modify to your needs!
|
||||
*/
|
||||
class Version401100Date20230206002744 extends SimpleMigrationStep
|
||||
{
|
||||
/**
|
||||
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
|
||||
*/
|
||||
public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
|
||||
*/
|
||||
public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
|
||||
{
|
||||
/** @var ISchemaWrapper $schema */
|
||||
$schema = $schemaClosure();
|
||||
|
||||
if (!$schema->hasTable('memories_places')) {
|
||||
$table = $schema->createTable('memories_places');
|
||||
$table->addColumn('id', 'integer', [
|
||||
'autoincrement' => true,
|
||||
'notnull' => true,
|
||||
]);
|
||||
$table->addColumn('fileid', Types::BIGINT, [
|
||||
'notnull' => true,
|
||||
'length' => 20,
|
||||
]);
|
||||
$table->addColumn('osm_id', Types::INTEGER, [
|
||||
'notnull' => true,
|
||||
]);
|
||||
|
||||
$table->setPrimaryKey(['id']);
|
||||
$table->addIndex(['fileid'], 'memories_fileid_index');
|
||||
$table->addIndex(['osm_id'], 'memories_osm_id_index');
|
||||
}
|
||||
|
||||
if (!$schema->hasTable('memories_planet')) {
|
||||
$table = $schema->createTable('memories_planet');
|
||||
$table->addColumn('id', 'integer', [
|
||||
'autoincrement' => true,
|
||||
'notnull' => true,
|
||||
]);
|
||||
$table->addColumn('osm_id', Types::INTEGER, [
|
||||
'notnull' => true,
|
||||
]);
|
||||
$table->addColumn('name', Types::TEXT, [
|
||||
'notnull' => true,
|
||||
]);
|
||||
$table->addColumn('admin_level', Types::INTEGER, [
|
||||
'notnull' => true,
|
||||
]);
|
||||
|
||||
$table->setPrimaryKey(['id']);
|
||||
$table->addIndex(['osm_id'], 'memories_osm_id_index');
|
||||
}
|
||||
|
||||
return $schema;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
|
||||
*/
|
||||
public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
|
||||
{
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue