RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

place-setup: recalculate places 

Signed-off-by: Varun Patil <varunpatil@ucla.edu>
pull/579/head
Varun Patil 2023-04-16 19:53:06 -07:00
parent 7a733d3d11
commit 526559b672
6 changed files with 173 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/Command/PlacesSetup.php
View File

@ -88,10 +88,16 @@ class PlacesSetup extends Command
}
// Download the planet database
$this->output->writeln('Downloading planet database');
$datafile = $this->places->downloadPlanet();
// Import the planet database
$this->places->importPlanet($datafile);
// Recalculate all places
$this->places->recalculateAll();
$this->output->writeln('Done');
return 0;
}
}

4
lib/Controller/OtherController.php
View File

@ -205,11 +205,9 @@ class OtherController extends GenericApiController
header('Content-Length: 0');
$places = \OC::$server->get(\OCA\Memories\Service\Places::class);
echo "Downloading planet file...\n";
flush();
$datafile = $places->downloadPlanet();
$places->importPlanet($datafile);
$places->recalculateAll();
echo "Done.\n";
} catch (\Exception $e) {

67
lib/Db/TimelineWriteOrphans.php
View File

@ -12,21 +12,78 @@ trait TimelineWriteOrphans
protected IDBConnection $connection;
/**
* Mark all files in the table as orphaned.
* Mark all or some files in the table as (un)orphaned.
*
* @param bool $value True to mark as orphaned, false to mark as un-orphaned
* @param int[] $fileIds List of file IDs to mark, or empty to mark all files
* @param bool $onlyMain Only mark the main file, not the live photo
*
* @return int Number of rows affected
*/
public function orphanAll(): int
public function orphanAll(bool $value = true, ?array $fileIds = null, bool $onlyMain = false): int
{
$do = function (string $table) {
$do = function (string $table) use ($value, $fileIds) {
$query = $this->connection->getQueryBuilder();
$query->update($table)
->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
->set('orphan', $query->createNamedParameter($value, IQueryBuilder::PARAM_BOOL))
;
if ($fileIds) {
$query->where($query->expr()->in('fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
}
return $query->executeStatement();
};
return $do('memories') + $do('memories_livephoto');
$count = $do('memories');
if ($onlyMain) {
return $count;
}
return $count + $do('memories_livephoto');
}
/**
* Orphan and run an update on all files.
*
* @param array $fields list of fields to select
* @param int $txnSize number of rows to process in a single transaction
* @param \Closure $callback will be passed each row
*/
public function orphanAndRun(array $fields, int $txnSize, \Closure $callback)
{
// Orphan all files. This means if we are interrupted,
// it will lead to a re-index of the whole library!
$this->orphanAll(true, null, true);
while (\count($orphans = $this->getSomeOrphans($txnSize, $fields))) {
$this->connection->beginTransaction();
foreach ($orphans as $row) {
$callback($row);
}
// Mark all files as not orphaned.
$fileIds = array_map(fn ($row) => $row['fileid'], $orphans);
$this->orphanAll(false, $fileIds, true);
$this->connection->commit();
}
}
/**
* Get a list of orphaned files.
*/
protected function getSomeOrphans(int $count, array $fields): array
{
$query = $this->connection->getQueryBuilder();
$query->select(...$fields)
->from('memories')
->where($query->expr()->eq('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)))
->setMaxResults($count)
;
return $query->executeQuery()->fetchAll();
}
}

96
lib/Db/TimelineWritePlaces.php
View File

@ -15,53 +15,6 @@ trait TimelineWritePlaces
{
protected IDBConnection $connection;
/**
* Process the location part of exif data.
*
* Also update the exif data with the tzid from location (LocationTZID)
* Performs an in-place update of the exif data.
*
* @param int $fileId The file ID
* @param array $exif The exif data (will change)
* @param array|bool $prevRow The previous row of data
*
* @return array Update values
*/
protected function processExifLocation(int $fileId, array &$exif, $prevRow): array
{
// Store location data
[$lat, $lon] = self::readCoord($exif);
$oldLat = $prevRow ? (float) $prevRow['lat'] : null;
$oldLon = $prevRow ? (float) $prevRow['lon'] : null;
$mapCluster = $prevRow ? (int) $prevRow['mapcluster'] : -1;
$osmIds = [];
if ($lat || $lon || $oldLat || $oldLon) {
try {
$mapCluster = $this->mapGetCluster($mapCluster, $lat, $lon, $oldLat, $oldLon);
} catch (\Exception $e) {
$logger = \OC::$server->get(LoggerInterface::class);
$logger->log(3, 'Error updating map cluster data: '.$e->getMessage(), ['app' => 'memories']);
}
try {
$osmIds = $this->updatePlacesData($fileId, $lat, $lon);
} catch (\Exception $e) {
$logger = \OC::$server->get(LoggerInterface::class);
$logger->log(3, 'Error updating places data: '.$e->getMessage(), ['app' => 'memories']);
}
}
// NULL if invalid
$mapCluster = $mapCluster <= 0 ? null : $mapCluster;
// Set tzid from location if not present
$this->setTzidFromLocation($exif, $osmIds);
// Return update values
return [$lat, $lon, $mapCluster, $osmIds];
}
/**
* Add places data for a file.
*
@ -71,7 +24,7 @@ trait TimelineWritePlaces
*
* @return array The list of osm_id of the places
*/
protected function updatePlacesData(int $fileId, $lat, $lon): array
public function updatePlacesData(int $fileId, $lat, $lon): array
{
// Get GIS type
$gisType = \OCA\Memories\Util::placesGISType();
@ -140,6 +93,53 @@ trait TimelineWritePlaces
return array_map(fn ($row) => $row['osm_id'], $rows);
}
/**
* Process the location part of exif data.
*
* Also update the exif data with the tzid from location (LocationTZID)
* Performs an in-place update of the exif data.
*
* @param int $fileId The file ID
* @param array $exif The exif data (will change)
* @param array|bool $prevRow The previous row of data
*
* @return array Update values
*/
protected function processExifLocation(int $fileId, array &$exif, $prevRow): array
{
// Store location data
[$lat, $lon] = self::readCoord($exif);
$oldLat = $prevRow ? (float) $prevRow['lat'] : null;
$oldLon = $prevRow ? (float) $prevRow['lon'] : null;
$mapCluster = $prevRow ? (int) $prevRow['mapcluster'] : -1;
$osmIds = [];
if ($lat || $lon || $oldLat || $oldLon) {
try {
$mapCluster = $this->mapGetCluster($mapCluster, $lat, $lon, $oldLat, $oldLon);
} catch (\Exception $e) {
$logger = \OC::$server->get(LoggerInterface::class);
$logger->log(3, 'Error updating map cluster data: '.$e->getMessage(), ['app' => 'memories']);
}
try {
$osmIds = $this->updatePlacesData($fileId, $lat, $lon);
} catch (\Exception $e) {
$logger = \OC::$server->get(LoggerInterface::class);
$logger->log(3, 'Error updating places data: '.$e->getMessage(), ['app' => 'memories']);
}
}
// NULL if invalid
$mapCluster = $mapCluster <= 0 ? null : $mapCluster;
// Set tzid from location if not present
$this->setTzidFromLocation($exif, $osmIds);
// Return update values
return [$lat, $lon, $mapCluster, $osmIds];
}
/**
* Set timezone offset from location if not present.
*

43
lib/Service/Places.php
View File

@ -2,6 +2,7 @@
namespace OCA\Memories\Service;
use OCA\Memories\Db\TimelineWrite;
use OCP\IConfig;
use OCP\IDBConnection;
@ -18,13 +19,16 @@ class Places
protected IConfig $config;
protected IDBConnection $connection;
protected TimelineWrite $timelineWrite;
public function __construct(
IConfig $config,
IDBConnection $connection
IDBConnection $connection,
TimelineWrite $timelineWrite
) {
$this->config = $config;
$this->connection = $connection;
$this->timelineWrite = $timelineWrite;
}
/**
@ -89,6 +93,9 @@ class Places
*/
public function downloadPlanet(): string
{
echo "Download planet data to temporary file...\n";
flush();
$filename = sys_get_temp_dir().'/planet_coarse_boundaries.zip';
unlink($filename);
@ -134,6 +141,7 @@ class Places
public function importPlanet(string $datafile): void
{
echo "Inserting planet data into database...\n";
flush();
// Detect the GIS type
$gis = $this->detectGisType();
@ -317,16 +325,47 @@ class Places
// Mark success
echo "Planet database imported successfully!\n";
echo "You should re-index your library now.\n";
flush();
$this->config->setSystemValue('memories.gis_type', $gis);
// Delete data file
unlink($datafile);
}
/**
* Recalculate all places for all users.
*/
public function recalculateAll()
{
echo "Recalculating places for all files (do not interrupt this process)...\n";
flush();
$count = 0;
$this->timelineWrite->orphanAndRun(['fileid', 'lat', 'lon'], 20, function (array $row) use (&$count) {
++$count;
// Only proceed if we have a valid location
$fileid = $row['fileid'];
$lat = (float) $row['lat'];
$lon = (float) $row['lon'];
// Update places
if ($lat || $lon) {
$this->timelineWrite->updatePlacesData($fileid, $lat, $lon);
}
// Print every 500 files
if (0 === $count % 500) {
echo "Updated places data for {$count} files\n";
flush();
}
});
}
/**
* Create database tables and indices.
*/
private function setupDatabase(): void
protected function setupDatabase(): void
{
try {
// Get Gis type

20
src/Admin.vue
View File

@ -697,16 +697,24 @@ export default defineComponent({
},
placesSetup(event: Event) {
const warnSetup = this.t(
"memories",
"Looks like the database is already setup. Are you sure you want to redownload planet data?"
);
const warnLong = this.t(
"memories",
"You are about to download the planet database. This may take a while."
);
const warnReindex = this.t(
"memories",
"This may also cause all photos to be re-indexed!"
);
const msg =
"Looks like the database is already setup. Are you sure you want to drop the table and redownload OSM data?";
if (this.status.gis_count && !confirm(msg)) {
(this.status.gis_count ? warnSetup : warnLong) + " " + warnReindex;
if (!confirm(msg)) {
event.preventDefault();
event.stopPropagation();
return;
} else {
alert(
"Please wait for the download and insertion to complete. This may take a while."
);
}
},