diff --git a/CHANGELOG.md b/CHANGELOG.md index b6a763dd..a1fc6182 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ This file is manually updated. Please file an issue if something is missing. +## Unreleased + +- **Feature**: Allow migrating Google Takeout metadata to EXIF ([#430](https://github.com/pulsejet/memories/issues/430)) + ## v4.12.1 (2023-03-15) - **Feature**: Load full image on zoom ([#266](https://github.com/pulsejet/memories/issues/266)) diff --git a/README.md b/README.md index 45fd2f41..000d74ba 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Memories is a _batteries-included_ photo management solution for Nextcloud with - **πŸ“¦ Archive**: Store photos you don't want to see in your timeline in a separate folder. - **πŸ“Ή Video Transcoding**: Memories transcodes videos and uses HLS for maximal performance. - **πŸ—ΊοΈ Map**: View your photos on a map, tagged with accurate reverse geocoding. +- **πŸ“¦ Migration**: Supports easy migration from Nextcloud Photos and Google Takeout. - **⚑️ Performance**: Memories is very fast. ## πŸš€ Installation diff --git a/appinfo/info.xml b/appinfo/info.xml index 3a848dbb..431546d1 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -19,6 +19,7 @@ Memories is a *batteries-included* photo management solution for Nextcloud with - **πŸ“¦ Archive**: Store photos you don't want to see in your timeline in a separate folder. - **πŸ“Ή Video Transcoding**: Memories transcodes videos and uses HLS for maximal performance. - **πŸ—ΊοΈ Map**: View your photos on a map, tagged with accurate reverse geocoding. +- **πŸ“¦ Migration**: Supports easy migration from Nextcloud Photos and Google Takeout. - **⚑️ Performance**: Memories is very fast. ## πŸš€ Installation @@ -44,6 +45,7 @@ Memories is a *batteries-included* photo management solution for Nextcloud with OCA\Memories\Command\Index OCA\Memories\Command\VideoSetup OCA\Memories\Command\PlacesSetup + OCA\Memories\Command\MigrateGoogleTakeout diff --git a/lib/Command/MigrateGoogleTakeout.php b/lib/Command/MigrateGoogleTakeout.php new file mode 100644 index 00000000..ac3d0f96 --- /dev/null +++ b/lib/Command/MigrateGoogleTakeout.php @@ -0,0 +1,302 @@ + + * @author Varun Patil + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCA\Memories\Command; + +use OCA\Memories\AppInfo\Application; +use OCA\Memories\Db\TimelineWrite; +use OCA\Memories\Exif; +use OCP\Files\File; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\IConfig; +use OCP\IDBConnection; +use OCP\ITempManager; +use OCP\IUser; +use OCP\IUserManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class MigrateGoogleTakeout extends Command +{ + protected const MIGRATOR_VERSION = 1; + protected const MIGRATED_KEY = 'memoriesMigratorVersion'; + + protected OutputInterface $output; + protected InputInterface $input; + + protected IUserManager $userManager; + protected IRootFolder $rootFolder; + protected IConfig $config; + protected IDBConnection $connection; + protected TimelineWrite $timelineWrite; + protected ITempManager $tempManager; + + // Stats + private int $nProcessed = 0; + + private array $mimeTypes = []; + + public function __construct( + IRootFolder $rootFolder, + IUserManager $userManager, + IConfig $config, + IDBConnection $connection, + ITempManager $tempManager + ) { + parent::__construct(); + + $this->userManager = $userManager; + $this->rootFolder = $rootFolder; + $this->config = $config; + $this->connection = $connection; + $this->tempManager = $tempManager; + $this->timelineWrite = new TimelineWrite($connection); + } + + protected function configure(): void + { + $this + ->setName('memories:migrate-google-takeout') + ->setDescription('Migrate JSON metadata from Google Takeout') + ->addOption('override', 'o', null, 'Override existing EXIF metadata') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->output = $output; + $this->input = $input; + $this->mimeTypes = array_merge(Application::IMAGE_MIMES, Application::VIDEO_MIMES); + + // Provide ample warnings + if ($input->isInteractive()) { + $output->writeln('This command will migrate JSON metadata from Google Takeout to EXIF metadata.'); + $output->writeln('Only metadata that is missing from EXIF will be migrated, unless --override is specified.'); + $output->writeln('It will also update the JSON files to mark them as migrated.'); + $output->writeln('Make sure you have a backup of your originals before running this command.'); + $output->writeln('Also make sure exiftool is working beforehand by running memories:index on some files.'); + $output->write('Are you sure you want to continue? (y/N): '); + $answer = trim(fgets(STDIN)); + if ('y' !== $answer) { + $output->writeln('Aborting'); + + return 1; + } + } + + // Start static exif process + Exif::ensureStaticExiftoolProc(); + + // Call migration for each user + $this->userManager->callForSeenUsers(function (IUser $user) { + $this->migrateUser($user); + }); + + // Print statistics + $output->writeln("\nMigrated JSON metadata from {$this->nProcessed} files"); + + return 0; + } + + protected function migrateUser(IUser $user): void + { + $this->output->writeln("Migrating user {$user->getUID()}"); + + // Get user's root folder + \OC_Util::tearDownFS(); + \OC_Util::setupFS($user->getUID()); + $userFolder = $this->rootFolder->getUserFolder($user->getUID()); + + // Iterate all files + $this->migrateFolder($userFolder); + } + + protected function migrateFolder(Folder $folder): void + { + $nodes = $folder->getDirectoryListing(); + $path = $folder->getPath(); + $this->output->writeln("Scanning folder {$path}"); + + foreach ($nodes as $i => $node) { + if ($node instanceof Folder) { + $this->migrateFolder($node); + } elseif ($node instanceof File) { + $this->migrateFile($node); + } + } + } + + protected function migrateFile(File $file): void + { + // Check if this is a supported file + if (!\in_array($file->getMimeType(), $this->mimeTypes, true)) { + return; + } + + // Check for existence of JSON metadata + $path = $file->getPath(); + $json = []; + + /** @var \OCP\Files\File */ + $jsonFile = null; + + try { + $jsonPath = $path.'.json'; + + /** @var \OCP\Files\File */ + $jsonFile = $this->rootFolder->get($jsonPath); + if (!$jsonFile->isReadable() || \OCP\Files\FileInfo::TYPE_FOLDER === $jsonFile->getType()) { + return; + } + + $json = json_decode($jsonFile->getContent(), true); + } catch (\OCP\Files\NotFoundException $e) { + return; + } catch (\Exception $e) { + $this->output->writeln("Error while reading JSON metadata for {$path}: {$e->getMessage()}"); + + return; + } + + // Check if JSON metadata is valid + // For now, check if it at least has either title or url + if (!isset($json['title']) && !isset($json['url'])) { + $this->output->writeln("JSON metadata for {$path} is invalid, skipping"); + + return; + } + + // Check if JSON metadata is already migrated + if (isset($json[self::MIGRATED_KEY]) && $json[self::MIGRATED_KEY] >= self::MIGRATOR_VERSION) { + return; + } + + // Convert Takeout metadata to exiftool JSON format + $txf = $this->takeoutToExiftoolJson($json); + + // Get current EXIF metadata + $exif = Exif::getExifFromFile($file); + + // Keep keys that are not in EXIF unless --override is specified + if (!((bool) $this->input->getOption('override'))) { + $txf = array_filter($txf, function ($value, $key) use ($exif) { + return !isset($exif[$key]); + }, ARRAY_FILTER_USE_BOTH); + } + + // Special case: if $txf has both GPSLatitude and GPSLongitude, + // also specify GPSCoordinates, since videos need this + if (isset($txf['GPSLatitude'], $txf['GPSLongitude'])) { + $txf['GPSCoordinates'] = $txf['GPSLatitude'].', '.$txf['GPSLongitude']; + } + + // Check if there is anything to write + if (\count($txf) > 0) { + $keysWritten = implode(', ', array_keys($txf)); + $this->output->writeln("Writing EXIF metadata for {$path} ({$keysWritten})"); + + // Write EXIF metadata + try { + $localPath = $file->getStorage()->getLocalFile($file->getInternalPath()); + Exif::setExif($localPath, $txf); + $file->touch(); + } catch (\Exception $e) { + $this->output->writeln("Error while writing EXIF metadata for {$path}: {$e->getMessage()}"); + + return; + } + } else { + $this->output->writeln("No new EXIF metadata to write for {$path}"); + } + + // Mark JSON metadata as migrated + $json[self::MIGRATED_KEY] = self::MIGRATOR_VERSION; + + // Write JSON metadata + try { + $jsonFile->putContent(json_encode($json, JSON_PRETTY_PRINT)); + } catch (\Exception $e) { + $this->output->writeln("Error while updating JSON file for {$path}: {$e->getMessage()}"); + + return; + } + + $this->nProcessed++; + } + + protected function takeoutToExiftoolJson(array $json) + { + // Helper to get a value from nested JSON + $get = function (string $source) use ($json) { + $keys = array_reverse(explode('.', $source)); + while (\count($keys) > 0) { + $key = array_pop($keys); + if (!isset($json[$key])) { + return null; + } + $json = $json[$key]; + } + + // Check if empty string + if (\is_string($json) && '' === $json) { + return null; + } + + // Check if numeric and zero + if (is_numeric($json)) { + if (0.0 === (float) $json) { + return null; + } + + return (float) $json; + } + + return $json; + }; + + $txf = []; + + // Description + $txf['Description'] = $get('description'); + + // Date/Time + $epoch = $get('photoTakenTime.timestamp'); + if (is_numeric($epoch)) { + $date = new \DateTime(); + $date->setTimestamp((int)$epoch); + $txf['DateTimeOriginal'] = $date->format('Y:m:d H:i:s'); + } + + // Location coordinates + $txf['GPSLatitude'] = $get('geoData.latitude'); + $txf['GPSLongitude'] = $get('geoData.longitude'); + $txf['GPSAltitude'] = $get('geoData.altitude'); + + // Remove all null values + return array_filter($txf, function ($value) { + return null !== $value; + }); + } +}