RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

index: allow cleaning up orphans (fix #326)

pull/363/head
Varun Patil 2023-01-17 20:52:26 -08:00
parent c6ef65f69a
commit b18a098f47
3 changed files with 165 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
lib/Command/Index.php
View File

@ -41,6 +41,20 @@ use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\ConsoleSectionOutput;
use Symfony\Component\Console\Output\OutputInterface;
class IndexOpts
{
public bool $refresh = false;
public bool $clear = false;
public bool $cleanup = false;
public function __construct(InputInterface $input)
{
$this->refresh = (bool) $input->getOption('refresh');
$this->clear = (bool) $input->getOption('clear');
$this->cleanup = (bool) $input->getOption('cleanup');
}
}
class Index extends Command
{
/** @var int[][] */
@ -99,7 +113,13 @@ class Index extends Command
'clear',
null,
InputOption::VALUE_NONE,
'Clear existing index before creating a new one (SLOW)'
'Clear existing index before creating a new one (slow)'
)
->addOption(
'cleanup',
null,
InputOption::VALUE_NONE,
'Remove orphaned entries from index (e.g. from .nomedia files)'
)
;
}
@ -131,11 +151,10 @@ class Index extends Command
}
// Get options and arguments
$refresh = $input->getOption('refresh') ? true : false;
$clear = $input->getOption('clear') ? true : false;
$opts = new IndexOpts($input);
// Clear index if asked for this
if ($clear && $input->isInteractive()) {
if ($opts->clear && $input->isInteractive()) {
$output->write('Are you sure you want to clear the existing index? (y/N): ');
$answer = trim(fgets(STDIN));
if ('y' !== $answer) {
@ -144,16 +163,23 @@ class Index extends Command
return 1;
}
}
if ($clear) {
if ($opts->clear) {
$this->timelineWrite->clear();
$output->writeln('Cleared existing index');
}
// Orphan all entries so we can delete them later
if ($opts->cleanup) {
$output->write('Marking all entries for cleanup ... ');
$count = $this->timelineWrite->orphanAll();
$output->writeln("{$count} marked");
}
// Run with the static process
try {
\OCA\Memories\Exif::ensureStaticExiftoolProc();
return $this->executeWithOpts($output, $refresh);
return $this->executeWithOpts($output, $opts);
} catch (\Exception $e) {
error_log('FATAL: '.$e->getMessage());
@ -163,7 +189,7 @@ class Index extends Command
}
}
protected function executeWithOpts(OutputInterface $output, bool &$refresh): int
protected function executeWithOpts(OutputInterface $output, IndexOpts &$opts): int
{
// Refuse to run without exiftool
if (!$this->testExif()) {
@ -184,10 +210,17 @@ class Index extends Command
}
$this->output = $output;
$this->userManager->callForSeenUsers(function (IUser &$user) use (&$refresh) {
$this->generateUserEntries($user, $refresh);
$this->userManager->callForSeenUsers(function (IUser &$user) use (&$opts) {
$this->generateUserEntries($user, $opts);
});
// Clear orphans if asked for this
if ($opts->cleanup) {
$output->write('Deleting orphaned entries ... ');
$count = $this->timelineWrite->removeOrphans();
$output->writeln("{$count} deleted");
}
// Show some stats
$endTime = microtime(true);
$execTime = (int) (($endTime - $startTime) * 1000) / 1000;
@ -239,7 +272,7 @@ class Index extends Command
return true;
}
private function generateUserEntries(IUser &$user, bool &$refresh): void
private function generateUserEntries(IUser &$user, IndexOpts &$opts): void
{
\OC_Util::tearDownFS();
\OC_Util::setupFS($user->getUID());
@ -247,12 +280,12 @@ class Index extends Command
$uid = $user->getUID();
$userFolder = $this->rootFolder->getUserFolder($uid);
$this->outputSection = $this->output->section();
$this->parseFolder($userFolder, $refresh, (float) $this->nUser, (float) $this->userManager->countSeenUsers());
$this->parseFolder($userFolder, $opts, (float) $this->nUser, (float) $this->userManager->countSeenUsers());
$this->outputSection->overwrite('Scanned '.$userFolder->getPath());
++$this->nUser;
}
private function parseFolder(Folder &$folder, bool $refresh, float $progress_i, float $progress_n): void
private function parseFolder(Folder &$folder, IndexOpts &$opts, float $progress_i, float $progress_n): void
{
try {
// Respect the '.nomedia' file. If present don't traverse the folder
@ -268,11 +301,11 @@ class Index extends Command
if ($node instanceof Folder) {
$new_progress_i = (float) ($progress_i * \count($nodes) + $i);
$new_progress_n = (float) ($progress_n * \count($nodes));
$this->parseFolder($node, $refresh, $new_progress_i, $new_progress_n);
$this->parseFolder($node, $opts, $new_progress_i, $new_progress_n);
} elseif ($node instanceof File) {
$progress = (float) (($progress_i / $progress_n) * 100);
$this->outputSection->overwrite(sprintf('%.2f%%', $progress).' scanning '.$node->getPath());
$this->parseFile($node, $refresh);
$this->parseFile($node, $opts);
}
}
} catch (\Exception $e) {
@ -284,13 +317,17 @@ class Index extends Command
}
}
private function parseFile(File &$file, bool &$refresh): void
private function parseFile(File &$file, IndexOpts &$opts): void
{
// Process the file
$res = 1;
try {
$res = $this->timelineWrite->processFile($file, $refresh);
$res = $this->timelineWrite->processFile($file, $opts->refresh);
if ($opts->cleanup) {
$this->timelineWrite->unorphan($file);
}
} catch (\Error $e) {
$this->output->writeln(sprintf(
'<error>Could not process file %s: %s</error>',

43
lib/Db/TimelineWrite.php
View File

@ -222,4 +222,47 @@ class TimelineWrite
$t2 = $p->getTruncateTableSQL('`*PREFIX*memories_livephoto`', false);
$this->connection->executeStatement("{$t1}; {$t2}");
}
/**
* Mark a file as not orphaned.
*/
public function unorphan(File &$file)
{
$query = $this->connection->getQueryBuilder();
$query->update('memories')
->set('orphan', $query->createNamedParameter(false, IQueryBuilder::PARAM_BOOL))
->where($query->expr()->eq('fileid', $query->createNamedParameter($file->getId(), IQueryBuilder::PARAM_INT)))
;
$query->executeStatement();
}
/**
* Mark all files in the table as orphaned.
*
* @return int Number of rows affected
*/
public function orphanAll(): int
{
$query = $this->connection->getQueryBuilder();
$query->update('memories')
->set('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
;
return $query->executeStatement();
}
/**
* Remove all entries that are orphans.
*
* @return int Number of rows affected
*/
public function removeOrphans(): int
{
$query = $this->connection->getQueryBuilder();
$query->delete('memories')
->where($query->expr()->eq('orphan', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)))
;
return $query->executeStatement();
}
}

69
lib/Migration/Version401000Date20230118043813.php 100644
View File

@ -0,0 +1,69 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Your name <your@email.com>
* @author Your name <your@email.com>
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Migration;
use OCP\DB\ISchemaWrapper;
use OCP\DB\Types;
use OCP\Migration\IOutput;
use OCP\Migration\SimpleMigrationStep;
/**
* Auto-generated migration step: Please modify to your needs!
*/
class Version401000Date20230118043813 extends SimpleMigrationStep
{
/**
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
*/
public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
{
}
/**
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
*/
public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
{
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
$table = $schema->getTable('memories');
if (!$table->hasColumn('orphan')) {
$table->addColumn('orphan', Types::BOOLEAN, [
'notnull' => false,
'default' => false,
]);
}
return $schema;
}
/**
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
*/
public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
{
}
}