RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use storage for lookups

pull/62/head
Varun Patil 2022-09-23 18:54:14 -07:00
parent 36dc5abb8f
commit f5eeb1ae9d
10 changed files with 179 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
appinfo/routes.php
View File

@ -17,8 +17,6 @@ return [
// API
['name' => 'api#days', 'url' => '/api/days', 'verb' => 'GET'],
['name' => 'api#day', 'url' => '/api/days/{id}', 'verb' => 'GET'],
['name' => 'api#folder', 'url' => '/api/folder/{folder}', 'verb' => 'GET'],
['name' => 'api#folderDay', 'url' => '/api/folder/{folder}/{dayId}', 'verb' => 'GET'],
// Config API
['name' => 'api#setUserConfig', 'url' => '/api/config/{key}', 'verb' => 'PUT'],

4
js/memories-main.js
View File

File diff suppressed because one or more lines are too long

2
js/memories-main.js.map
View File

File diff suppressed because one or more lines are too long

144
lib/Controller/ApiController.php
View File

@ -29,6 +29,7 @@ use OC\Files\Search\SearchComparison;
use OC\Files\Search\SearchQuery;
use OCA\Memories\AppInfo\Application;
use OCA\Memories\Db\TimelineQuery;
use OCA\Memories\Exif;
use OCP\AppFramework\Controller;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\JSONResponse;
@ -38,6 +39,7 @@ use OCP\IDBConnection;
use OCP\IRequest;
use OCP\IUserSession;
use OCP\Files\FileInfo;
use OCP\Files\Folder;
use OCP\Files\Search\ISearchComparison;
class ApiController extends Controller {
@ -83,15 +85,16 @@ class ApiController extends Controller {
}
/** Preload a few "day" at the start of "days" response */
private function preloadDays(array &$days) {
private function preloadDays(array &$days, Folder &$folder, bool $recursive) {
$uid = $this->userSession->getUser()->getUID();
$transforms = $this->getTransformations();
$preloaded = 0;
foreach ($days as &$day) {
$day["detail"] = $this->timelineQuery->getDay(
$this->config,
$folder,
$uid,
$day["dayid"],
$recursive,
$transforms,
);
$day["count"] = count($day["detail"]); // make sure count is accurate
@ -103,6 +106,30 @@ class ApiController extends Controller {
}
}
/** Get the Folder object relevant to the request */
private function getRequestFolder() {
$uid = $this->userSession->getUser()->getUID();
try {
$folder = null;
$folderPath = $this->request->getParam('folder');
$userFolder = $this->rootFolder->getUserFolder($uid);
if (!is_null($folderPath)) {
$folder = $userFolder->get($folderPath);
} else {
$configPath = Exif::removeExtraSlash(Exif::getPhotosPath($this->config, $uid));
$folder = $userFolder->get($configPath);
}
if (!$folder instanceof Folder) {
throw new \Exception("Folder not found");
}
} catch (\Exception $e) {
return null;
}
return $folder;
}
/**
* @NoAdminRequired
*
@ -113,13 +140,31 @@ class ApiController extends Controller {
if (is_null($user)) {
return new JSONResponse([], Http::STATUS_PRECONDITION_FAILED);
}
$uid = $user->getUID();
// Get the folder to show
$folder = $this->getRequestFolder();
$recursive = is_null($this->request->getParam('folder'));
if (is_null($folder)) {
return new JSONResponse([], Http::STATUS_NOT_FOUND);
}
// Run actual query
$list = $this->timelineQuery->getDays(
$this->config,
$user->getUID(),
$folder,
$uid,
$recursive,
$this->getTransformations(),
);
$this->preloadDays($list);
// Preload some day responses
$this->preloadDays($list, $folder, $recursive);
// Add subfolder info if querying non-recursively
if (!$recursive) {
$this->addSubfolders($folder, $list, $user);
}
return new JSONResponse($list, Http::STATUS_OK);
}
@ -130,70 +175,39 @@ class ApiController extends Controller {
*/
public function day(string $id): JSONResponse {
$user = $this->userSession->getUser();
if (is_null($user) || !is_numeric($id)) {
if (is_null($user)) {
return new JSONResponse([], Http::STATUS_PRECONDITION_FAILED);
}
$uid = $user->getUID();
// Get the folder to show
$folder = $this->getRequestFolder();
$recursive = is_null($this->request->getParam('folder'));
if (is_null($folder)) {
return new JSONResponse([], Http::STATUS_NOT_FOUND);
}
// Run actual query
$list = $this->timelineQuery->getDay(
$this->config,
$user->getUID(),
$folder,
$uid,
intval($id),
$recursive,
$this->getTransformations(),
);
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* Check if folder is allowed and get it if yes
*/
private function getAllowedFolder(int $folder, $user) {
// Get root if folder not specified
$root = $this->rootFolder->getUserFolder($user->getUID());
if ($folder === 0) {
$folder = $root->getId();
}
// Check access to folder
$nodes = $root->getById($folder);
if (empty($nodes)) {
return NULL;
}
// Check it is a folder
$node = $nodes[0];
if (!$node instanceof \OCP\Files\Folder) {
return NULL;
}
return $node;
}
/**
* @NoAdminRequired
*
* @return JSONResponse
*/
public function folder(string $folder): JSONResponse {
$user = $this->userSession->getUser();
if (is_null($user) || !is_numeric($folder)) {
return new JSONResponse([], Http::STATUS_PRECONDITION_FAILED);
}
// Check permissions
$node = $this->getAllowedFolder(intval($folder), $user);
if (is_null($node)) {
return new JSONResponse([], Http::STATUS_FORBIDDEN);
}
// Get response from db
$list = $this->timelineQuery->getDaysFolder($node->getId());
public function addSubfolders(Folder &$folder, &$list, &$user) {
// Get subdirectories
$sub = $node->search(new SearchQuery(
$sub = $folder->search(new SearchQuery(
new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', FileInfo::MIMETYPE_FOLDER),
0, 0, [], $user));
$sub = array_filter($sub, function ($item) use ($node) {
return $item->getParent()->getId() === $node->getId();
$sub = array_filter($sub, function ($item) use (&$folder) {
return $item->getParent()->getId() === $folder->getId();
});
// Sort by name
@ -204,7 +218,7 @@ class ApiController extends Controller {
// Map sub to JSON array
$subdirArray = [
"dayid" => \OCA\Memories\Util::$TAG_DAYID_FOLDERS,
"detail" => array_map(function ($node) {
"detail" => array_map(function (&$node) {
return [
"fileid" => $node->getId(),
"name" => $node->getName(),
@ -215,28 +229,6 @@ class ApiController extends Controller {
];
$subdirArray["count"] = count($subdirArray["detail"]);
array_unshift($list, $subdirArray);
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* @NoAdminRequired
*
* @return JSONResponse
*/
public function folderDay(string $folder, string $dayId): JSONResponse {
$user = $this->userSession->getUser();
if (is_null($user) || !is_numeric($folder) || !is_numeric($dayId)) {
return new JSONResponse([], Http::STATUS_PRECONDITION_FAILED);
}
$node = $this->getAllowedFolder(intval($folder), $user);
if ($node === NULL) {
return new JSONResponse([], Http::STATUS_FORBIDDEN);
}
$list = $this->timelineQuery->getDayFolder($user->getUID(), $node->getId(), intval($dayId));
return new JSONResponse($list, Http::STATUS_OK);
}
/**

1
lib/Db/TimelineQuery.php
View File

@ -7,7 +7,6 @@ use OCP\IDBConnection;
class TimelineQuery {
use TimelineQueryDays;
use TimelineQueryDay;
use TimelineQueryFilters;
protected IDBConnection $connection;

121
lib/Db/TimelineQueryDay.php
View File

@ -1,121 +0,0 @@
<?php
declare(strict_types=1);
namespace OCA\Memories\Db;
use OCA\Memories\Exif;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\DB\QueryBuilder\IQueryBuilder;
trait TimelineQueryDay {
protected IDBConnection $connection;
/**
* Process the single day response
* @param array $day
*/
private function processDay(&$day) {
foreach($day as &$row) {
// We don't need date taken (see query builder)
unset($row['datetaken']);
// Convert field types
$row["fileid"] = intval($row["fileid"]);
$row["isvideo"] = intval($row["isvideo"]);
if (!$row["isvideo"]) {
unset($row["isvideo"]);
}
if ($row["categoryid"]) {
$row["isfavorite"] = 1;
}
unset($row["categoryid"]);
}
return $day;
}
/** Get the base query builder for day */
private function makeQueryDay(
IQueryBuilder &$query,
int $dayid,
string $user,
$whereFilecache
) {
// Get all entries also present in filecache
$fileid = $query->createFunction('DISTINCT m.fileid');
// We don't actually use m.datetaken here, but postgres
// needs that all fields in ORDER BY are also in SELECT
// when using DISTINCT on selected fields
$query->select($fileid, 'f.etag', 'm.isvideo', 'vco.categoryid', 'm.datetaken')
->from('memories', 'm')
->innerJoin('m', 'filecache', 'f',
$query->expr()->andX(
$query->expr()->eq('f.fileid', 'm.fileid'),
$whereFilecache
))
->andWhere($query->expr()->eq('m.dayid', $query->createNamedParameter($dayid, IQueryBuilder::PARAM_INT)));
// Add favorite field
$this->addFavoriteTag($query, $user);
// Group and sort by date taken
$query->orderBy('m.datetaken', 'DESC');
return $query;
}
/**
* Get a day response from the database for the timeline
* @param IConfig $config
* @param string $userId
* @param int $dayId
*/
public function getDay(
IConfig &$config,
string $user,
int $dayId,
array $queryTransforms = []
): array {
// Filter by path starting with timeline path
$configPath = Exif::getPhotosPath($config, $user);
$likeHome = Exif::removeExtraSlash("files/" . $configPath . "%");
$likeExt = Exif::removeLeadingSlash(Exif::removeExtraSlash($configPath . "%"));
$query = $this->connection->getQueryBuilder();
$this->makeQueryDay($query, $dayId, $user, $query->expr()->orX(
$query->expr()->like('f.path', $query->createNamedParameter($likeHome)),
$query->expr()->like('f.path', $query->createNamedParameter($likeExt)),
));
// Filter by UID
$query->andWhere($query->expr()->eq('m.uid', $query->createNamedParameter($user)));
// Apply all transformations
foreach ($queryTransforms as &$transform) {
$transform($query, $user);
}
$rows = $query->executeQuery()->fetchAll();
return $this->processDay($rows);
}
/**
* Get a day response from the database for one folder
* @param int $folderId
* @param int $dayId
*/
public function getDayFolder(
string $user,
int $folderId,
int $dayId
): array {
$query = $this->connection->getQueryBuilder();
$this->makeQueryDay($query, $dayId, $user, $query->expr()->orX(
$query->expr()->eq('f.parent', $query->createNamedParameter($folderId, IQueryBuilder::PARAM_INT)),
$query->expr()->eq('f.fileid', $query->createNamedParameter($folderId, IQueryBuilder::PARAM_INT)),
));
$rows = $query->executeQuery()->fetchAll();
return $this->processDay($rows);
}
}

144
lib/Db/TimelineQueryDays.php
View File

@ -3,10 +3,9 @@ declare(strict_types=1);
namespace OCA\Memories\Db;
use OCA\Memories\Exif;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Folder;
trait TimelineQueryDays {
protected IDBConnection $connection;
@ -23,73 +22,128 @@ trait TimelineQueryDays {
return $days;
}
/** Get the base query builder for days */
private function makeQueryDays(
IQueryBuilder &$query,
$whereFilecache
) {
// Get all entries also present in filecache
$count = $query->func()->count($query->createFunction('DISTINCT m.fileid'), 'count');
$query->select('m.dayid', $count)
->from('memories', 'm')
->innerJoin('m', 'filecache', 'f',
$query->expr()->andX(
$query->expr()->eq('f.fileid', 'm.fileid'),
$whereFilecache
));
/**
* Process the single day response
* @param array $day
*/
private function processDay(&$day) {
foreach($day as &$row) {
// We don't need date taken (see query builder)
unset($row['datetaken']);
// Group and sort by dayid
$query->groupBy('m.dayid')
->orderBy('m.dayid', 'DESC');
return $query;
// Convert field types
$row["fileid"] = intval($row["fileid"]);
$row["isvideo"] = intval($row["isvideo"]);
if (!$row["isvideo"]) {
unset($row["isvideo"]);
}
if ($row["categoryid"]) {
$row["isfavorite"] = 1;
}
unset($row["categoryid"]);
}
return $day;
}
/** Get the query for oc_filecache join */
private function getFilecacheJoinQuery(IQueryBuilder &$query, Folder &$folder, bool $recursive) {
// Subquery to get storage and path
$subQuery = $query->getConnection()->getQueryBuilder();
$finfo = $subQuery->select('path', 'storage')->from('filecache')->where(
$subQuery->expr()->eq('fileid', $subQuery->createNamedParameter($folder->getId())),
)->executeQuery()->fetch();
if (empty($finfo)) {
throw new \Exception("Folder not found");
}
$pathQuery = null;
if ($recursive) {
// Filter by path for recursive query
$likePath = $finfo["path"];
if (!empty($likePath)) {
$likePath .= '/';
}
$likePath = $likePath . '%';
$pathQuery = $query->expr()->like('f.path', $query->createNamedParameter($likePath));
} else {
// If getting non-recursively folder only check for parent
$pathQuery = $query->expr()->eq('f.parent', $query->createNamedParameter($folder->getId(), IQueryBuilder::PARAM_INT));
}
return $query->expr()->andX(
$query->expr()->eq('f.fileid', 'm.fileid'),
$query->expr()->in('f.storage', $query->createNamedParameter($finfo["storage"])),
$pathQuery,
);
}
/**
* Get the days response from the database for the timeline
* @param IConfig $config
* @param string $userId
*/
public function getDays(
IConfig &$config,
string $user,
Folder &$folder,
string $uid,
bool $recursive,
array $queryTransforms = []
): array {
// Filter by path starting with timeline path
$configPath = Exif::getPhotosPath($config, $user);
$likeHome = Exif::removeExtraSlash("files/" . $configPath . "%");
$likeExt = Exif::removeLeadingSlash(Exif::removeExtraSlash($configPath . "%"));
$query = $this->connection->getQueryBuilder();
$this->makeQueryDays($query, $query->expr()->orX(
$query->expr()->like('f.path', $query->createNamedParameter($likeHome)),
$query->expr()->like('f.path', $query->createNamedParameter($likeExt)),
));
// Filter by user
$query->andWhere($query->expr()->eq('m.uid', $query->createNamedParameter($user)));
// Get all entries also present in filecache
$count = $query->func()->count($query->createFunction('DISTINCT m.fileid'), 'count');
$query->select('m.dayid', $count)
->from('memories', 'm')
->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, $recursive));
// Group and sort by dayid
$query->groupBy('m.dayid')
->orderBy('m.dayid', 'DESC');
// Apply all transformations
foreach ($queryTransforms as &$transform) {
$transform($query, $user);
$transform($query, $uid);
}
$rows = $query->executeQuery()->fetchAll();
return $this->processDays($rows);
}
/**
* Get the days response from the database for one folder
* @param int $folderId
/**
* Get the days response from the database for the timeline
* @param string $userId
*/
public function getDaysFolder(int $folderId) {
public function getDay(
Folder &$folder,
string $uid,
int $dayid,
bool $recursive,
array $queryTransforms = []
): array {
$query = $this->connection->getQueryBuilder();
$this->makeQueryDays($query, $query->expr()->orX(
$query->expr()->eq('f.parent', $query->createNamedParameter($folderId, IQueryBuilder::PARAM_INT)),
$query->expr()->eq('f.fileid', $query->createNamedParameter($folderId, IQueryBuilder::PARAM_INT)),
));
// Get all entries also present in filecache
$fileid = $query->createFunction('DISTINCT m.fileid');
// We don't actually use m.datetaken here, but postgres
// needs that all fields in ORDER BY are also in SELECT
// when using DISTINCT on selected fields
$query->select($fileid, 'f.etag', 'm.isvideo', 'vco.categoryid', 'm.datetaken')
->from('memories', 'm')
->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, $recursive))
->andWhere($query->expr()->eq('m.dayid', $query->createNamedParameter($dayid, IQueryBuilder::PARAM_INT)));
// Add favorite field
$this->addFavoriteTag($query, $uid);
// Group and sort by date taken
$query->orderBy('m.datetaken', 'DESC');
// Apply all transformations
foreach ($queryTransforms as &$transform) {
$transform($query, $uid);
}
$rows = $query->executeQuery()->fetchAll();
return $this->processDays($rows);
return $this->processDay($rows);
}
}

6
src/components/Folder.vue
View File

@ -3,7 +3,7 @@
hasPreview: previewFileInfos.length > 0,
onePreview: previewFileInfos.length === 1,
}"
@click="openFolder(data.fileid)"
@click="openFolder(data)"
v-bind:style="{
width: rowHeight + 'px',
height: rowHeight + 'px',
@ -88,9 +88,9 @@ export default class Folder extends Mixins(GlobalMixin) {
}
/** Open folder */
openFolder(id: number) {
openFolder(folder: IFolder) {
this.$router.push({ name: 'folders', params: {
id: id.toString(),
path: folder.path.split('/').slice(3).join('/'),
}});
}
}

18
src/components/Timeline.vue
View File

@ -141,9 +141,6 @@ const MIN_COLS = 3; // Min number of columns (on phone, e.g.
const API_ROUTES = {
DAYS: 'days',
DAY: 'days/{dayId}',
FOLDER_DAYS: 'folder/{folderId}',
FOLDER_DAY: 'folder/{folderId}/{dayId}',
};
for (const [key, value] of Object.entries(API_ROUTES)) {
API_ROUTES[key] = '/apps/memories/api/' + value;
@ -415,6 +412,11 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
query.set('vid', '1');
}
// Folder
if (this.$route.name === 'folders') {
query.set('folder', this.$route.params.path || '/');
}
// Create query string and append to URL
const queryStr = query.toString();
if (queryStr) {
@ -452,11 +454,6 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
let url = API_ROUTES.DAYS;
let params: any = {};
if (this.$route.name === 'folders') {
url = API_ROUTES.FOLDER_DAYS;
params.folderId = this.$route.params.id || 0;
}
try {
this.loading++;
const startState = this.state;
@ -555,11 +552,6 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
let url = API_ROUTES.DAY;
const params: any = { dayId };
if (this.$route.name === 'folders') {
url = API_ROUTES.FOLDER_DAY;
params.folderId = this.$route.params.id || 0;
}
// Do this in advance to prevent duplicate requests
this.loadedDays.add(dayId);

2
src/router.ts
View File

@ -56,7 +56,7 @@
},
{
path: '/folders/:id*',
path: '/folders/:path*',
component: Timeline,
name: 'folders',
props: route => ({