From de845bd543279a9aa1d9f8c586ff24df7655c741 Mon Sep 17 00:00:00 2001 From: Varun Patil Date: Thu, 27 Oct 2022 12:54:51 -0700 Subject: [PATCH] Speed up tags --- lib/Controller/ApiController.php | 2 +- lib/Db/TimelineQuery.php | 17 +++++++--- lib/Db/TimelineQueryDays.php | 10 +++--- lib/Db/TimelineQueryTags.php | 57 +++++++++++++++++++++++++++----- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 205448be..51b56dea 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -278,7 +278,7 @@ class ApiController extends Controller // Run actual query $list = []; - $t = (int) ($this->request->getParam('t')); + $t = (int) $this->request->getParam('t'); if ($t & 1) { // personal $list = array_merge($list, $this->timelineQuery->getAlbums($user->getUID())); } diff --git a/lib/Db/TimelineQuery.php b/lib/Db/TimelineQuery.php index def3035f..3de70e1f 100644 --- a/lib/Db/TimelineQuery.php +++ b/lib/Db/TimelineQuery.php @@ -22,20 +22,27 @@ class TimelineQuery $this->connection = $connection; } - public static function debugQuery(IQueryBuilder $query, string $sql = '') + public static function debugQuery(IQueryBuilder &$query, string $sql = '') { // Print the query and exit $sql = empty($sql) ? $query->getSQL() : $sql; - $params = $query->getParameters(); $sql = str_replace('*PREFIX*', 'oc_', $sql); - foreach ($params as $key => $value) { - $sql = str_replace(':'.$key, $query->getConnection()->getDatabasePlatform()->quoteStringLiteral($value), $sql); - } + self::replaceQueryParams($query, $sql); echo "{$sql}"; exit; } + public static function replaceQueryParams(IQueryBuilder &$query, string $sql) + { + $params = $query->getParameters(); + foreach ($params as $key => $value) { + $sql = str_replace(':'.$key, $query->getConnection()->getDatabasePlatform()->quoteStringLiteral($value), $sql); + } + + return $sql; + } + public function getInfoById(int $id): array { $qb = $this->connection->getQueryBuilder(); diff --git a/lib/Db/TimelineQueryDays.php b/lib/Db/TimelineQueryDays.php index 5d0382bf..aff822ae 100644 --- a/lib/Db/TimelineQueryDays.php +++ b/lib/Db/TimelineQueryDays.php @@ -179,9 +179,9 @@ trait TimelineQueryDays return $day; } - private function executeQueryWithCTEs(IQueryBuilder &$query) + private function executeQueryWithCTEs(IQueryBuilder &$query, string $psql = '') { - $sql = $query->getSQL(); + $sql = empty($psql) ? $query->getSQL() : $psql; $params = $query->getParameters(); $types = $query->getParameterTypes(); @@ -196,7 +196,7 @@ trait TimelineQueryDays /** * Get all folders inside a top folder. */ - private function joinSubfoldersRecursive( + private function addSubfolderJoinParams( IQueryBuilder &$query, Folder &$folder, bool $archive @@ -226,7 +226,6 @@ trait TimelineQueryDays // Add query parameters $query->setParameter('topFolderId', $topFolderId, IQueryBuilder::PARAM_INT); $query->setParameter('excludedFolderId', $excludedFolderId, IQueryBuilder::PARAM_INT); - $query->innerJoin('f', 'cte_folders', 'cte_f', $query->expr()->eq('f.parent', 'cte_f.fileid')); } /** @@ -253,7 +252,8 @@ trait TimelineQueryDays $pathOp = null; if ($recursive) { // Join with folders CTE - $this->joinSubfoldersRecursive($query, $folder, $archive); + $this->addSubfolderJoinParams($query, $folder, $archive); + $query->innerJoin('f', 'cte_folders', 'cte_f', $query->expr()->eq('f.parent', 'cte_f.fileid')); } else { // If getting non-recursively folder only check for parent $pathOp = $query->expr()->eq('f.parent', $query->createNamedParameter($folder->getId(), IQueryBuilder::PARAM_INT)); diff --git a/lib/Db/TimelineQueryTags.php b/lib/Db/TimelineQueryTags.php index 3af7079f..f4b97e33 100644 --- a/lib/Db/TimelineQueryTags.php +++ b/lib/Db/TimelineQueryTags.php @@ -58,7 +58,17 @@ trait TimelineQueryTags $query->innerJoin('stom', 'memories', 'm', $query->expr()->eq('m.fileid', 'stom.objectid')); // WHERE these photos are in the user's requested folder recursively - $query = $this->joinFilecache($query, $folder, true, false); + // This is a hack to speed up the query instead of using joinFilecache + // The problem is objectid is VARCHAR(64) and fileid is BIGINT(20), so a + // join is extremely slow. Instead, we use a subquery to check existence. + // + // https://blog.sqlauthority.com/2010/06/05/sql-server-convert-in-to-exists-performance-talk/ + + $this->addSubfolderJoinParams($query, $folder, false); + $query->innerJoin('m', 'filecache', 'f', $query->expr()->andX( + $query->expr()->eq('f.fileid', 'm.fileid'), + $query->createFunction('EXISTS (SELECT 1 from *PREFIX*cte_folders WHERE *PREFIX*cte_folders.fileid = `f`.parent)') + )); // GROUP and ORDER by tag name $query->groupBy('st.name'); @@ -80,11 +90,17 @@ trait TimelineQueryTags public function getTagPreviews(array &$tags, Folder &$folder) { + // This is really horrible but will have to do for now + $sql = ''; foreach ($tags as &$tag) { + if (!empty($sql)) { + $sql .= ' UNION ALL '; + } + $query = $this->connection->getQueryBuilder(); // SELECT all photos with this tag - $query->select('f.fileid', 'f.etag')->from( + $query->select('f.fileid', 'f.etag', 'stom.systemtagid')->from( 'systemtag_object_mapping', 'stom' )->where( @@ -96,19 +112,42 @@ trait TimelineQueryTags $query->innerJoin('stom', 'memories', 'm', $query->expr()->eq('m.fileid', 'stom.objectid')); // WHERE these photos are in the user's requested folder recursively - $query = $this->joinFilecache($query, $folder, true, false); + // See the function above for an explanation of this hack + $this->addSubfolderJoinParams($query, $folder, false); + $query->innerJoin('m', 'filecache', 'f', $query->expr()->andX( + $query->expr()->eq('f.fileid', 'm.fileid'), + $query->createFunction('EXISTS (SELECT 1 from *PREFIX*cte_folders WHERE *PREFIX*cte_folders.fileid = `f`.parent)') + )); // MAX 4 $query->setMaxResults(4); - // FETCH tag previews - $cursor = $this->executeQueryWithCTEs($query); - $tag['previews'] = $cursor->fetchAll(); + // Replace parameters + $thisSql = self::replaceQueryParams($query, $query->getSQL()); - // Post-process - foreach ($tag['previews'] as &$row) { - $row['fileid'] = (int) $row['fileid']; + // Add clause + $sql .= "({$thisSql})"; + } + + // FETCH tag previews + $cursor = $this->executeQueryWithCTEs($query, $sql); + $ans = $cursor->fetchAll(); + + // Post-process + $previewMap = []; + foreach ($ans as &$row) { + $row['fileid'] = (int) $row['fileid']; + $key = (int) $row['systemtagid']; + unset($row['systemtagid']); + if (!isset($previewMap[$key])) { + $previewMap[$key] = []; } + $previewMap[$key][] = $row; + } + + // Add previews to tags + foreach ($tags as &$tag) { + $tag['previews'] = $previewMap[$tag['id']] ?? []; } } }