Speed up tags

lib/Controller/ApiController.php

@@ -278,7 +278,7 @@ class ApiController extends Controller
 
         // Run actual query
         $list = [];
-        $t = (int) ($this->request->getParam('t'));
+        $t = (int) $this->request->getParam('t');
         if ($t & 1) { // personal
             $list = array_merge($list, $this->timelineQuery->getAlbums($user->getUID()));
         }

lib/Db/TimelineQuery.php

@@ -22,20 +22,27 @@ class TimelineQuery
         $this->connection = $connection;
     }
 
-    public static function debugQuery(IQueryBuilder $query, string $sql = '')
+    public static function debugQuery(IQueryBuilder &$query, string $sql = '')
     {
         // Print the query and exit
         $sql = empty($sql) ? $query->getSQL() : $sql;
-        $params = $query->getParameters();
         $sql = str_replace('*PREFIX*', 'oc_', $sql);
-        foreach ($params as $key => $value) {
+        self::replaceQueryParams($query, $sql);
-            $sql = str_replace(':'.$key, $query->getConnection()->getDatabasePlatform()->quoteStringLiteral($value), $sql);
-        }
         echo "{$sql}";
 
         exit;
     }
 
+    public static function replaceQueryParams(IQueryBuilder &$query, string $sql)
+    {
+        $params = $query->getParameters();
+        foreach ($params as $key => $value) {
+            $sql = str_replace(':'.$key, $query->getConnection()->getDatabasePlatform()->quoteStringLiteral($value), $sql);
+        }
+
+        return $sql;
+    }
+
     public function getInfoById(int $id): array
     {
         $qb = $this->connection->getQueryBuilder();

lib/Db/TimelineQueryDays.php

@@ -179,9 +179,9 @@ trait TimelineQueryDays
         return $day;
     }
 
-    private function executeQueryWithCTEs(IQueryBuilder &$query)
+    private function executeQueryWithCTEs(IQueryBuilder &$query, string $psql = '')
     {
-        $sql = $query->getSQL();
+        $sql = empty($psql) ? $query->getSQL() : $psql;
         $params = $query->getParameters();
         $types = $query->getParameterTypes();
 
@@ -196,7 +196,7 @@ trait TimelineQueryDays
     /**
      * Get all folders inside a top folder.
      */
-    private function joinSubfoldersRecursive(
+    private function addSubfolderJoinParams(
         IQueryBuilder &$query,
         Folder &$folder,
         bool $archive
@@ -226,7 +226,6 @@ trait TimelineQueryDays
         // Add query parameters
         $query->setParameter('topFolderId', $topFolderId, IQueryBuilder::PARAM_INT);
         $query->setParameter('excludedFolderId', $excludedFolderId, IQueryBuilder::PARAM_INT);
-        $query->innerJoin('f', 'cte_folders', 'cte_f', $query->expr()->eq('f.parent', 'cte_f.fileid'));
     }
 
     /**
@@ -253,7 +252,8 @@ trait TimelineQueryDays
         $pathOp = null;
         if ($recursive) {
             // Join with folders CTE
-            $this->joinSubfoldersRecursive($query, $folder, $archive);
+            $this->addSubfolderJoinParams($query, $folder, $archive);
+            $query->innerJoin('f', 'cte_folders', 'cte_f', $query->expr()->eq('f.parent', 'cte_f.fileid'));
         } else {
             // If getting non-recursively folder only check for parent
             $pathOp = $query->expr()->eq('f.parent', $query->createNamedParameter($folder->getId(), IQueryBuilder::PARAM_INT));

lib/Db/TimelineQueryTags.php

@@ -58,7 +58,17 @@ trait TimelineQueryTags
         $query->innerJoin('stom', 'memories', 'm', $query->expr()->eq('m.fileid', 'stom.objectid'));
 
         // WHERE these photos are in the user's requested folder recursively
-        $query = $this->joinFilecache($query, $folder, true, false);
+        // This is a hack to speed up the query instead of using joinFilecache
+        // The problem is objectid is VARCHAR(64) and fileid is BIGINT(20), so a
+        // join is extremely slow. Instead, we use a subquery to check existence.
+        //
+        // https://blog.sqlauthority.com/2010/06/05/sql-server-convert-in-to-exists-performance-talk/
+
+        $this->addSubfolderJoinParams($query, $folder, false);
+        $query->innerJoin('m', 'filecache', 'f', $query->expr()->andX(
+            $query->expr()->eq('f.fileid', 'm.fileid'),
+            $query->createFunction('EXISTS (SELECT 1 from *PREFIX*cte_folders WHERE *PREFIX*cte_folders.fileid = `f`.parent)')
+        ));
 
         // GROUP and ORDER by tag name
         $query->groupBy('st.name');
@@ -80,11 +90,17 @@ trait TimelineQueryTags
 
     public function getTagPreviews(array &$tags, Folder &$folder)
     {
+        // This is really horrible but will have to do for now
+        $sql = '';
         foreach ($tags as &$tag) {
+            if (!empty($sql)) {
+                $sql .= ' UNION ALL ';
+            }
+
             $query = $this->connection->getQueryBuilder();
 
             // SELECT all photos with this tag
-            $query->select('f.fileid', 'f.etag')->from(
+            $query->select('f.fileid', 'f.etag', 'stom.systemtagid')->from(
                 'systemtag_object_mapping',
                 'stom'
             )->where(
@@ -96,19 +112,42 @@ trait TimelineQueryTags
             $query->innerJoin('stom', 'memories', 'm', $query->expr()->eq('m.fileid', 'stom.objectid'));
 
             // WHERE these photos are in the user's requested folder recursively
-            $query = $this->joinFilecache($query, $folder, true, false);
+            // See the function above for an explanation of this hack
+            $this->addSubfolderJoinParams($query, $folder, false);
+            $query->innerJoin('m', 'filecache', 'f', $query->expr()->andX(
+                $query->expr()->eq('f.fileid', 'm.fileid'),
+                $query->createFunction('EXISTS (SELECT 1 from *PREFIX*cte_folders WHERE *PREFIX*cte_folders.fileid = `f`.parent)')
+            ));
 
             // MAX 4
             $query->setMaxResults(4);
 
+            // Replace parameters
+            $thisSql = self::replaceQueryParams($query, $query->getSQL());
+
+            // Add clause
+            $sql .= "({$thisSql})";
+        }
+
         // FETCH tag previews
-            $cursor = $this->executeQueryWithCTEs($query);
+        $cursor = $this->executeQueryWithCTEs($query, $sql);
-            $tag['previews'] = $cursor->fetchAll();
+        $ans = $cursor->fetchAll();
 
         // Post-process
-            foreach ($tag['previews'] as &$row) {
+        $previewMap = [];
+        foreach ($ans as &$row) {
             $row['fileid'] = (int) $row['fileid'];
+            $key = (int) $row['systemtagid'];
+            unset($row['systemtagid']);
+            if (!isset($previewMap[$key])) {
+                $previewMap[$key] = [];
             }
+            $previewMap[$key][] = $row;
+        }
+
+        // Add previews to tags
+        foreach ($tags as &$tag) {
+            $tag['previews'] = $previewMap[$tag['id']] ?? [];
         }
     }
 }