refactor: use generic for recognize

Signed-off-by: Varun Patil <varunpatil@ucla.edu>

appinfo/routes.php

@@ -59,8 +59,9 @@ return [
         ['name' => 'Tags#preview', 'url' => '/api/tags/preview/{name}', 'verb' => 'GET'],
         ['name' => 'Tags#set', 'url' => '/api/tags/set/{id}', 'verb' => 'PATCH'],
 
-        ['name' => 'People#recognizePeople', 'url' => '/api/recognize/people', 'verb' => 'GET'],
+        ['name' => 'PeopleRecognize#list', 'url' => '/api/recognize/people', 'verb' => 'GET'],
-        ['name' => 'People#recognizePeoplePreview', 'url' => '/api/recognize/people/preview/{id}', 'verb' => 'GET'],
+        ['name' => 'PeopleRecognize#preview', 'url' => '/api/recognize/people/preview/{name}', 'verb' => 'GET'],
+
         ['name' => 'People#facerecognitionPeople', 'url' => '/api/facerecognition/people', 'verb' => 'GET'],
         ['name' => 'People#facerecognitionPeoplePreview', 'url' => '/api/facerecognition/people/preview/{id}', 'verb' => 'GET'],
 

lib/Controller/AlbumsController.php

@@ -85,6 +85,7 @@ class AlbumsController extends GenericClusterController
 
         // Get files
         $id = (int) $album['album_id'];
+
         return $this->timelineQuery->getAlbumFiles($id, $limit) ?? [];
     }
 }

lib/Controller/GenericApiControllerUtils.php

@@ -23,10 +23,7 @@ declare(strict_types=1);
 
 namespace OCA\Memories\Controller;
 
-use OCA\Memories\Errors;
 use OCP\App\IAppManager;
-use OCP\AppFramework\Http;
-use OCP\AppFramework\Http\DataDisplayResponse;
 use OCP\IConfig;
 
 trait GenericApiControllerUtils
@@ -92,43 +89,4 @@ trait GenericApiControllerUtils
     {
         return \OCA\Memories\Util::placesGISType() > 0;
     }
-
-    /**
-     * Given a list of file ids, return the first preview image possible.
-     */
-    protected function getPreviewFromImageList(array $list, int $quality = 512): Http\Response
-    {
-        // Get preview manager
-        $previewManager = \OC::$server->get(\OCP\IPreview::class);
-
-        // Try to get a preview
-        $userFolder = $this->rootFolder->getUserFolder($this->getUID());
-        foreach ($list as &$img) {
-            // Get the file
-            $files = $userFolder->getById($img);
-            if (0 === \count($files)) {
-                continue;
-            }
-
-            // Check read permission
-            if (!$files[0]->isReadable()) {
-                continue;
-            }
-
-            // Get preview image
-            try {
-                $preview = $previewManager->getPreview($files[0], $quality, $quality, false);
-                $response = new DataDisplayResponse($preview->getContent(), Http::STATUS_OK, [
-                    'Content-Type' => $preview->getMimeType(),
-                ]);
-                $response->cacheFor(3600 * 24, false, false);
-
-                return $response;
-            } catch (\Exception $e) {
-                continue;
-            }
-        }
-
-        return Errors::NotFound('preview from list');
-    }
 }

lib/Controller/GenericClusterController.php

@@ -27,6 +27,7 @@ use OCA\Memories\Db\TimelineRoot;
 use OCA\Memories\Errors;
 use OCA\Memories\HttpResponseException;
 use OCP\AppFramework\Http;
+use OCP\AppFramework\Http\DataDisplayResponse;
 use OCP\AppFramework\Http\JSONResponse;
 
 abstract class GenericClusterController extends GenericApiController
@@ -80,7 +81,7 @@ abstract class GenericClusterController extends GenericApiController
             $this->sortFilesForPreview($files);
 
             // Get preview from image list
-            return $this->getPreviewFromImageList($this->getFileIds($files));
+            return $this->getPreviewFromImageList($files);
         } catch (HttpResponseException $e) {
             return $e->response;
         } catch (\Exception $e) {
@@ -101,7 +102,8 @@ abstract class GenericClusterController extends GenericApiController
             $this->init();
 
             // Get list of all files in this cluster
-            $fileIds = $this->getFileIds($this->getFiles($name));
+            $files = $this->getFiles($name);
+            $fileIds = array_map([$this, 'getFileId'], $files);
 
             // Get download handle
             $filename = $this->clusterName($name);
@@ -140,6 +142,60 @@ abstract class GenericClusterController extends GenericApiController
      */
     abstract protected function getFiles(string $name, ?int $limit = null): array;
 
+    /**
+     * Human readable name for the cluster.
+     */
+    protected function clusterName(string $name)
+    {
+        return $name;
+    }
+
+    /**
+     * Put the file objects in priority list.
+     * Works on the array in place.
+     */
+    protected function sortFilesForPreview(array &$files)
+    {
+        shuffle($files);
+    }
+
+    /**
+     * Quality to use for the preview file.
+     */
+    protected function getPreviewQuality(): int
+    {
+        return 512;
+    }
+
+    /**
+     * Perform any post processing and get the blob from the preview file.
+     *
+     * @param \OCP\Files\SimpleFS\ISimpleFile $file
+     * @param array                           $object The file object
+     *
+     * @return [Blob, mimetype] of data
+     */
+    protected function getPreviewBlob($file, $object): array
+    {
+        return [$file->getContent(), $file->getMimeType()];
+    }
+
+    /**
+     * Get the file ID for a file object.
+     */
+    protected function getFileId(array $file): int
+    {
+        return (int) $file['fileid'];
+    }
+
+    /**
+     * Should the timeline root be queried?
+     */
+    protected function useTimelineRoot(): bool
+    {
+        return true;
+    }
+
     /**
      * Initialize and check if the app is enabled.
      * Gets the root node if required.
@@ -165,43 +221,45 @@ abstract class GenericClusterController extends GenericApiController
     }
 
     /**
-     * Should the timeline root be queried?
+     * Given a list of file objects, return the first preview image possible.
      */
-    protected function useTimelineRoot(): bool
+    private function getPreviewFromImageList(array $list): Http\Response
     {
-        return true;
+        // Get preview manager
+        $previewManager = \OC::$server->get(\OCP\IPreview::class);
+
+        // Try to get a preview
+        $userFolder = $this->rootFolder->getUserFolder($this->getUID());
+        foreach ($list as $img) {
+            // Get the file
+            $files = $userFolder->getById($this->getFileId($img));
+            if (0 === \count($files)) {
+                continue;
             }
 
-    /**
+            // Check read permission
-     * Human readable name for the cluster.
+            if (!$files[0]->isReadable()) {
-     */
+                continue;
-    protected function clusterName(string $name)
-    {
-        return $name;
             }
 
-    /**
+            // Get preview image
-     * Put the file objects in priority list.
+            try {
-     * Works on the array in place.
+                $quality = $this->getPreviewQuality();
-     */
+                $file = $previewManager->getPreview($files[0], $quality, $quality, false);
-    protected function sortFilesForPreview(array &$files)
+
-    {
+                [$blob, $mimetype] = $this->getPreviewBlob($file, $img);
-        shuffle($files);
+
+                $response = new DataDisplayResponse($blob, Http::STATUS_OK, [
+                    'Content-Type' => $mimetype,
+                ]);
+                $response->cacheFor(3600 * 24, false, false);
+
+                return $response;
+            } catch (\Exception $e) {
+                continue;
+            }
         }
 
-    /**
+        return Errors::NotFound('preview from list');
-     * Get the file ID for a file object.
-     */
-    protected function getFileId(array $file): int
-    {
-        return (int) $file['fileid'];
-    }
-
-    /**
-     * Get array of fileIds from array of file objects.
-     */
-    private function getFileIds(array $files): array
-    {
-        return array_map([$this, 'getFileId'], $files);
     }
 }

lib/Controller/PeopleController.php

@@ -32,77 +32,6 @@ use OCP\Files\FileInfo;
 
 class PeopleController extends GenericApiController
 {
-    /**
-     * @NoAdminRequired
-     *
-     * Get list of faces with counts of images
-     */
-    public function recognizePeople(): Http\Response
-    {
-        try {
-            $uid = $this->getUID();
-        } catch (\Exception $e) {
-            return Errors::NotLoggedIn();
-        }
-
-        // Check faces enabled for this user
-        if (!$this->recognizeIsEnabled()) {
-            return Errors::NotEnabled('Recognize');
-        }
-
-        // If this isn't the timeline folder then things aren't going to work
-        $root = $this->getRequestRoot();
-        if ($root->isEmpty()) {
-            return Errors::NoRequestRoot();
-        }
-
-        // Run actual query
-        $list = $this->timelineQuery->getPeopleRecognize(
-            $root,
-            $uid
-        );
-
-        return new JSONResponse($list, Http::STATUS_OK);
-    }
-
-    /**
-     * @NoAdminRequired
-     *
-     * @NoCSRFRequired
-     *
-     * Get face preview image cropped with imagick
-     *
-     * @return DataResponse
-     */
-    public function recognizePeoplePreview(int $id): Http\Response
-    {
-        try {
-            $uid = $this->getUID();
-        } catch (\Exception $e) {
-            return Errors::NotLoggedIn();
-        }
-
-        // Check faces enabled for this user
-        if (!$this->recognizeIsEnabled()) {
-            return Errors::NotEnabled('Recognize');
-        }
-
-        // Get folder to search for
-        $root = $this->getRequestRoot();
-        if ($root->isEmpty()) {
-            return Errors::NoRequestRoot();
-        }
-
-        // Run actual query
-        $detections = $this->timelineQuery->getPeopleRecognizePreview($root, $id, $uid);
-
-        if (0 === \count($detections)) {
-            return Errors::NotFound('detections');
-        }
-
-        return $this->getPreviewResponse($detections, 1.5);
-    }
-
     /**
      * @NoAdminRequired
      *

lib/Controller/PeopleControllerUtils.php

@@ -0,0 +1,114 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Varun Patil <radialapps@gmail.com>
+ * @author Varun Patil <radialapps@gmail.com>
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Controller;
+
+trait PeopleControllerUtils
+{
+    private function sortByScores(array &$list): void
+    {
+        // Score the face detections
+        foreach ($list as &$p) {
+            // Make sure we have integers
+            $p['fileid'] = (int) $p['fileid'];
+
+            // Get actual pixel size of face
+            $iw = min((int) ($p['image_width'] ?: 512), 2048);
+            $ih = min((int) ($p['image_height'] ?: 512), 2048);
+            $w = (float) $p['width'];
+            $h = (float) $p['height'];
+
+            // Get center of face
+            $x = (float) $p['x'] + (float) $p['width'] / 2;
+            $y = (float) $p['y'] + (float) $p['height'] / 2;
+
+            // 3D normal distribution - if the face is closer to the center, it's better
+            $positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4);
+
+            // Root size distribution - if the image is bigger, it's better,
+            // but it doesn't matter beyond a certain point
+            $imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2);
+
+            // Faces occupying too much of the image don't look particularly good
+            $faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h);
+
+            // Combine scores
+            $p['score'] = $positionScore * $imgSizeScore * $faceSizeScore;
+        }
+
+        // Sort previews by score descending
+        usort($list, fn ($a, $b) => $b['score'] <=> $a['score']);
+    }
+
+    /**
+     * Crop the preview to the face.
+     *
+     * @param \OCP\Files\SimpleFS\ISimpleFile $file
+     * @param array                           $object The face object
+     *
+     * @return [Blob, mimetype] of resulting image
+     *
+     * @throws \Exception if file could not be used
+     */
+    private function cropFace($file, array $object, float $padding)
+    {
+        /** @var \Imagick */
+        $image = null;
+
+        try {
+            $image = new \Imagick();
+            $image->readImageBlob($file->getContent());
+        } catch (\ImagickException $e) {
+            throw new \Exception('Could not read image');
+        }
+
+        $iw = $image->getImageWidth();
+        $ih = $image->getImageHeight();
+
+        if ($iw <= 0 || $ih <= 0) {
+            $image = null;
+
+            throw new \Exception('Invalid image size');
+        }
+
+        // Set quality and make progressive
+        $image->setImageCompressionQuality(80);
+        $image->setInterlaceScheme(\Imagick::INTERLACE_PLANE);
+
+        // Crop image
+        $dw = (float) $object['width'];
+        $dh = (float) $object['height'];
+        $dcx = (float) $object['x'] + (float) $object['width'] / 2;
+        $dcy = (float) $object['y'] + (float) $object['height'] / 2;
+        $faceDim = max($dw * $iw, $dh * $ih) * $padding;
+        $image->cropImage(
+            (int) $faceDim,
+            (int) $faceDim,
+            (int) ($dcx * $iw - $faceDim / 2),
+            (int) ($dcy * $ih - $faceDim / 2),
+        );
+        $image->scaleImage(512, 512, true);
+
+        return [$image->getImageBlob(), $image->getImageMimeType()];
+    }
+}

lib/Controller/PeopleRecognizeController.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2022 Varun Patil <radialapps@gmail.com>
+ * @author Varun Patil <radialapps@gmail.com>
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Controller;
+
+class PeopleRecognizeController extends GenericClusterController
+{
+    use PeopleControllerUtils;
+
+    protected function appName(): string
+    {
+        return 'Recognize';
+    }
+
+    protected function isEnabled(): bool
+    {
+        return $this->recognizeIsEnabled();
+    }
+
+    protected function getClusters(): array
+    {
+        return $this->timelineQuery->getPeopleRecognize($this->root, $this->getUID());
+    }
+
+    protected function getFiles(string $name, ?int $limit = null): array
+    {
+        return $this->timelineQuery->getPeopleRecognizeFiles((int) $name, $this->root, $limit) ?? [];
+    }
+
+    protected function sortFilesForPreview(array &$files)
+    {
+        $this->sortByScores($files);
+    }
+
+    protected function getPreviewBlob($file, $object): array
+    {
+        return $this->cropFace($file, $object, 1.5);
+    }
+}

lib/Db/TimelineQueryPeopleRecognize.php

@@ -105,7 +105,7 @@ trait TimelineQueryPeopleRecognize
         return $faces;
     }
 
-    public function getPeopleRecognizePreview(TimelineRoot &$root, int $id, string $uid): array
+    public function getPeopleRecognizeFiles(int $id, TimelineRoot $root, ?int $limit): array
     {
         $query = $this->connection->getQueryBuilder();
 
@@ -122,11 +122,8 @@ trait TimelineQueryPeopleRecognize
             'm.datetaken',              // Just in case, for postgres
         )->from('recognize_face_detections', 'rfd');
 
-        // WHERE detection belongs to this cluster AND user
+        // WHERE detection belongs to this cluster
-        $query->where($query->expr()->andX(
+        $query->where($query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)));
-            $query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)),
-            $query->expr()->eq('rfd.user_id', $query->createNamedParameter($uid)),
-        ));
 
         // WHERE these photos are memories indexed
         $query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id'));
@@ -135,51 +132,16 @@ trait TimelineQueryPeopleRecognize
         $query = $this->joinFilecache($query, $root, true, false);
 
         // LIMIT results
-        $query->setMaxResults(15);
+        if (null !== $limit) {
+            $query->setMaxResults($limit);
+        }
 
         // Sort by date taken so we get recent photos
         $query->orderBy('m.datetaken', 'DESC');
         $query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
 
         // FETCH face detections
-        $cursor = $this->executeQueryWithCTEs($query);
+        return $this->executeQueryWithCTEs($query)->fetchAll();
-        $previews = $cursor->fetchAll();
-        if (empty($previews)) {
-            return [];
-        }
-
-        // Score the face detections
-        foreach ($previews as &$p) {
-            // Get actual pixel size of face
-            $iw = min((int) ($p['image_width'] ?: 512), 2048);
-            $ih = min((int) ($p['image_height'] ?: 512), 2048);
-            $w = (float) $p['width'];
-            $h = (float) $p['height'];
-
-            // Get center of face
-            $x = (float) $p['x'] + (float) $p['width'] / 2;
-            $y = (float) $p['y'] + (float) $p['height'] / 2;
-
-            // 3D normal distribution - if the face is closer to the center, it's better
-            $positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4);
-
-            // Root size distribution - if the image is bigger, it's better,
-            // but it doesn't matter beyond a certain point
-            $imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2);
-
-            // Faces occupying too much of the image don't look particularly good
-            $faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h);
-
-            // Combine scores
-            $p['score'] = $positionScore * $imgSizeScore * $faceSizeScore;
-        }
-
-        // Sort previews by score descending
-        usort($previews, function ($a, $b) {
-            return $b['score'] <=> $a['score'];
-        });
-
-        return $previews;
     }
 
     /** Convert face fields to object */