refactor: face recognition to use generic

Signed-off-by: Varun Patil <varunpatil@ucla.edu>

appinfo/routes.php

@@ -59,14 +59,14 @@ return [
         ['name' => 'Tags#preview', 'url' => '/api/tags/preview/{name}', 'verb' => 'GET'],
         ['name' => 'Tags#set', 'url' => '/api/tags/set/{id}', 'verb' => 'PATCH'],
 
+        ['name' => 'Places#list', 'url' => '/api/places', 'verb' => 'GET'],
+        ['name' => 'Places#preview', 'url' => '/api/places/preview/{name}', 'verb' => 'GET'],
+
         ['name' => 'PeopleRecognize#list', 'url' => '/api/recognize/people', 'verb' => 'GET'],
         ['name' => 'PeopleRecognize#preview', 'url' => '/api/recognize/people/preview/{name}', 'verb' => 'GET'],
 
-        ['name' => 'People#facerecognitionPeople', 'url' => '/api/facerecognition/people', 'verb' => 'GET'],
+        ['name' => 'PeopleFaceRecognition#list', 'url' => '/api/facerecognition/people', 'verb' => 'GET'],
-        ['name' => 'People#facerecognitionPeoplePreview', 'url' => '/api/facerecognition/people/preview/{id}', 'verb' => 'GET'],
+        ['name' => 'PeopleFaceRecognition#preview', 'url' => '/api/facerecognition/people/preview/{name}', 'verb' => 'GET'],
-
-        ['name' => 'Places#list', 'url' => '/api/places', 'verb' => 'GET'],
-        ['name' => 'Places#preview', 'url' => '/api/places/preview/{name}', 'verb' => 'GET'],
 
         ['name' => 'Map#clusters', 'url' => '/api/map/clusters', 'verb' => 'GET'],
 

lib/Controller/GenericApiControllerUtils.php

@@ -46,6 +46,8 @@ trait GenericApiControllerUtils
 
     /**
      * Runa function and catch exceptions to return HTTP response.
+     *
+     * @param mixed $function
      */
     protected function guardEx($function): \OCP\AppFramework\Http\Response
     {

lib/Controller/GenericClusterController.php

@@ -47,6 +47,7 @@ abstract class GenericClusterController extends GenericApiController
             $this->init();
 
             $list = $this->getClusters();
+
             return new JSONResponse($list, Http::STATUS_OK);
         });
     }

lib/Controller/PeopleController.php

@@ -1,209 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-/**
- * @copyright Copyright (c) 2022 Varun Patil <radialapps@gmail.com>
- * @author Varun Patil <radialapps@gmail.com>
- * @license AGPL-3.0-or-later
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-namespace OCA\Memories\Controller;
-
-use OCA\Memories\Errors;
-use OCP\AppFramework\Http;
-use OCP\AppFramework\Http\DataDisplayResponse;
-use OCP\AppFramework\Http\DataResponse;
-use OCP\AppFramework\Http\JSONResponse;
-use OCP\Files\FileInfo;
-
-class PeopleController extends GenericApiController
-{
-    /**
-     * @NoAdminRequired
-     *
-     * Get list of faces with counts of images
-     */
-    public function facerecognitionPeople(): Http\Response
-    {
-        $user = $this->userSession->getUser();
-        if (null === $user) {
-            return Errors::NotLoggedIn();
-        }
-
-        // Check if face recognition is installed and enabled for this user
-        if (!$this->facerecognitionIsInstalled()) {
-            return Errors::NotEnabled('Face Recognition');
-        }
-
-        // If this isn't the timeline folder then things aren't going to work
-        $root = $this->getRequestRoot();
-        if ($root->isEmpty()) {
-            return Errors::NoRequestRoot();
-        }
-
-        // If the user has recognition disabled, just returns an empty response.
-        if (!$this->facerecognitionIsEnabled()) {
-            return new JSONResponse([]);
-        }
-
-        // Run actual query
-        $currentModel = (int) $this->config->getAppValue('facerecognition', 'model', -1);
-        $list = $this->timelineQuery->getFaceRecognitionPersons(
-            $root,
-            $currentModel
-        );
-        // Just append unnamed clusters to the end.
-        $list = array_merge($list, $this->timelineQuery->getFaceRecognitionClusters(
-            $root,
-            $currentModel
-        ));
-
-        return new JSONResponse($list, Http::STATUS_OK);
-    }
-
-    /**
-     * @NoAdminRequired
-     *
-     * @NoCSRFRequired
-     *
-     * Get face preview image cropped with imagick
-     *
-     * @return DataResponse
-     */
-    public function facerecognitionPeoplePreview(string $id): Http\Response
-    {
-        $user = $this->userSession->getUser();
-        if (null === $user) {
-            return Errors::NotLoggedIn();
-        }
-
-        // Check if face recognition is installed and enabled for this user
-        if (!$this->facerecognitionIsInstalled()) {
-            return Errors::NotEnabled('Face Recognition');
-        }
-
-        // Get folder to search for
-        $root = $this->getRequestRoot();
-        if ($root->isEmpty()) {
-            return Errors::NoRequestRoot();
-        }
-
-        // If the user has facerecognition disabled, just returns an empty response.
-        if (!$this->facerecognitionIsEnabled()) {
-            return new JSONResponse([]);
-        }
-
-        // Run actual query
-        $currentModel = (int) $this->config->getAppValue('facerecognition', 'model', -1);
-        $detections = $this->timelineQuery->getFaceRecognitionPreview($root, $currentModel, $id);
-
-        if (null === $detections || 0 === \count($detections)) {
-            return Errors::NotFound('detections');
-        }
-
-        return $this->getPreviewResponse($detections, 1.8);
-    }
-
-    /**
-     * Get face preview image cropped with imagick.
-     *
-     * @param array      $detections Array of detections to search
-     * @param \OCP\IUser $user       User to search for
-     * @param int        $padding    Padding to add to the face in preview
-     */
-    private function getPreviewResponse(
-        array $detections,
-        float $padding
-    ): Http\Response {
-        // Get preview manager
-        $previewManager = \OC::$server->get(\OCP\IPreview::class);
-
-        /** @var \Imagick */
-        $image = null;
-
-        // Find the first detection that has a preview
-        $userFolder = $this->rootFolder->getUserFolder($this->getUID());
-
-        foreach ($detections as &$detection) {
-            // Get the file (also checks permissions)
-            $files = $userFolder->getById($detection['file_id']);
-            if (0 === \count($files) || FileInfo::TYPE_FILE !== $files[0]->getType()) {
-                continue;
-            }
-
-            // Check read permission
-            if (!$files[0]->isReadable()) {
-                continue;
-            }
-
-            // Get (hopefully cached) preview image
-            try {
-                $preview = $previewManager->getPreview($files[0], 2048, 2048, false);
-
-                $image = new \Imagick();
-                if (!$image->readImageBlob($preview->getContent())) {
-                    throw new \Exception('Failed to read image blob');
-                }
-                $iw = $image->getImageWidth();
-                $ih = $image->getImageHeight();
-
-                if ($iw <= 0 || $ih <= 0) {
-                    $image = null;
-
-                    throw new \Exception('Invalid image size');
-                }
-            } catch (\Exception $e) {
-                continue;
-            }
-
-            // Got the preview
-            break;
-        }
-
-        // Make sure the preview is valid
-        if (null === $image) {
-            return Errors::NotFound('preview');
-        }
-
-        // Set quality and make progressive
-        $image->setImageCompressionQuality(80);
-        $image->setInterlaceScheme(\Imagick::INTERLACE_PLANE);
-
-        // Crop image
-        $dw = (float) $detection['width'];
-        $dh = (float) $detection['height'];
-        $dcx = (float) $detection['x'] + (float) $detection['width'] / 2;
-        $dcy = (float) $detection['y'] + (float) $detection['height'] / 2;
-        $faceDim = max($dw * $iw, $dh * $ih) * $padding;
-        $image->cropImage(
-            (int) $faceDim,
-            (int) $faceDim,
-            (int) ($dcx * $iw - $faceDim / 2),
-            (int) ($dcy * $ih - $faceDim / 2),
-        );
-        $image->scaleImage(512, 512, true);
-        $blob = $image->getImageBlob();
-
-        // Create and send response
-        $response = new DataDisplayResponse($blob, Http::STATUS_OK, [
-            'Content-Type' => $image->getImageMimeType(),
-        ]);
-        $response->cacheFor(3600 * 24, false, false);
-
-        return $response;
-    }
-}

lib/Controller/PeopleFaceRecognitionController.php

@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2022 Varun Patil <radialapps@gmail.com>
+ * @author Varun Patil <radialapps@gmail.com>
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Controller;
+
+class PeopleFaceRecognitionController extends GenericClusterController
+{
+    use PeopleControllerUtils;
+
+    protected function appName(): string
+    {
+        return 'Face Recognition';
+    }
+
+    protected function isEnabled(): bool
+    {
+        return $this->recognizeIsEnabled();
+    }
+
+    protected function getClusters(): array
+    {
+        return array_merge(
+            $this->timelineQuery->getFaceRecognitionPersons($this->root, $this->model()),
+            $this->timelineQuery->getFaceRecognitionClusters($this->root, $this->model())
+        );
+    }
+
+    protected function getPhotos(string $name, ?int $limit = null): array
+    {
+        return $this->timelineQuery->getFaceRecognitionPhotos($name, $this->model(), $this->root, $limit) ?? [];
+    }
+
+    protected function sortPhotosForPreview(array &$photos)
+    {
+        // Convert to recognize format (percentage position-size)
+        foreach ($photos as &$p) {
+            $p['x'] = (float) $p['x'] / $p['image_width'];
+            $p['y'] = (float) $p['y'] / $p['image_height'];
+            $p['width'] = (float) $p['width'] / $p['image_width'];
+            $p['height'] = (float) $p['height'] / $p['image_height'];
+        }
+
+        $this->sortByScores($photos);
+    }
+
+    protected function getPreviewBlob($file, $photo): array
+    {
+        return $this->cropFace($file, $photo, 1.8);
+    }
+
+    private function model(): int
+    {
+        return (int) $this->config->getAppValue('facerecognition', 'model', -1);
+    }
+}

lib/Db/TimelineQueryPeopleFaceRecognition.php

@@ -5,7 +5,6 @@ declare(strict_types=1);
 namespace OCA\Memories\Db;
 
 use OCP\DB\QueryBuilder\IQueryBuilder;
-use OCP\Files\Folder;
 use OCP\IDBConnection;
 
 trait TimelineQueryPeopleFaceRecognition
@@ -59,7 +58,7 @@ trait TimelineQueryPeopleFaceRecognition
         );
     }
 
-    public function getFaceRecognitionPreview(TimelineRoot &$root, $currentModel, $previewId)
+    public function getFaceRecognitionPhotos(string $id, int $currentModel, TimelineRoot &$root, ?int $limit)
     {
         $query = $this->connection->getQueryBuilder();
 
@@ -87,70 +86,28 @@ trait TimelineQueryPeopleFaceRecognition
         $query->innerJoin('fri', 'memories', 'm', $query->expr()->eq('m.fileid', 'fri.file'));
 
         $query->innerJoin('frf', 'facerecog_persons', 'frp', $query->expr()->eq('frp.id', 'frf.person'));
-        if (is_numeric($previewId)) {
+        if (is_numeric($id)) {
             // WHERE faces are from id persons (a cluster).
-            $query->where($query->expr()->eq('frp.id', $query->createNamedParameter($previewId)));
+            $query->where($query->expr()->eq('frp.id', $query->createNamedParameter($id)));
         } else {
             // WHERE faces are from name on persons.
-            $query->where($query->expr()->eq('frp.name', $query->createNamedParameter($previewId)));
+            $query->where($query->expr()->eq('frp.name', $query->createNamedParameter($id)));
         }
 
         // WHERE these photos are in the user's requested folder recursively
         $query = $this->joinFilecache($query, $root, true, false);
 
         // LIMIT results
-        $query->setMaxResults(15);
+        if (null !== $limit) {
+            $query->setMaxResults($limit);
+        }
 
         // Sort by date taken so we get recent photos
         $query->orderBy('m.datetaken', 'DESC');
         $query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
 
         // FETCH face detections
-        $cursor = $this->executeQueryWithCTEs($query);
+        return $this->executeQueryWithCTEs($query)->fetchAll();
-        $previews = $cursor->fetchAll();
-        if (empty($previews)) {
-            return null;
-        }
-
-        // Score the face detections
-        foreach ($previews as &$p) {
-            // Get actual pixel size of face
-            $iw = min((int) ($p['image_width'] ?: 512), 2048);
-            $ih = min((int) ($p['image_height'] ?: 512), 2048);
-
-            // Get percentage position and size
-            $p['x'] = (float) $p['x'] / $p['image_width'];
-            $p['y'] = (float) $p['y'] / $p['image_height'];
-            $p['width'] = (float) $p['width'] / $p['image_width'];
-            $p['height'] = (float) $p['height'] / $p['image_height'];
-
-            $w = (float) $p['width'];
-            $h = (float) $p['height'];
-
-            // Get center of face
-            $x = (float) $p['x'] + (float) $p['width'] / 2;
-            $y = (float) $p['y'] + (float) $p['height'] / 2;
-
-            // 3D normal distribution - if the face is closer to the center, it's better
-            $positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4);
-
-            // Root size distribution - if the image is bigger, it's better,
-            // but it doesn't matter beyond a certain point
-            $imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2);
-
-            // Faces occupying too much of the image don't look particularly good
-            $faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h);
-
-            // Combine scores
-            $p['score'] = $positionScore * $imgSizeScore * $faceSizeScore * $p['confidence'];
-        }
-
-        // Sort previews by score descending
-        usort($previews, function ($a, $b) {
-            return $b['score'] <=> $a['score'];
-        });
-
-        return $previews;
     }
 
     public function getFaceRecognitionClusters(TimelineRoot &$root, int $currentModel, bool $show_singles = false, bool $show_hidden = false)
@@ -261,7 +218,7 @@ trait TimelineQueryPeopleFaceRecognition
     }
 
     /** Convert face fields to object */
-    private function processFaceRecognitionDetection(&$row, $days = false)
+    private function processFaceRecognitionDetection(&$row)
     {
         if (!isset($row)) {
             return;
@@ -272,15 +229,13 @@ trait TimelineQueryPeopleFaceRecognition
             return;
         }
 
-        if (!$days) {
+        // Get percentage position and size
-            $row['facerect'] = [
+        $row['facerect'] = [
-                // Get percentage position and size
+            'w' => (float) $row['face_width'] / $row['image_width'],
-                'w' => (float) $row['face_width'] / $row['image_width'],
+            'h' => (float) $row['face_height'] / $row['image_height'],
-                'h' => (float) $row['face_height'] / $row['image_height'],
+            'x' => (float) $row['face_x'] / $row['image_width'],
-                'x' => (float) $row['face_x'] / $row['image_width'],
+            'y' => (float) $row['face_y'] / $row['image_height'],
-                'y' => (float) $row['face_y'] / $row['image_height'],
+        ];
-            ];
-        }
 
         unset($row['face_x'], $row['face_y'], $row['face_w'], $row['face_h'], $row['image_height'], $row['image_width']);
     }