faces: crop with imagick (#72)

appinfo/routes.php

@@ -26,6 +26,7 @@ return [
         ['name' => 'api#day', 'url' => '/api/days/{id}', 'verb' => 'GET'],
         ['name' => 'api#tags', 'url' => '/api/tags', 'verb' => 'GET'],
         ['name' => 'api#faces', 'url' => '/api/faces', 'verb' => 'GET'],
+        ['name' => 'api#facePreview', 'url' => '/api/faces/preview/{id}', 'verb' => 'GET'],
         ['name' => 'api#imageInfo', 'url' => '/api/info/{id}', 'verb' => 'GET'],
         ['name' => 'api#imageEdit', 'url' => '/api/edit/{id}', 'verb' => 'PATCH'],
         ['name' => 'api#archive', 'url' => '/api/archive/{id}', 'verb' => 'PATCH'],

lib/Controller/ApiController.php

@@ -35,13 +35,16 @@ use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\JSONResponse;
 use OCP\AppFramework\Http\StreamResponse;
 use OCP\AppFramework\Http\ContentSecurityPolicy;
+use OCP\AppFramework\Http\DataResponse;
+use OCP\AppFramework\Http\DataDisplayResponse;
 use OCP\Files\IRootFolder;
+use OCP\Files\FileInfo;
+use OCP\Files\Folder;
 use OCP\IConfig;
 use OCP\IDBConnection;
 use OCP\IRequest;
 use OCP\IUserSession;
-use OCP\Files\FileInfo;
+use OCP\IPreview;
-use OCP\Files\Folder;
 
 class ApiController extends Controller {
     private IConfig $config;
@@ -51,6 +54,7 @@ class ApiController extends Controller {
     private IAppManager $appManager;
     private TimelineQuery $timelineQuery;
     private TimelineWrite $timelineWrite;
+    private IPreview $previewManager;
 
     public function __construct(
         IRequest $request,
@@ -58,7 +62,8 @@ class ApiController extends Controller {
         IUserSession $userSession,
         IDBConnection $connection,
         IRootFolder $rootFolder,
-        IAppManager $appManager) {
+        IAppManager $appManager,
+        IPreview $previewManager) {
 
         parent::__construct(Application::APPNAME, $request);
 
@@ -67,6 +72,7 @@ class ApiController extends Controller {
         $this->connection = $connection;
         $this->rootFolder = $rootFolder;
         $this->appManager = $appManager;
+        $this->previewManager = $previewManager;
         $this->timelineQuery = new TimelineQuery($this->connection);
         $this->timelineWrite = new TimelineWrite($connection);
     }
@@ -385,33 +391,91 @@ class ApiController extends Controller {
             $folder,
         );
 
-        // Preload all face previews
-        $previews = $this->timelineQuery->getFacePreviews($folder);
-
-        // Convert to map with key as cluster_id
-        $previews_map = [];
-        foreach ($previews as &$preview) {
-            $key = $preview["cluster_id"];
-            if (!array_key_exists($key, $previews_map)) {
-                $previews_map[$key] = [];
-            }
-            unset($preview["cluster_id"]);
-            $previews_map[$key][] = $preview;
-        }
-
-        // Add all previews to list
-        foreach ($list as &$face) {
-            $key = $face["id"];
-            if (array_key_exists($key, $previews_map)) {
-                $face["previews"] = $previews_map[$key];
-            } else {
-                $face["previews"] = [];
-            }
-        }
-
         return new JSONResponse($list, Http::STATUS_OK);
     }
 
+    /**
+     * @NoAdminRequired
+     * @NoCSRFRequired
+     *
+     * Get face preview image cropped with imagick
+     * @return DataResponse
+     */
+    public function facePreview(string $id): Http\Response {
+        $user = $this->userSession->getUser();
+        if (is_null($user)) {
+            return new DataResponse([], Http::STATUS_PRECONDITION_FAILED);
+        }
+
+        // Check faces enabled for this user
+        if (!$this->recognizeIsEnabled()) {
+            return new DataResponse([], Http::STATUS_PRECONDITION_FAILED);
+        }
+
+        // Get folder to search for
+        $folder = $this->getRequestFolder();
+        if (is_null($folder)) {
+            return new JSONResponse([], Http::STATUS_NOT_FOUND);
+        }
+
+        // Run actual query
+        $detections = $this->timelineQuery->getFacePreviewDetection($folder, intval($id));
+        if (is_null($detections) || count($detections) == 0) {
+            return new DataResponse([], Http::STATUS_NOT_FOUND);
+        }
+
+        // Find the first detection that has a preview
+        $preview = null;
+        foreach ($detections as &$detection) {
+            // Get the file (also checks permissions)
+            $files = $folder->getById($detection["file_id"]);
+            if (count($files) == 0 || $files[0]->getType() != FileInfo::TYPE_FILE) {
+                continue;
+            }
+
+            // Get (hopefully cached) preview image
+            try {
+                $preview = $this->previewManager->getPreview($files[0], 2048, 2048, false);
+            } catch (\Exception $e) {
+                continue;
+            }
+
+            // Got the preview
+            break;
+        }
+
+        // Make sure the preview is valid
+        if (is_null($preview)) {
+            return new DataResponse([], Http::STATUS_NOT_FOUND);
+        }
+
+        // Crop image
+        $image = new \Imagick();
+        $image->readImageBlob($preview->getContent());
+        $iw = $image->getImageWidth();
+        $ih = $image->getImageHeight();
+        $dw = floatval($detection["width"]);
+        $dh = floatval($detection["height"]);
+        $dcx = floatval($detection["x"]) + floatval($detection["width"]) / 2;
+        $dcy = floatval($detection["y"]) + floatval($detection["height"]) / 2;
+        $faceDim = max($dw * $iw, $dh * $ih) * 1.5;
+        $image->cropImage(
+            intval($faceDim),
+            intval($faceDim),
+            intval($dcx * $iw - $faceDim / 2),
+            intval($dcy * $ih - $faceDim / 2),
+        );
+        $image->resizeImage(256, 256, \Imagick::FILTER_LANCZOS, 1);
+        $blob = $image->getImageBlob();
+
+        // Create and send response
+        $response = new DataDisplayResponse($blob, Http::STATUS_OK, [
+            'Content-Type' => $image->getImageMimeType(),
+        ]);
+        $response->cacheFor(3600 * 24, false, false);
+        return $response;
+    }
+
     /**
      * @NoAdminRequired
      *

lib/Db/TimelineQueryFaces.php

@@ -73,21 +73,17 @@ trait TimelineQueryFaces {
         return $faces;
     }
 
-    public function getFacePreviews(Folder $folder) {
+    public function getFacePreviewDetection(Folder &$folder, int $id) {
         $query = $this->connection->getQueryBuilder();
 
-        // Windowing
-        $rowNumber = $query->createFunction('ROW_NUMBER() OVER (PARTITION BY rfd.cluster_id) as n');
-
         // SELECT face detections for ID
         $query->select(
-            'rfd.cluster_id',
+            'rfd.file_id',                                  // Needed to get the actual file
-            'rfd.file_id',
+            'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height',    // Image cropping
-            'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height',
+            'm.w as image_width', 'm.h as image_height',    // Scoring
-            'f.etag',
+            'm.fileid', 'm.datetaken',                      // Just in case, for postgres
-            $rowNumber,
         )->from('recognize_face_detections', 'rfd');
-        $query->where($query->expr()->isNotNull('rfd.cluster_id'));
+        $query->where($query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)));
 
         // WHERE these photos are memories indexed
         $query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id'));
@@ -95,33 +91,47 @@ trait TimelineQueryFaces {
         // WHERE these photos are in the user's requested folder recursively
         $query->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, true, false));
 
-        // Make this a sub query
+        // LIMIT results
-        $fun = $query->createFunction('(' . $query->getSQL() . ')');
+        $query->setMaxResults(15);
 
-        // Create outer query
+        // Sort by date taken so we get recent photos
-        $outerQuery = $this->connection->getQueryBuilder();
+        $query->orderBy('m.datetaken', 'DESC');
-        $outerQuery->setParameters($query->getParameters());
+        $query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
-        $outerQuery->select('*')->from($fun, 't');
-        $outerQuery->where($query->expr()->lte('t.n', $outerQuery->createParameter('nc')));
-        $outerQuery->setParameter('nc', 4, IQueryBuilder::PARAM_INT);
 
-        // FETCH all face detections
+        // FETCH face detections
-        $previews = $outerQuery->executeQuery()->fetchAll();
+        $previews = $query->executeQuery()->fetchAll();
-
+        if (empty($previews)) {
-        // Post-process, everthing is a number
+            return null;
-        foreach($previews as &$row) {
-            $row["cluster_id"] = intval($row["cluster_id"]);
-            $row["fileid"] = intval($row["file_id"]);
-            $row["x"] = floatval($row["x"]);
-            $row["y"] = floatval($row["y"]);
-            $row["width"] = floatval($row["width"]);
-            $row["height"] = floatval($row["height"]);
-
-            // remove stale
-            unset($row["file_id"]);
-            unset($row["n"]);
         }
 
+        // Score the face detections
+        foreach ($previews as &$p) {
+            // Get actual pixel size of face
+            $iw = min(intval($p["image_width"] ?: 512), 2048);
+            $ih = min(intval($p["image_height"] ?: 512), 2048);
+            $w = floatval($p["width"]) * $iw;
+            $h = floatval($p["height"]) * $ih;
+
+            // Get center of face
+            $x = floatval($p["x"]) + floatval($p["width"]) / 2;
+            $y = floatval($p["y"]) + floatval($p["height"]) / 2;
+
+            // 3D normal distribution - if the face is closer to the center, it's better
+            $positionScore = exp(-pow($x - 0.5, 2) * 4) * exp(-pow($y - 0.5, 2) * 4);
+
+            // Root size distribution - if the face is bigger, it's better,
+            // but it doesn't matter beyond a certain point, especially 256px ;)
+            $sizeScore = pow($w * 100, 1/4) * pow($h * 100, 1/4);
+
+            // Combine scores
+            $p["score"] = $positionScore * $sizeScore;
+        }
+
+        // Sort previews by score descending
+        usort($previews, function($a, $b) {
+            return $b["score"] <=> $a["score"];
+        });
+
         return $previews;
     }
 }

src/components/frame/Tag.vue

@@ -16,7 +16,6 @@
                     :class="{ 'error': info.flag & c.FLAG_LOAD_FAIL }"
                     :key="'fpreview-' + info.fileid"
                     :src="getPreviewUrl(info.fileid, info.etag)"
-                    :style="getCoverStyle(info)"
                     @error="info.flag |= c.FLAG_LOAD_FAIL" />
             </div>
         </div>
@@ -34,13 +33,6 @@ import { NcCounterBubble } from '@nextcloud/vue'
 import GlobalMixin from '../../mixins/GlobalMixin';
 import { constants } from '../../services/Utils';
 
-interface IFaceDetection extends IPhoto {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}
-
 @Component({
     components: {
         NcCounterBubble,
@@ -67,7 +59,7 @@ export default class Tag extends Mixins(GlobalMixin) {
 
     getPreviewUrl(fileid: number, etag: string) {
         if (this.isFace) {
-            return getPreviewUrl(fileid, etag, false, 2048);
+            return generateUrl('/apps/memories/api/faces/preview/' + this.data.fileid);
         }
         return getPreviewUrl(fileid, etag, true, 256);
     }
@@ -80,24 +72,24 @@ export default class Tag extends Mixins(GlobalMixin) {
         // Reset state
         this.error = false;
 
-        // Look for previews
+        // Add dummy preview if face
-        if (!this.data.previews) {
+        if (this.isFace) {
+            this.previews = [{ fileid: 0, etag: '', flag: 0 }];
             return;
         }
 
+        // Look for previews
+        if (!this.data.previews) return;
+
         // Reset flag
         this.data.previews.forEach((p) => p.flag = 0);
 
-        if (this.isFace) {
+        // Get 4 or 1 preview(s)
-            const face = this.chooseFaceDetection(this.data.previews as IFaceDetection[]);
+        let data = this.data.previews;
-            this.previews = [face];
+        if (data.length < 4) {
-        } else {
+            data = data.slice(0, 1);
-            let data = this.data.previews;
-            if (data.length < 4) {
-                data = data.slice(0, 1);
-            }
-            this.previews = data;
         }
+        this.previews = data;
 
         this.error = this.previews.length === 0;
     }
@@ -117,56 +109,6 @@ export default class Tag extends Mixins(GlobalMixin) {
             this.$router.push({ name: 'tags', params: { name: this.data.name }});
         }
     }
-
-    /** Choose the most appropriate face detection */
-    private chooseFaceDetection(detections: IFaceDetection[]) {
-        const scoreFacePosition = (faceDetection: IFaceDetection) => {
-            return Math.max(0, -1 * (faceDetection.x - faceDetection.width * 0.5))
-            + Math.max(0, -1 * (faceDetection.y - faceDetection.height * 0.5))
-            + Math.max(0, -1 * (1 - (faceDetection.x + faceDetection.width) - faceDetection.width * 0.5))
-            + Math.max(0, -1 * (1 - (faceDetection.y + faceDetection.height) - faceDetection.height * 0.5))
-        }
-
-        const scoreFace = (faceDetection: IFaceDetection) => {
-            return (1 - faceDetection.width * faceDetection.height) + scoreFacePosition(faceDetection);
-        }
-
-        return detections.sort((a, b) => scoreFace(a) - scoreFace(b))[0];
-    }
-
-    /**
-     * This will produce an inline style to apply to images
-     * to zoom toward the detected face
-     */
-    getCoverStyle(photo: IPhoto) {
-        if (!this.isFace) {
-            return {};
-        }
-
-        // Pass the same thing
-        const detection = photo as IFaceDetection;
-
-        // Zoom into the picture so that the face fills the --photos-face-width box nicely
-        // if the face is larger than the image, we don't zoom out (reason for the Math.max)
-        const zoom = Math.max(1, (1 / detection.width) * 0.4)
-
-        // Get center coordinate in percent
-        const horizontalCenterOfFace = (detection.x + detection.width / 2) * 100
-        const verticalCenterOfFace = (detection.y + detection.height / 2) * 100
-
-        // Get preview element dimensions
-        const elem = this.$refs.previews as HTMLElement;
-        const elemWidth = elem.clientWidth;
-        const elemHeight = elem.clientHeight;
-
-        return {
-            // we translate the image so that the center of the detected face is in the center
-            // and add the zoom
-            transform: `translate(calc(${elemWidth}px/2 - ${horizontalCenterOfFace}% ), calc(${elemHeight}px/2 - ${verticalCenterOfFace}% )) scale(${zoom})`,
-            // this is necessary for the zoom to zoom toward the center of the face
-            transformOrigin: `${horizontalCenterOfFace}% ${verticalCenterOfFace}%`,
-        }
-    }
 }
 </script>