From 3e54bc72c1d829c4a6cf5f4fa1bc604050eec0a8 Mon Sep 17 00:00:00 2001 From: Varun Patil Date: Mon, 17 Oct 2022 10:41:58 -0700 Subject: [PATCH] faces: crop with imagick (#72) --- appinfo/routes.php | 1 + lib/Controller/ApiController.php | 118 ++++++++++++++++++++++++------- lib/Db/TimelineQueryFaces.php | 76 +++++++++++--------- src/components/frame/Tag.vue | 82 ++++----------------- 4 files changed, 147 insertions(+), 130 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index e8d800dd..5e363bcd 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -26,6 +26,7 @@ return [ ['name' => 'api#day', 'url' => '/api/days/{id}', 'verb' => 'GET'], ['name' => 'api#tags', 'url' => '/api/tags', 'verb' => 'GET'], ['name' => 'api#faces', 'url' => '/api/faces', 'verb' => 'GET'], + ['name' => 'api#facePreview', 'url' => '/api/faces/preview/{id}', 'verb' => 'GET'], ['name' => 'api#imageInfo', 'url' => '/api/info/{id}', 'verb' => 'GET'], ['name' => 'api#imageEdit', 'url' => '/api/edit/{id}', 'verb' => 'PATCH'], ['name' => 'api#archive', 'url' => '/api/archive/{id}', 'verb' => 'PATCH'], diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index b6550fbb..87786dd1 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -35,13 +35,16 @@ use OCP\AppFramework\Http; use OCP\AppFramework\Http\JSONResponse; use OCP\AppFramework\Http\StreamResponse; use OCP\AppFramework\Http\ContentSecurityPolicy; +use OCP\AppFramework\Http\DataResponse; +use OCP\AppFramework\Http\DataDisplayResponse; use OCP\Files\IRootFolder; +use OCP\Files\FileInfo; +use OCP\Files\Folder; use OCP\IConfig; use OCP\IDBConnection; use OCP\IRequest; use OCP\IUserSession; -use OCP\Files\FileInfo; -use OCP\Files\Folder; +use OCP\IPreview; class ApiController extends Controller { private IConfig $config; @@ -51,6 +54,7 @@ class ApiController extends Controller { private IAppManager $appManager; private TimelineQuery $timelineQuery; private TimelineWrite $timelineWrite; + private IPreview $previewManager; public function __construct( IRequest $request, @@ -58,7 +62,8 @@ class ApiController extends Controller { IUserSession $userSession, IDBConnection $connection, IRootFolder $rootFolder, - IAppManager $appManager) { + IAppManager $appManager, + IPreview $previewManager) { parent::__construct(Application::APPNAME, $request); @@ -67,6 +72,7 @@ class ApiController extends Controller { $this->connection = $connection; $this->rootFolder = $rootFolder; $this->appManager = $appManager; + $this->previewManager = $previewManager; $this->timelineQuery = new TimelineQuery($this->connection); $this->timelineWrite = new TimelineWrite($connection); } @@ -385,33 +391,91 @@ class ApiController extends Controller { $folder, ); - // Preload all face previews - $previews = $this->timelineQuery->getFacePreviews($folder); - - // Convert to map with key as cluster_id - $previews_map = []; - foreach ($previews as &$preview) { - $key = $preview["cluster_id"]; - if (!array_key_exists($key, $previews_map)) { - $previews_map[$key] = []; - } - unset($preview["cluster_id"]); - $previews_map[$key][] = $preview; - } - - // Add all previews to list - foreach ($list as &$face) { - $key = $face["id"]; - if (array_key_exists($key, $previews_map)) { - $face["previews"] = $previews_map[$key]; - } else { - $face["previews"] = []; - } - } - return new JSONResponse($list, Http::STATUS_OK); } + /** + * @NoAdminRequired + * @NoCSRFRequired + * + * Get face preview image cropped with imagick + * @return DataResponse + */ + public function facePreview(string $id): Http\Response { + $user = $this->userSession->getUser(); + if (is_null($user)) { + return new DataResponse([], Http::STATUS_PRECONDITION_FAILED); + } + + // Check faces enabled for this user + if (!$this->recognizeIsEnabled()) { + return new DataResponse([], Http::STATUS_PRECONDITION_FAILED); + } + + // Get folder to search for + $folder = $this->getRequestFolder(); + if (is_null($folder)) { + return new JSONResponse([], Http::STATUS_NOT_FOUND); + } + + // Run actual query + $detections = $this->timelineQuery->getFacePreviewDetection($folder, intval($id)); + if (is_null($detections) || count($detections) == 0) { + return new DataResponse([], Http::STATUS_NOT_FOUND); + } + + // Find the first detection that has a preview + $preview = null; + foreach ($detections as &$detection) { + // Get the file (also checks permissions) + $files = $folder->getById($detection["file_id"]); + if (count($files) == 0 || $files[0]->getType() != FileInfo::TYPE_FILE) { + continue; + } + + // Get (hopefully cached) preview image + try { + $preview = $this->previewManager->getPreview($files[0], 2048, 2048, false); + } catch (\Exception $e) { + continue; + } + + // Got the preview + break; + } + + // Make sure the preview is valid + if (is_null($preview)) { + return new DataResponse([], Http::STATUS_NOT_FOUND); + } + + // Crop image + $image = new \Imagick(); + $image->readImageBlob($preview->getContent()); + $iw = $image->getImageWidth(); + $ih = $image->getImageHeight(); + $dw = floatval($detection["width"]); + $dh = floatval($detection["height"]); + $dcx = floatval($detection["x"]) + floatval($detection["width"]) / 2; + $dcy = floatval($detection["y"]) + floatval($detection["height"]) / 2; + $faceDim = max($dw * $iw, $dh * $ih) * 1.5; + $image->cropImage( + intval($faceDim), + intval($faceDim), + intval($dcx * $iw - $faceDim / 2), + intval($dcy * $ih - $faceDim / 2), + ); + $image->resizeImage(256, 256, \Imagick::FILTER_LANCZOS, 1); + $blob = $image->getImageBlob(); + + // Create and send response + $response = new DataDisplayResponse($blob, Http::STATUS_OK, [ + 'Content-Type' => $image->getImageMimeType(), + ]); + $response->cacheFor(3600 * 24, false, false); + return $response; + } + /** * @NoAdminRequired * diff --git a/lib/Db/TimelineQueryFaces.php b/lib/Db/TimelineQueryFaces.php index 12e10079..850fdb6b 100644 --- a/lib/Db/TimelineQueryFaces.php +++ b/lib/Db/TimelineQueryFaces.php @@ -73,21 +73,17 @@ trait TimelineQueryFaces { return $faces; } - public function getFacePreviews(Folder $folder) { + public function getFacePreviewDetection(Folder &$folder, int $id) { $query = $this->connection->getQueryBuilder(); - // Windowing - $rowNumber = $query->createFunction('ROW_NUMBER() OVER (PARTITION BY rfd.cluster_id) as n'); - // SELECT face detections for ID $query->select( - 'rfd.cluster_id', - 'rfd.file_id', - 'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height', - 'f.etag', - $rowNumber, + 'rfd.file_id', // Needed to get the actual file + 'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height', // Image cropping + 'm.w as image_width', 'm.h as image_height', // Scoring + 'm.fileid', 'm.datetaken', // Just in case, for postgres )->from('recognize_face_detections', 'rfd'); - $query->where($query->expr()->isNotNull('rfd.cluster_id')); + $query->where($query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id))); // WHERE these photos are memories indexed $query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id')); @@ -95,33 +91,47 @@ trait TimelineQueryFaces { // WHERE these photos are in the user's requested folder recursively $query->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, true, false)); - // Make this a sub query - $fun = $query->createFunction('(' . $query->getSQL() . ')'); + // LIMIT results + $query->setMaxResults(15); - // Create outer query - $outerQuery = $this->connection->getQueryBuilder(); - $outerQuery->setParameters($query->getParameters()); - $outerQuery->select('*')->from($fun, 't'); - $outerQuery->where($query->expr()->lte('t.n', $outerQuery->createParameter('nc'))); - $outerQuery->setParameter('nc', 4, IQueryBuilder::PARAM_INT); + // Sort by date taken so we get recent photos + $query->orderBy('m.datetaken', 'DESC'); + $query->addOrderBy('m.fileid', 'DESC'); // tie-breaker - // FETCH all face detections - $previews = $outerQuery->executeQuery()->fetchAll(); - - // Post-process, everthing is a number - foreach($previews as &$row) { - $row["cluster_id"] = intval($row["cluster_id"]); - $row["fileid"] = intval($row["file_id"]); - $row["x"] = floatval($row["x"]); - $row["y"] = floatval($row["y"]); - $row["width"] = floatval($row["width"]); - $row["height"] = floatval($row["height"]); - - // remove stale - unset($row["file_id"]); - unset($row["n"]); + // FETCH face detections + $previews = $query->executeQuery()->fetchAll(); + if (empty($previews)) { + return null; } + // Score the face detections + foreach ($previews as &$p) { + // Get actual pixel size of face + $iw = min(intval($p["image_width"] ?: 512), 2048); + $ih = min(intval($p["image_height"] ?: 512), 2048); + $w = floatval($p["width"]) * $iw; + $h = floatval($p["height"]) * $ih; + + // Get center of face + $x = floatval($p["x"]) + floatval($p["width"]) / 2; + $y = floatval($p["y"]) + floatval($p["height"]) / 2; + + // 3D normal distribution - if the face is closer to the center, it's better + $positionScore = exp(-pow($x - 0.5, 2) * 4) * exp(-pow($y - 0.5, 2) * 4); + + // Root size distribution - if the face is bigger, it's better, + // but it doesn't matter beyond a certain point, especially 256px ;) + $sizeScore = pow($w * 100, 1/4) * pow($h * 100, 1/4); + + // Combine scores + $p["score"] = $positionScore * $sizeScore; + } + + // Sort previews by score descending + usort($previews, function($a, $b) { + return $b["score"] <=> $a["score"]; + }); + return $previews; } } \ No newline at end of file diff --git a/src/components/frame/Tag.vue b/src/components/frame/Tag.vue index 6c2edb52..915e945b 100644 --- a/src/components/frame/Tag.vue +++ b/src/components/frame/Tag.vue @@ -16,7 +16,6 @@ :class="{ 'error': info.flag & c.FLAG_LOAD_FAIL }" :key="'fpreview-' + info.fileid" :src="getPreviewUrl(info.fileid, info.etag)" - :style="getCoverStyle(info)" @error="info.flag |= c.FLAG_LOAD_FAIL" /> @@ -34,13 +33,6 @@ import { NcCounterBubble } from '@nextcloud/vue' import GlobalMixin from '../../mixins/GlobalMixin'; import { constants } from '../../services/Utils'; -interface IFaceDetection extends IPhoto { - x: number; - y: number; - width: number; - height: number; -} - @Component({ components: { NcCounterBubble, @@ -67,7 +59,7 @@ export default class Tag extends Mixins(GlobalMixin) { getPreviewUrl(fileid: number, etag: string) { if (this.isFace) { - return getPreviewUrl(fileid, etag, false, 2048); + return generateUrl('/apps/memories/api/faces/preview/' + this.data.fileid); } return getPreviewUrl(fileid, etag, true, 256); } @@ -80,24 +72,24 @@ export default class Tag extends Mixins(GlobalMixin) { // Reset state this.error = false; - // Look for previews - if (!this.data.previews) { + // Add dummy preview if face + if (this.isFace) { + this.previews = [{ fileid: 0, etag: '', flag: 0 }]; return; } + // Look for previews + if (!this.data.previews) return; + // Reset flag this.data.previews.forEach((p) => p.flag = 0); - if (this.isFace) { - const face = this.chooseFaceDetection(this.data.previews as IFaceDetection[]); - this.previews = [face]; - } else { - let data = this.data.previews; - if (data.length < 4) { - data = data.slice(0, 1); - } - this.previews = data; + // Get 4 or 1 preview(s) + let data = this.data.previews; + if (data.length < 4) { + data = data.slice(0, 1); } + this.previews = data; this.error = this.previews.length === 0; } @@ -117,56 +109,6 @@ export default class Tag extends Mixins(GlobalMixin) { this.$router.push({ name: 'tags', params: { name: this.data.name }}); } } - - /** Choose the most appropriate face detection */ - private chooseFaceDetection(detections: IFaceDetection[]) { - const scoreFacePosition = (faceDetection: IFaceDetection) => { - return Math.max(0, -1 * (faceDetection.x - faceDetection.width * 0.5)) - + Math.max(0, -1 * (faceDetection.y - faceDetection.height * 0.5)) - + Math.max(0, -1 * (1 - (faceDetection.x + faceDetection.width) - faceDetection.width * 0.5)) - + Math.max(0, -1 * (1 - (faceDetection.y + faceDetection.height) - faceDetection.height * 0.5)) - } - - const scoreFace = (faceDetection: IFaceDetection) => { - return (1 - faceDetection.width * faceDetection.height) + scoreFacePosition(faceDetection); - } - - return detections.sort((a, b) => scoreFace(a) - scoreFace(b))[0]; - } - - /** - * This will produce an inline style to apply to images - * to zoom toward the detected face - */ - getCoverStyle(photo: IPhoto) { - if (!this.isFace) { - return {}; - } - - // Pass the same thing - const detection = photo as IFaceDetection; - - // Zoom into the picture so that the face fills the --photos-face-width box nicely - // if the face is larger than the image, we don't zoom out (reason for the Math.max) - const zoom = Math.max(1, (1 / detection.width) * 0.4) - - // Get center coordinate in percent - const horizontalCenterOfFace = (detection.x + detection.width / 2) * 100 - const verticalCenterOfFace = (detection.y + detection.height / 2) * 100 - - // Get preview element dimensions - const elem = this.$refs.previews as HTMLElement; - const elemWidth = elem.clientWidth; - const elemHeight = elem.clientHeight; - - return { - // we translate the image so that the center of the detected face is in the center - // and add the zoom - transform: `translate(calc(${elemWidth}px/2 - ${horizontalCenterOfFace}% ), calc(${elemHeight}px/2 - ${verticalCenterOfFace}% )) scale(${zoom})`, - // this is necessary for the zoom to zoom toward the center of the face - transformOrigin: `${horizontalCenterOfFace}% ${verticalCenterOfFace}%`, - } - } }