From a701ccd2f4d4266d055f6feb0cb50ee9a3db0def Mon Sep 17 00:00:00 2001 From: Varun Patil Date: Wed, 22 Mar 2023 17:42:30 -0700 Subject: [PATCH] refactor: face recognition to use generic Signed-off-by: Varun Patil --- appinfo/routes.php | 10 +- lib/Controller/GenericApiControllerUtils.php | 2 + lib/Controller/GenericClusterController.php | 1 + lib/Controller/PeopleController.php | 209 ------------------ .../PeopleFaceRecognitionController.php | 75 +++++++ lib/Db/TimelineQueryPeopleFaceRecognition.php | 77 ++----- 6 files changed, 99 insertions(+), 275 deletions(-) delete mode 100644 lib/Controller/PeopleController.php create mode 100644 lib/Controller/PeopleFaceRecognitionController.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 07dfa207..8cce9730 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -59,14 +59,14 @@ return [ ['name' => 'Tags#preview', 'url' => '/api/tags/preview/{name}', 'verb' => 'GET'], ['name' => 'Tags#set', 'url' => '/api/tags/set/{id}', 'verb' => 'PATCH'], + ['name' => 'Places#list', 'url' => '/api/places', 'verb' => 'GET'], + ['name' => 'Places#preview', 'url' => '/api/places/preview/{name}', 'verb' => 'GET'], + ['name' => 'PeopleRecognize#list', 'url' => '/api/recognize/people', 'verb' => 'GET'], ['name' => 'PeopleRecognize#preview', 'url' => '/api/recognize/people/preview/{name}', 'verb' => 'GET'], - ['name' => 'People#facerecognitionPeople', 'url' => '/api/facerecognition/people', 'verb' => 'GET'], - ['name' => 'People#facerecognitionPeoplePreview', 'url' => '/api/facerecognition/people/preview/{id}', 'verb' => 'GET'], - - ['name' => 'Places#list', 'url' => '/api/places', 'verb' => 'GET'], - ['name' => 'Places#preview', 'url' => '/api/places/preview/{name}', 'verb' => 'GET'], + ['name' => 'PeopleFaceRecognition#list', 'url' => '/api/facerecognition/people', 'verb' => 'GET'], + ['name' => 'PeopleFaceRecognition#preview', 'url' => '/api/facerecognition/people/preview/{name}', 'verb' => 'GET'], ['name' => 'Map#clusters', 'url' => '/api/map/clusters', 'verb' => 'GET'], diff --git a/lib/Controller/GenericApiControllerUtils.php b/lib/Controller/GenericApiControllerUtils.php index ed9cbdbc..79e76709 100644 --- a/lib/Controller/GenericApiControllerUtils.php +++ b/lib/Controller/GenericApiControllerUtils.php @@ -46,6 +46,8 @@ trait GenericApiControllerUtils /** * Runa function and catch exceptions to return HTTP response. + * + * @param mixed $function */ protected function guardEx($function): \OCP\AppFramework\Http\Response { diff --git a/lib/Controller/GenericClusterController.php b/lib/Controller/GenericClusterController.php index 85841a04..57ee10ec 100644 --- a/lib/Controller/GenericClusterController.php +++ b/lib/Controller/GenericClusterController.php @@ -47,6 +47,7 @@ abstract class GenericClusterController extends GenericApiController $this->init(); $list = $this->getClusters(); + return new JSONResponse($list, Http::STATUS_OK); }); } diff --git a/lib/Controller/PeopleController.php b/lib/Controller/PeopleController.php deleted file mode 100644 index 5edc1fdc..00000000 --- a/lib/Controller/PeopleController.php +++ /dev/null @@ -1,209 +0,0 @@ - - * @author Varun Patil - * @license AGPL-3.0-or-later - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -namespace OCA\Memories\Controller; - -use OCA\Memories\Errors; -use OCP\AppFramework\Http; -use OCP\AppFramework\Http\DataDisplayResponse; -use OCP\AppFramework\Http\DataResponse; -use OCP\AppFramework\Http\JSONResponse; -use OCP\Files\FileInfo; - -class PeopleController extends GenericApiController -{ - /** - * @NoAdminRequired - * - * Get list of faces with counts of images - */ - public function facerecognitionPeople(): Http\Response - { - $user = $this->userSession->getUser(); - if (null === $user) { - return Errors::NotLoggedIn(); - } - - // Check if face recognition is installed and enabled for this user - if (!$this->facerecognitionIsInstalled()) { - return Errors::NotEnabled('Face Recognition'); - } - - // If this isn't the timeline folder then things aren't going to work - $root = $this->getRequestRoot(); - if ($root->isEmpty()) { - return Errors::NoRequestRoot(); - } - - // If the user has recognition disabled, just returns an empty response. - if (!$this->facerecognitionIsEnabled()) { - return new JSONResponse([]); - } - - // Run actual query - $currentModel = (int) $this->config->getAppValue('facerecognition', 'model', -1); - $list = $this->timelineQuery->getFaceRecognitionPersons( - $root, - $currentModel - ); - // Just append unnamed clusters to the end. - $list = array_merge($list, $this->timelineQuery->getFaceRecognitionClusters( - $root, - $currentModel - )); - - return new JSONResponse($list, Http::STATUS_OK); - } - - /** - * @NoAdminRequired - * - * @NoCSRFRequired - * - * Get face preview image cropped with imagick - * - * @return DataResponse - */ - public function facerecognitionPeoplePreview(string $id): Http\Response - { - $user = $this->userSession->getUser(); - if (null === $user) { - return Errors::NotLoggedIn(); - } - - // Check if face recognition is installed and enabled for this user - if (!$this->facerecognitionIsInstalled()) { - return Errors::NotEnabled('Face Recognition'); - } - - // Get folder to search for - $root = $this->getRequestRoot(); - if ($root->isEmpty()) { - return Errors::NoRequestRoot(); - } - - // If the user has facerecognition disabled, just returns an empty response. - if (!$this->facerecognitionIsEnabled()) { - return new JSONResponse([]); - } - - // Run actual query - $currentModel = (int) $this->config->getAppValue('facerecognition', 'model', -1); - $detections = $this->timelineQuery->getFaceRecognitionPreview($root, $currentModel, $id); - - if (null === $detections || 0 === \count($detections)) { - return Errors::NotFound('detections'); - } - - return $this->getPreviewResponse($detections, 1.8); - } - - /** - * Get face preview image cropped with imagick. - * - * @param array $detections Array of detections to search - * @param \OCP\IUser $user User to search for - * @param int $padding Padding to add to the face in preview - */ - private function getPreviewResponse( - array $detections, - float $padding - ): Http\Response { - // Get preview manager - $previewManager = \OC::$server->get(\OCP\IPreview::class); - - /** @var \Imagick */ - $image = null; - - // Find the first detection that has a preview - $userFolder = $this->rootFolder->getUserFolder($this->getUID()); - - foreach ($detections as &$detection) { - // Get the file (also checks permissions) - $files = $userFolder->getById($detection['file_id']); - if (0 === \count($files) || FileInfo::TYPE_FILE !== $files[0]->getType()) { - continue; - } - - // Check read permission - if (!$files[0]->isReadable()) { - continue; - } - - // Get (hopefully cached) preview image - try { - $preview = $previewManager->getPreview($files[0], 2048, 2048, false); - - $image = new \Imagick(); - if (!$image->readImageBlob($preview->getContent())) { - throw new \Exception('Failed to read image blob'); - } - $iw = $image->getImageWidth(); - $ih = $image->getImageHeight(); - - if ($iw <= 0 || $ih <= 0) { - $image = null; - - throw new \Exception('Invalid image size'); - } - } catch (\Exception $e) { - continue; - } - - // Got the preview - break; - } - - // Make sure the preview is valid - if (null === $image) { - return Errors::NotFound('preview'); - } - - // Set quality and make progressive - $image->setImageCompressionQuality(80); - $image->setInterlaceScheme(\Imagick::INTERLACE_PLANE); - - // Crop image - $dw = (float) $detection['width']; - $dh = (float) $detection['height']; - $dcx = (float) $detection['x'] + (float) $detection['width'] / 2; - $dcy = (float) $detection['y'] + (float) $detection['height'] / 2; - $faceDim = max($dw * $iw, $dh * $ih) * $padding; - $image->cropImage( - (int) $faceDim, - (int) $faceDim, - (int) ($dcx * $iw - $faceDim / 2), - (int) ($dcy * $ih - $faceDim / 2), - ); - $image->scaleImage(512, 512, true); - $blob = $image->getImageBlob(); - - // Create and send response - $response = new DataDisplayResponse($blob, Http::STATUS_OK, [ - 'Content-Type' => $image->getImageMimeType(), - ]); - $response->cacheFor(3600 * 24, false, false); - - return $response; - } -} diff --git a/lib/Controller/PeopleFaceRecognitionController.php b/lib/Controller/PeopleFaceRecognitionController.php new file mode 100644 index 00000000..63d1a2fa --- /dev/null +++ b/lib/Controller/PeopleFaceRecognitionController.php @@ -0,0 +1,75 @@ + + * @author Varun Patil + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCA\Memories\Controller; + +class PeopleFaceRecognitionController extends GenericClusterController +{ + use PeopleControllerUtils; + + protected function appName(): string + { + return 'Face Recognition'; + } + + protected function isEnabled(): bool + { + return $this->recognizeIsEnabled(); + } + + protected function getClusters(): array + { + return array_merge( + $this->timelineQuery->getFaceRecognitionPersons($this->root, $this->model()), + $this->timelineQuery->getFaceRecognitionClusters($this->root, $this->model()) + ); + } + + protected function getPhotos(string $name, ?int $limit = null): array + { + return $this->timelineQuery->getFaceRecognitionPhotos($name, $this->model(), $this->root, $limit) ?? []; + } + + protected function sortPhotosForPreview(array &$photos) + { + // Convert to recognize format (percentage position-size) + foreach ($photos as &$p) { + $p['x'] = (float) $p['x'] / $p['image_width']; + $p['y'] = (float) $p['y'] / $p['image_height']; + $p['width'] = (float) $p['width'] / $p['image_width']; + $p['height'] = (float) $p['height'] / $p['image_height']; + } + + $this->sortByScores($photos); + } + + protected function getPreviewBlob($file, $photo): array + { + return $this->cropFace($file, $photo, 1.8); + } + + private function model(): int + { + return (int) $this->config->getAppValue('facerecognition', 'model', -1); + } +} diff --git a/lib/Db/TimelineQueryPeopleFaceRecognition.php b/lib/Db/TimelineQueryPeopleFaceRecognition.php index 9d63b924..1a891bd1 100644 --- a/lib/Db/TimelineQueryPeopleFaceRecognition.php +++ b/lib/Db/TimelineQueryPeopleFaceRecognition.php @@ -5,7 +5,6 @@ declare(strict_types=1); namespace OCA\Memories\Db; use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\Files\Folder; use OCP\IDBConnection; trait TimelineQueryPeopleFaceRecognition @@ -59,7 +58,7 @@ trait TimelineQueryPeopleFaceRecognition ); } - public function getFaceRecognitionPreview(TimelineRoot &$root, $currentModel, $previewId) + public function getFaceRecognitionPhotos(string $id, int $currentModel, TimelineRoot &$root, ?int $limit) { $query = $this->connection->getQueryBuilder(); @@ -87,70 +86,28 @@ trait TimelineQueryPeopleFaceRecognition $query->innerJoin('fri', 'memories', 'm', $query->expr()->eq('m.fileid', 'fri.file')); $query->innerJoin('frf', 'facerecog_persons', 'frp', $query->expr()->eq('frp.id', 'frf.person')); - if (is_numeric($previewId)) { + if (is_numeric($id)) { // WHERE faces are from id persons (a cluster). - $query->where($query->expr()->eq('frp.id', $query->createNamedParameter($previewId))); + $query->where($query->expr()->eq('frp.id', $query->createNamedParameter($id))); } else { // WHERE faces are from name on persons. - $query->where($query->expr()->eq('frp.name', $query->createNamedParameter($previewId))); + $query->where($query->expr()->eq('frp.name', $query->createNamedParameter($id))); } // WHERE these photos are in the user's requested folder recursively $query = $this->joinFilecache($query, $root, true, false); // LIMIT results - $query->setMaxResults(15); + if (null !== $limit) { + $query->setMaxResults($limit); + } // Sort by date taken so we get recent photos $query->orderBy('m.datetaken', 'DESC'); $query->addOrderBy('m.fileid', 'DESC'); // tie-breaker // FETCH face detections - $cursor = $this->executeQueryWithCTEs($query); - $previews = $cursor->fetchAll(); - if (empty($previews)) { - return null; - } - - // Score the face detections - foreach ($previews as &$p) { - // Get actual pixel size of face - $iw = min((int) ($p['image_width'] ?: 512), 2048); - $ih = min((int) ($p['image_height'] ?: 512), 2048); - - // Get percentage position and size - $p['x'] = (float) $p['x'] / $p['image_width']; - $p['y'] = (float) $p['y'] / $p['image_height']; - $p['width'] = (float) $p['width'] / $p['image_width']; - $p['height'] = (float) $p['height'] / $p['image_height']; - - $w = (float) $p['width']; - $h = (float) $p['height']; - - // Get center of face - $x = (float) $p['x'] + (float) $p['width'] / 2; - $y = (float) $p['y'] + (float) $p['height'] / 2; - - // 3D normal distribution - if the face is closer to the center, it's better - $positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4); - - // Root size distribution - if the image is bigger, it's better, - // but it doesn't matter beyond a certain point - $imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2); - - // Faces occupying too much of the image don't look particularly good - $faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h); - - // Combine scores - $p['score'] = $positionScore * $imgSizeScore * $faceSizeScore * $p['confidence']; - } - - // Sort previews by score descending - usort($previews, function ($a, $b) { - return $b['score'] <=> $a['score']; - }); - - return $previews; + return $this->executeQueryWithCTEs($query)->fetchAll(); } public function getFaceRecognitionClusters(TimelineRoot &$root, int $currentModel, bool $show_singles = false, bool $show_hidden = false) @@ -261,7 +218,7 @@ trait TimelineQueryPeopleFaceRecognition } /** Convert face fields to object */ - private function processFaceRecognitionDetection(&$row, $days = false) + private function processFaceRecognitionDetection(&$row) { if (!isset($row)) { return; @@ -272,15 +229,13 @@ trait TimelineQueryPeopleFaceRecognition return; } - if (!$days) { - $row['facerect'] = [ - // Get percentage position and size - 'w' => (float) $row['face_width'] / $row['image_width'], - 'h' => (float) $row['face_height'] / $row['image_height'], - 'x' => (float) $row['face_x'] / $row['image_width'], - 'y' => (float) $row['face_y'] / $row['image_height'], - ]; - } + // Get percentage position and size + $row['facerect'] = [ + 'w' => (float) $row['face_width'] / $row['image_width'], + 'h' => (float) $row['face_height'] / $row['image_height'], + 'x' => (float) $row['face_x'] / $row['image_width'], + 'y' => (float) $row['face_y'] / $row['image_height'], + ]; unset($row['face_x'], $row['face_y'], $row['face_w'], $row['face_h'], $row['image_height'], $row['image_width']); }