RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: face recognition to use generic 

Signed-off-by: Varun Patil <varunpatil@ucla.edu>
pull/563/head
Varun Patil 2023-03-22 17:42:30 -07:00
parent ca49eb19e8
commit a701ccd2f4
6 changed files with 99 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
appinfo/routes.php
View File

@ -59,14 +59,14 @@ return [
['name' => 'Tags#preview', 'url' => '/api/tags/preview/{name}', 'verb' => 'GET'],
['name' => 'Tags#set', 'url' => '/api/tags/set/{id}', 'verb' => 'PATCH'],
['name' => 'Places#list', 'url' => '/api/places', 'verb' => 'GET'],
['name' => 'Places#preview', 'url' => '/api/places/preview/{name}', 'verb' => 'GET'],
['name' => 'PeopleRecognize#list', 'url' => '/api/recognize/people', 'verb' => 'GET'],
['name' => 'PeopleRecognize#preview', 'url' => '/api/recognize/people/preview/{name}', 'verb' => 'GET'],
['name' => 'People#facerecognitionPeople', 'url' => '/api/facerecognition/people', 'verb' => 'GET'],
['name' => 'People#facerecognitionPeoplePreview', 'url' => '/api/facerecognition/people/preview/{id}', 'verb' => 'GET'],
['name' => 'Places#list', 'url' => '/api/places', 'verb' => 'GET'],
['name' => 'Places#preview', 'url' => '/api/places/preview/{name}', 'verb' => 'GET'],
['name' => 'PeopleFaceRecognition#list', 'url' => '/api/facerecognition/people', 'verb' => 'GET'],
['name' => 'PeopleFaceRecognition#preview', 'url' => '/api/facerecognition/people/preview/{name}', 'verb' => 'GET'],
['name' => 'Map#clusters', 'url' => '/api/map/clusters', 'verb' => 'GET'],

2
lib/Controller/GenericApiControllerUtils.php
View File

@ -46,6 +46,8 @@ trait GenericApiControllerUtils
/**
* Runa function and catch exceptions to return HTTP response.
*
* @param mixed $function
*/
protected function guardEx($function): \OCP\AppFramework\Http\Response
{

1
lib/Controller/GenericClusterController.php
View File

@ -47,6 +47,7 @@ abstract class GenericClusterController extends GenericApiController
$this->init();
$list = $this->getClusters();
return new JSONResponse($list, Http::STATUS_OK);
});
}

209
lib/Controller/PeopleController.php
View File

@ -1,209 +0,0 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Varun Patil <radialapps@gmail.com>
* @author Varun Patil <radialapps@gmail.com>
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Controller;
use OCA\Memories\Errors;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\DataDisplayResponse;
use OCP\AppFramework\Http\DataResponse;
use OCP\AppFramework\Http\JSONResponse;
use OCP\Files\FileInfo;
class PeopleController extends GenericApiController
{
/**
* @NoAdminRequired
*
* Get list of faces with counts of images
*/
public function facerecognitionPeople(): Http\Response
{
$user = $this->userSession->getUser();
if (null === $user) {
return Errors::NotLoggedIn();
}
// Check if face recognition is installed and enabled for this user
if (!$this->facerecognitionIsInstalled()) {
return Errors::NotEnabled('Face Recognition');
}
// If this isn't the timeline folder then things aren't going to work
$root = $this->getRequestRoot();
if ($root->isEmpty()) {
return Errors::NoRequestRoot();
}
// If the user has recognition disabled, just returns an empty response.
if (!$this->facerecognitionIsEnabled()) {
return new JSONResponse([]);
}
// Run actual query
$currentModel = (int) $this->config->getAppValue('facerecognition', 'model', -1);
$list = $this->timelineQuery->getFaceRecognitionPersons(
$root,
$currentModel
);
// Just append unnamed clusters to the end.
$list = array_merge($list, $this->timelineQuery->getFaceRecognitionClusters(
$root,
$currentModel
));
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* @NoAdminRequired
*
* @NoCSRFRequired
*
* Get face preview image cropped with imagick
*
* @return DataResponse
*/
public function facerecognitionPeoplePreview(string $id): Http\Response
{
$user = $this->userSession->getUser();
if (null === $user) {
return Errors::NotLoggedIn();
}
// Check if face recognition is installed and enabled for this user
if (!$this->facerecognitionIsInstalled()) {
return Errors::NotEnabled('Face Recognition');
}
// Get folder to search for
$root = $this->getRequestRoot();
if ($root->isEmpty()) {
return Errors::NoRequestRoot();
}
// If the user has facerecognition disabled, just returns an empty response.
if (!$this->facerecognitionIsEnabled()) {
return new JSONResponse([]);
}
// Run actual query
$currentModel = (int) $this->config->getAppValue('facerecognition', 'model', -1);
$detections = $this->timelineQuery->getFaceRecognitionPreview($root, $currentModel, $id);
if (null === $detections || 0 === \count($detections)) {
return Errors::NotFound('detections');
}
return $this->getPreviewResponse($detections, 1.8);
}
/**
* Get face preview image cropped with imagick.
*
* @param array $detections Array of detections to search
* @param \OCP\IUser $user User to search for
* @param int $padding Padding to add to the face in preview
*/
private function getPreviewResponse(
array $detections,
float $padding
): Http\Response {
// Get preview manager
$previewManager = \OC::$server->get(\OCP\IPreview::class);
/** @var \Imagick */
$image = null;
// Find the first detection that has a preview
$userFolder = $this->rootFolder->getUserFolder($this->getUID());
foreach ($detections as &$detection) {
// Get the file (also checks permissions)
$files = $userFolder->getById($detection['file_id']);
if (0 === \count($files) || FileInfo::TYPE_FILE !== $files[0]->getType()) {
continue;
}
// Check read permission
if (!$files[0]->isReadable()) {
continue;
}
// Get (hopefully cached) preview image
try {
$preview = $previewManager->getPreview($files[0], 2048, 2048, false);
$image = new \Imagick();
if (!$image->readImageBlob($preview->getContent())) {
throw new \Exception('Failed to read image blob');
}
$iw = $image->getImageWidth();
$ih = $image->getImageHeight();
if ($iw <= 0 || $ih <= 0) {
$image = null;
throw new \Exception('Invalid image size');
}
} catch (\Exception $e) {
continue;
}
// Got the preview
break;
}
// Make sure the preview is valid
if (null === $image) {
return Errors::NotFound('preview');
}
// Set quality and make progressive
$image->setImageCompressionQuality(80);
$image->setInterlaceScheme(\Imagick::INTERLACE_PLANE);
// Crop image
$dw = (float) $detection['width'];
$dh = (float) $detection['height'];
$dcx = (float) $detection['x'] + (float) $detection['width'] / 2;
$dcy = (float) $detection['y'] + (float) $detection['height'] / 2;
$faceDim = max($dw * $iw, $dh * $ih) * $padding;
$image->cropImage(
(int) $faceDim,
(int) $faceDim,
(int) ($dcx * $iw - $faceDim / 2),
(int) ($dcy * $ih - $faceDim / 2),
);
$image->scaleImage(512, 512, true);
$blob = $image->getImageBlob();
// Create and send response
$response = new DataDisplayResponse($blob, Http::STATUS_OK, [
'Content-Type' => $image->getImageMimeType(),
]);
$response->cacheFor(3600 * 24, false, false);
return $response;
}
}

75
lib/Controller/PeopleFaceRecognitionController.php 100644
View File

@ -0,0 +1,75 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Varun Patil <radialapps@gmail.com>
* @author Varun Patil <radialapps@gmail.com>
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Controller;
class PeopleFaceRecognitionController extends GenericClusterController
{
use PeopleControllerUtils;
protected function appName(): string
{
return 'Face Recognition';
}
protected function isEnabled(): bool
{
return $this->recognizeIsEnabled();
}
protected function getClusters(): array
{
return array_merge(
$this->timelineQuery->getFaceRecognitionPersons($this->root, $this->model()),
$this->timelineQuery->getFaceRecognitionClusters($this->root, $this->model())
);
}
protected function getPhotos(string $name, ?int $limit = null): array
{
return $this->timelineQuery->getFaceRecognitionPhotos($name, $this->model(), $this->root, $limit) ?? [];
}
protected function sortPhotosForPreview(array &$photos)
{
// Convert to recognize format (percentage position-size)
foreach ($photos as &$p) {
$p['x'] = (float) $p['x'] / $p['image_width'];
$p['y'] = (float) $p['y'] / $p['image_height'];
$p['width'] = (float) $p['width'] / $p['image_width'];
$p['height'] = (float) $p['height'] / $p['image_height'];
}
$this->sortByScores($photos);
}
protected function getPreviewBlob($file, $photo): array
{
return $this->cropFace($file, $photo, 1.8);
}
private function model(): int
{
return (int) $this->config->getAppValue('facerecognition', 'model', -1);
}
}

65
lib/Db/TimelineQueryPeopleFaceRecognition.php
View File

@ -5,7 +5,6 @@ declare(strict_types=1);
namespace OCA\Memories\Db;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Folder;
use OCP\IDBConnection;
trait TimelineQueryPeopleFaceRecognition
@ -59,7 +58,7 @@ trait TimelineQueryPeopleFaceRecognition
);
}
public function getFaceRecognitionPreview(TimelineRoot &$root, $currentModel, $previewId)
public function getFaceRecognitionPhotos(string $id, int $currentModel, TimelineRoot &$root, ?int $limit)
{
$query = $this->connection->getQueryBuilder();
@ -87,70 +86,28 @@ trait TimelineQueryPeopleFaceRecognition
$query->innerJoin('fri', 'memories', 'm', $query->expr()->eq('m.fileid', 'fri.file'));
$query->innerJoin('frf', 'facerecog_persons', 'frp', $query->expr()->eq('frp.id', 'frf.person'));
if (is_numeric($previewId)) {
if (is_numeric($id)) {
// WHERE faces are from id persons (a cluster).
$query->where($query->expr()->eq('frp.id', $query->createNamedParameter($previewId)));
$query->where($query->expr()->eq('frp.id', $query->createNamedParameter($id)));
} else {
// WHERE faces are from name on persons.
$query->where($query->expr()->eq('frp.name', $query->createNamedParameter($previewId)));
$query->where($query->expr()->eq('frp.name', $query->createNamedParameter($id)));
}
// WHERE these photos are in the user's requested folder recursively
$query = $this->joinFilecache($query, $root, true, false);
// LIMIT results
$query->setMaxResults(15);
if (null !== $limit) {
$query->setMaxResults($limit);
}
// Sort by date taken so we get recent photos
$query->orderBy('m.datetaken', 'DESC');
$query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
// FETCH face detections
$cursor = $this->executeQueryWithCTEs($query);
$previews = $cursor->fetchAll();
if (empty($previews)) {
return null;
}
// Score the face detections
foreach ($previews as &$p) {
// Get actual pixel size of face
$iw = min((int) ($p['image_width'] ?: 512), 2048);
$ih = min((int) ($p['image_height'] ?: 512), 2048);
// Get percentage position and size
$p['x'] = (float) $p['x'] / $p['image_width'];
$p['y'] = (float) $p['y'] / $p['image_height'];
$p['width'] = (float) $p['width'] / $p['image_width'];
$p['height'] = (float) $p['height'] / $p['image_height'];
$w = (float) $p['width'];
$h = (float) $p['height'];
// Get center of face
$x = (float) $p['x'] + (float) $p['width'] / 2;
$y = (float) $p['y'] + (float) $p['height'] / 2;
// 3D normal distribution - if the face is closer to the center, it's better
$positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4);
// Root size distribution - if the image is bigger, it's better,
// but it doesn't matter beyond a certain point
$imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2);
// Faces occupying too much of the image don't look particularly good
$faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h);
// Combine scores
$p['score'] = $positionScore * $imgSizeScore * $faceSizeScore * $p['confidence'];
}
// Sort previews by score descending
usort($previews, function ($a, $b) {
return $b['score'] <=> $a['score'];
});
return $previews;
return $this->executeQueryWithCTEs($query)->fetchAll();
}
public function getFaceRecognitionClusters(TimelineRoot &$root, int $currentModel, bool $show_singles = false, bool $show_hidden = false)
@ -261,7 +218,7 @@ trait TimelineQueryPeopleFaceRecognition
}
/** Convert face fields to object */
private function processFaceRecognitionDetection(&$row, $days = false)
private function processFaceRecognitionDetection(&$row)
{
if (!isset($row)) {
return;
@ -272,15 +229,13 @@ trait TimelineQueryPeopleFaceRecognition
return;
}
if (!$days) {
$row['facerect'] = [
// Get percentage position and size
$row['facerect'] = [
'w' => (float) $row['face_width'] / $row['image_width'],
'h' => (float) $row['face_height'] / $row['image_height'],
'x' => (float) $row['face_x'] / $row['image_width'],
'y' => (float) $row['face_y'] / $row['image_height'],
];
}
unset($row['face_x'], $row['face_y'], $row['face_w'], $row['face_h'], $row['image_height'], $row['image_width']);
}