RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: use generic for recognize 

Signed-off-by: Varun Patil <varunpatil@ucla.edu>
pull/563/head
Varun Patil 2023-03-22 16:44:51 -07:00
parent af7000a037
commit 094a4b32d7
8 changed files with 276 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
appinfo/routes.php
View File

@ -59,8 +59,9 @@ return [
['name' => 'Tags#preview', 'url' => '/api/tags/preview/{name}', 'verb' => 'GET'],
['name' => 'Tags#set', 'url' => '/api/tags/set/{id}', 'verb' => 'PATCH'],
['name' => 'People#recognizePeople', 'url' => '/api/recognize/people', 'verb' => 'GET'],
['name' => 'People#recognizePeoplePreview', 'url' => '/api/recognize/people/preview/{id}', 'verb' => 'GET'],
['name' => 'PeopleRecognize#list', 'url' => '/api/recognize/people', 'verb' => 'GET'],
['name' => 'PeopleRecognize#preview', 'url' => '/api/recognize/people/preview/{name}', 'verb' => 'GET'],
['name' => 'People#facerecognitionPeople', 'url' => '/api/facerecognition/people', 'verb' => 'GET'],
['name' => 'People#facerecognitionPeoplePreview', 'url' => '/api/facerecognition/people/preview/{id}', 'verb' => 'GET'],

1
lib/Controller/AlbumsController.php
View File

@ -85,6 +85,7 @@ class AlbumsController extends GenericClusterController
// Get files
$id = (int) $album['album_id'];
return $this->timelineQuery->getAlbumFiles($id, $limit) ?? [];
}
}

42
lib/Controller/GenericApiControllerUtils.php
View File

@ -23,10 +23,7 @@ declare(strict_types=1);
namespace OCA\Memories\Controller;
use OCA\Memories\Errors;
use OCP\App\IAppManager;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\DataDisplayResponse;
use OCP\IConfig;
trait GenericApiControllerUtils
@ -92,43 +89,4 @@ trait GenericApiControllerUtils
{
return \OCA\Memories\Util::placesGISType() > 0;
}
/**
* Given a list of file ids, return the first preview image possible.
*/
protected function getPreviewFromImageList(array $list, int $quality = 512): Http\Response
{
// Get preview manager
$previewManager = \OC::$server->get(\OCP\IPreview::class);
// Try to get a preview
$userFolder = $this->rootFolder->getUserFolder($this->getUID());
foreach ($list as &$img) {
// Get the file
$files = $userFolder->getById($img);
if (0 === \count($files)) {
continue;
}
// Check read permission
if (!$files[0]->isReadable()) {
continue;
}
// Get preview image
try {
$preview = $previewManager->getPreview($files[0], $quality, $quality, false);
$response = new DataDisplayResponse($preview->getContent(), Http::STATUS_OK, [
'Content-Type' => $preview->getMimeType(),
]);
$response->cacheFor(3600 * 24, false, false);
return $response;
} catch (\Exception $e) {
continue;
}
}
return Errors::NotFound('preview from list');
}
}

126
lib/Controller/GenericClusterController.php
View File

@ -27,6 +27,7 @@ use OCA\Memories\Db\TimelineRoot;
use OCA\Memories\Errors;
use OCA\Memories\HttpResponseException;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\DataDisplayResponse;
use OCP\AppFramework\Http\JSONResponse;
abstract class GenericClusterController extends GenericApiController
@ -80,7 +81,7 @@ abstract class GenericClusterController extends GenericApiController
$this->sortFilesForPreview($files);
// Get preview from image list
return $this->getPreviewFromImageList($this->getFileIds($files));
return $this->getPreviewFromImageList($files);
} catch (HttpResponseException $e) {
return $e->response;
} catch (\Exception $e) {
@ -101,7 +102,8 @@ abstract class GenericClusterController extends GenericApiController
$this->init();
// Get list of all files in this cluster
$fileIds = $this->getFileIds($this->getFiles($name));
$files = $this->getFiles($name);
$fileIds = array_map([$this, 'getFileId'], $files);
// Get download handle
$filename = $this->clusterName($name);
@ -140,6 +142,60 @@ abstract class GenericClusterController extends GenericApiController
*/
abstract protected function getFiles(string $name, ?int $limit = null): array;
/**
* Human readable name for the cluster.
*/
protected function clusterName(string $name)
{
return $name;
}
/**
* Put the file objects in priority list.
* Works on the array in place.
*/
protected function sortFilesForPreview(array &$files)
{
shuffle($files);
}
/**
* Quality to use for the preview file.
*/
protected function getPreviewQuality(): int
{
return 512;
}
/**
* Perform any post processing and get the blob from the preview file.
*
* @param \OCP\Files\SimpleFS\ISimpleFile $file
* @param array $object The file object
*
* @return [Blob, mimetype] of data
*/
protected function getPreviewBlob($file, $object): array
{
return [$file->getContent(), $file->getMimeType()];
}
/**
* Get the file ID for a file object.
*/
protected function getFileId(array $file): int
{
return (int) $file['fileid'];
}
/**
* Should the timeline root be queried?
*/
protected function useTimelineRoot(): bool
{
return true;
}
/**
* Initialize and check if the app is enabled.
* Gets the root node if required.
@ -165,43 +221,45 @@ abstract class GenericClusterController extends GenericApiController
}
/**
* Should the timeline root be queried?
* Given a list of file objects, return the first preview image possible.
*/
protected function useTimelineRoot(): bool
private function getPreviewFromImageList(array $list): Http\Response
{
return true;
}
// Get preview manager
$previewManager = \OC::$server->get(\OCP\IPreview::class);
/**
* Human readable name for the cluster.
*/
protected function clusterName(string $name)
{
return $name;
}
// Try to get a preview
$userFolder = $this->rootFolder->getUserFolder($this->getUID());
foreach ($list as $img) {
// Get the file
$files = $userFolder->getById($this->getFileId($img));
if (0 === \count($files)) {
continue;
}
/**
* Put the file objects in priority list.
* Works on the array in place.
*/
protected function sortFilesForPreview(array &$files)
{
shuffle($files);
}
// Check read permission
if (!$files[0]->isReadable()) {
continue;
}
/**
* Get the file ID for a file object.
*/
protected function getFileId(array $file): int
{
return (int) $file['fileid'];
}
// Get preview image
try {
$quality = $this->getPreviewQuality();
$file = $previewManager->getPreview($files[0], $quality, $quality, false);
/**
* Get array of fileIds from array of file objects.
*/
private function getFileIds(array $files): array
{
return array_map([$this, 'getFileId'], $files);
[$blob, $mimetype] = $this->getPreviewBlob($file, $img);
$response = new DataDisplayResponse($blob, Http::STATUS_OK, [
'Content-Type' => $mimetype,
]);
$response->cacheFor(3600 * 24, false, false);
return $response;
} catch (\Exception $e) {
continue;
}
}
return Errors::NotFound('preview from list');
}
}

71
lib/Controller/PeopleController.php
View File

@ -32,77 +32,6 @@ use OCP\Files\FileInfo;
class PeopleController extends GenericApiController
{
/**
* @NoAdminRequired
*
* Get list of faces with counts of images
*/
public function recognizePeople(): Http\Response
{
try {
$uid = $this->getUID();
} catch (\Exception $e) {
return Errors::NotLoggedIn();
}
// Check faces enabled for this user
if (!$this->recognizeIsEnabled()) {
return Errors::NotEnabled('Recognize');
}
// If this isn't the timeline folder then things aren't going to work
$root = $this->getRequestRoot();
if ($root->isEmpty()) {
return Errors::NoRequestRoot();
}
// Run actual query
$list = $this->timelineQuery->getPeopleRecognize(
$root,
$uid
);
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* @NoAdminRequired
*
* @NoCSRFRequired
*
* Get face preview image cropped with imagick
*
* @return DataResponse
*/
public function recognizePeoplePreview(int $id): Http\Response
{
try {
$uid = $this->getUID();
} catch (\Exception $e) {
return Errors::NotLoggedIn();
}
// Check faces enabled for this user
if (!$this->recognizeIsEnabled()) {
return Errors::NotEnabled('Recognize');
}
// Get folder to search for
$root = $this->getRequestRoot();
if ($root->isEmpty()) {
return Errors::NoRequestRoot();
}
// Run actual query
$detections = $this->timelineQuery->getPeopleRecognizePreview($root, $id, $uid);
if (0 === \count($detections)) {
return Errors::NotFound('detections');
}
return $this->getPreviewResponse($detections, 1.5);
}
/**
* @NoAdminRequired
*

114
lib/Controller/PeopleControllerUtils.php 100644
View File

@ -0,0 +1,114 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Varun Patil <radialapps@gmail.com>
* @author Varun Patil <radialapps@gmail.com>
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Controller;
trait PeopleControllerUtils
{
private function sortByScores(array &$list): void
{
// Score the face detections
foreach ($list as &$p) {
// Make sure we have integers
$p['fileid'] = (int) $p['fileid'];
// Get actual pixel size of face
$iw = min((int) ($p['image_width'] ?: 512), 2048);
$ih = min((int) ($p['image_height'] ?: 512), 2048);
$w = (float) $p['width'];
$h = (float) $p['height'];
// Get center of face
$x = (float) $p['x'] + (float) $p['width'] / 2;
$y = (float) $p['y'] + (float) $p['height'] / 2;
// 3D normal distribution - if the face is closer to the center, it's better
$positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4);
// Root size distribution - if the image is bigger, it's better,
// but it doesn't matter beyond a certain point
$imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2);
// Faces occupying too much of the image don't look particularly good
$faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h);
// Combine scores
$p['score'] = $positionScore * $imgSizeScore * $faceSizeScore;
}
// Sort previews by score descending
usort($list, fn ($a, $b) => $b['score'] <=> $a['score']);
}
/**
* Crop the preview to the face.
*
* @param \OCP\Files\SimpleFS\ISimpleFile $file
* @param array $object The face object
*
* @return [Blob, mimetype] of resulting image
*
* @throws \Exception if file could not be used
*/
private function cropFace($file, array $object, float $padding)
{
/** @var \Imagick */
$image = null;
try {
$image = new \Imagick();
$image->readImageBlob($file->getContent());
} catch (\ImagickException $e) {
throw new \Exception('Could not read image');
}
$iw = $image->getImageWidth();
$ih = $image->getImageHeight();
if ($iw <= 0 || $ih <= 0) {
$image = null;
throw new \Exception('Invalid image size');
}
// Set quality and make progressive
$image->setImageCompressionQuality(80);
$image->setInterlaceScheme(\Imagick::INTERLACE_PLANE);
// Crop image
$dw = (float) $object['width'];
$dh = (float) $object['height'];
$dcx = (float) $object['x'] + (float) $object['width'] / 2;
$dcy = (float) $object['y'] + (float) $object['height'] / 2;
$faceDim = max($dw * $iw, $dh * $ih) * $padding;
$image->cropImage(
(int) $faceDim,
(int) $faceDim,
(int) ($dcx * $iw - $faceDim / 2),
(int) ($dcy * $ih - $faceDim / 2),
);
$image->scaleImage(512, 512, true);
return [$image->getImageBlob(), $image->getImageMimeType()];
}
}

59
lib/Controller/PeopleRecognizeController.php 100644
View File

@ -0,0 +1,59 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Varun Patil <radialapps@gmail.com>
* @author Varun Patil <radialapps@gmail.com>
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace OCA\Memories\Controller;
class PeopleRecognizeController extends GenericClusterController
{
use PeopleControllerUtils;
protected function appName(): string
{
return 'Recognize';
}
protected function isEnabled(): bool
{
return $this->recognizeIsEnabled();
}
protected function getClusters(): array
{
return $this->timelineQuery->getPeopleRecognize($this->root, $this->getUID());
}
protected function getFiles(string $name, ?int $limit = null): array
{
return $this->timelineQuery->getPeopleRecognizeFiles((int) $name, $this->root, $limit) ?? [];
}
protected function sortFilesForPreview(array &$files)
{
$this->sortByScores($files);
}
protected function getPreviewBlob($file, $object): array
{
return $this->cropFace($file, $object, 1.5);
}
}

52
lib/Db/TimelineQueryPeopleRecognize.php
View File

@ -105,7 +105,7 @@ trait TimelineQueryPeopleRecognize
return $faces;
}
public function getPeopleRecognizePreview(TimelineRoot &$root, int $id, string $uid): array
public function getPeopleRecognizeFiles(int $id, TimelineRoot $root, ?int $limit): array
{
$query = $this->connection->getQueryBuilder();
@ -122,11 +122,8 @@ trait TimelineQueryPeopleRecognize
'm.datetaken', // Just in case, for postgres
)->from('recognize_face_detections', 'rfd');
// WHERE detection belongs to this cluster AND user
$query->where($query->expr()->andX(
$query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)),
$query->expr()->eq('rfd.user_id', $query->createNamedParameter($uid)),
));
// WHERE detection belongs to this cluster
$query->where($query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)));
// WHERE these photos are memories indexed
$query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id'));
@ -135,51 +132,16 @@ trait TimelineQueryPeopleRecognize
$query = $this->joinFilecache($query, $root, true, false);
// LIMIT results
$query->setMaxResults(15);
if (null !== $limit) {
$query->setMaxResults($limit);
}
// Sort by date taken so we get recent photos
$query->orderBy('m.datetaken', 'DESC');
$query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
// FETCH face detections
$cursor = $this->executeQueryWithCTEs($query);
$previews = $cursor->fetchAll();
if (empty($previews)) {
return [];
}
// Score the face detections
foreach ($previews as &$p) {
// Get actual pixel size of face
$iw = min((int) ($p['image_width'] ?: 512), 2048);
$ih = min((int) ($p['image_height'] ?: 512), 2048);
$w = (float) $p['width'];
$h = (float) $p['height'];
// Get center of face
$x = (float) $p['x'] + (float) $p['width'] / 2;
$y = (float) $p['y'] + (float) $p['height'] / 2;
// 3D normal distribution - if the face is closer to the center, it's better
$positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4);
// Root size distribution - if the image is bigger, it's better,
// but it doesn't matter beyond a certain point
$imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2);
// Faces occupying too much of the image don't look particularly good
$faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h);
// Combine scores
$p['score'] = $positionScore * $imgSizeScore * $faceSizeScore;
}
// Sort previews by score descending
usort($previews, function ($a, $b) {
return $b['score'] <=> $a['score'];
});
return $previews;
return $this->executeQueryWithCTEs($query)->fetchAll();
}
/** Convert face fields to object */