RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

faces: crop with imagick (#72)

old-stable24
Varun Patil 2022-10-17 10:41:58 -07:00
parent c3719a3049
commit 5d813b27cc
4 changed files with 147 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
appinfo/routes.php
View File

@ -26,6 +26,7 @@ return [
['name' => 'api#day', 'url' => '/api/days/{id}', 'verb' => 'GET'],
['name' => 'api#tags', 'url' => '/api/tags', 'verb' => 'GET'],
['name' => 'api#faces', 'url' => '/api/faces', 'verb' => 'GET'],
['name' => 'api#facePreview', 'url' => '/api/faces/preview/{id}', 'verb' => 'GET'],
['name' => 'api#imageInfo', 'url' => '/api/info/{id}', 'verb' => 'GET'],
['name' => 'api#imageEdit', 'url' => '/api/edit/{id}', 'verb' => 'PATCH'],
['name' => 'api#archive', 'url' => '/api/archive/{id}', 'verb' => 'PATCH'],

118
lib/Controller/ApiController.php
View File

@ -35,13 +35,16 @@ use OCP\AppFramework\Http;
use OCP\AppFramework\Http\JSONResponse;
use OCP\AppFramework\Http\StreamResponse;
use OCP\AppFramework\Http\ContentSecurityPolicy;
use OCP\AppFramework\Http\DataResponse;
use OCP\AppFramework\Http\DataDisplayResponse;
use OCP\Files\IRootFolder;
use OCP\Files\FileInfo;
use OCP\Files\Folder;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IRequest;
use OCP\IUserSession;
use OCP\Files\FileInfo;
use OCP\Files\Folder;
use OCP\IPreview;
class ApiController extends Controller {
private IConfig $config;
@ -51,6 +54,7 @@ class ApiController extends Controller {
private IAppManager $appManager;
private TimelineQuery $timelineQuery;
private TimelineWrite $timelineWrite;
private IPreview $previewManager;
public function __construct(
IRequest $request,
@ -58,7 +62,8 @@ class ApiController extends Controller {
IUserSession $userSession,
IDBConnection $connection,
IRootFolder $rootFolder,
IAppManager $appManager) {
IAppManager $appManager,
IPreview $previewManager) {
parent::__construct(Application::APPNAME, $request);
@ -67,6 +72,7 @@ class ApiController extends Controller {
$this->connection = $connection;
$this->rootFolder = $rootFolder;
$this->appManager = $appManager;
$this->previewManager = $previewManager;
$this->timelineQuery = new TimelineQuery($this->connection);
$this->timelineWrite = new TimelineWrite($connection);
}
@ -385,33 +391,91 @@ class ApiController extends Controller {
$folder,
);
// Preload all face previews
$previews = $this->timelineQuery->getFacePreviews($folder);
// Convert to map with key as cluster_id
$previews_map = [];
foreach ($previews as &$preview) {
$key = $preview["cluster_id"];
if (!array_key_exists($key, $previews_map)) {
$previews_map[$key] = [];
}
unset($preview["cluster_id"]);
$previews_map[$key][] = $preview;
}
// Add all previews to list
foreach ($list as &$face) {
$key = $face["id"];
if (array_key_exists($key, $previews_map)) {
$face["previews"] = $previews_map[$key];
} else {
$face["previews"] = [];
}
}
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* @NoAdminRequired
* @NoCSRFRequired
*
* Get face preview image cropped with imagick
* @return DataResponse
*/
public function facePreview(string $id): Http\Response {
$user = $this->userSession->getUser();
if (is_null($user)) {
return new DataResponse([], Http::STATUS_PRECONDITION_FAILED);
}
// Check faces enabled for this user
if (!$this->recognizeIsEnabled()) {
return new DataResponse([], Http::STATUS_PRECONDITION_FAILED);
}
// Get folder to search for
$folder = $this->getRequestFolder();
if (is_null($folder)) {
return new JSONResponse([], Http::STATUS_NOT_FOUND);
}
// Run actual query
$detections = $this->timelineQuery->getFacePreviewDetection($folder, intval($id));
if (is_null($detections) || count($detections) == 0) {
return new DataResponse([], Http::STATUS_NOT_FOUND);
}
// Find the first detection that has a preview
$preview = null;
foreach ($detections as &$detection) {
// Get the file (also checks permissions)
$files = $folder->getById($detection["file_id"]);
if (count($files) == 0 || $files[0]->getType() != FileInfo::TYPE_FILE) {
continue;
}
// Get (hopefully cached) preview image
try {
$preview = $this->previewManager->getPreview($files[0], 2048, 2048, false);
} catch (\Exception $e) {
continue;
}
// Got the preview
break;
}
// Make sure the preview is valid
if (is_null($preview)) {
return new DataResponse([], Http::STATUS_NOT_FOUND);
}
// Crop image
$image = new \Imagick();
$image->readImageBlob($preview->getContent());
$iw = $image->getImageWidth();
$ih = $image->getImageHeight();
$dw = floatval($detection["width"]);
$dh = floatval($detection["height"]);
$dcx = floatval($detection["x"]) + floatval($detection["width"]) / 2;
$dcy = floatval($detection["y"]) + floatval($detection["height"]) / 2;
$faceDim = max($dw * $iw, $dh * $ih) * 1.5;
$image->cropImage(
intval($faceDim),
intval($faceDim),
intval($dcx * $iw - $faceDim / 2),
intval($dcy * $ih - $faceDim / 2),
);
$image->resizeImage(256, 256, \Imagick::FILTER_LANCZOS, 1);
$blob = $image->getImageBlob();
// Create and send response
$response = new DataDisplayResponse($blob, Http::STATUS_OK, [
'Content-Type' => $image->getImageMimeType(),
]);
$response->cacheFor(3600 * 24, false, false);
return $response;
}
/**
* @NoAdminRequired
*

76
lib/Db/TimelineQueryFaces.php
View File

@ -73,21 +73,17 @@ trait TimelineQueryFaces {
return $faces;
}
public function getFacePreviews(Folder $folder) {
public function getFacePreviewDetection(Folder &$folder, int $id) {
$query = $this->connection->getQueryBuilder();
// Windowing
$rowNumber = $query->createFunction('ROW_NUMBER() OVER (PARTITION BY rfd.cluster_id) as n');
// SELECT face detections for ID
$query->select(
'rfd.cluster_id',
'rfd.file_id',
'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height',
'f.etag',
$rowNumber,
'rfd.file_id', // Needed to get the actual file
'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height', // Image cropping
'm.w as image_width', 'm.h as image_height', // Scoring
'm.fileid', 'm.datetaken', // Just in case, for postgres
)->from('recognize_face_detections', 'rfd');
$query->where($query->expr()->isNotNull('rfd.cluster_id'));
$query->where($query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)));
// WHERE these photos are memories indexed
$query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id'));
@ -95,33 +91,47 @@ trait TimelineQueryFaces {
// WHERE these photos are in the user's requested folder recursively
$query->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, true, false));
// Make this a sub query
$fun = $query->createFunction('(' . $query->getSQL() . ')');
// LIMIT results
$query->setMaxResults(15);
// Create outer query
$outerQuery = $this->connection->getQueryBuilder();
$outerQuery->setParameters($query->getParameters());
$outerQuery->select('*')->from($fun, 't');
$outerQuery->where($query->expr()->lte('t.n', $outerQuery->createParameter('nc')));
$outerQuery->setParameter('nc', 4, IQueryBuilder::PARAM_INT);
// Sort by date taken so we get recent photos
$query->orderBy('m.datetaken', 'DESC');
$query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
// FETCH all face detections
$previews = $outerQuery->executeQuery()->fetchAll();
// Post-process, everthing is a number
foreach($previews as &$row) {
$row["cluster_id"] = intval($row["cluster_id"]);
$row["fileid"] = intval($row["file_id"]);
$row["x"] = floatval($row["x"]);
$row["y"] = floatval($row["y"]);
$row["width"] = floatval($row["width"]);
$row["height"] = floatval($row["height"]);
// remove stale
unset($row["file_id"]);
unset($row["n"]);
// FETCH face detections
$previews = $query->executeQuery()->fetchAll();
if (empty($previews)) {
return null;
}
// Score the face detections
foreach ($previews as &$p) {
// Get actual pixel size of face
$iw = min(intval($p["image_width"] ?: 512), 2048);
$ih = min(intval($p["image_height"] ?: 512), 2048);
$w = floatval($p["width"]) * $iw;
$h = floatval($p["height"]) * $ih;
// Get center of face
$x = floatval($p["x"]) + floatval($p["width"]) / 2;
$y = floatval($p["y"]) + floatval($p["height"]) / 2;
// 3D normal distribution - if the face is closer to the center, it's better
$positionScore = exp(-pow($x - 0.5, 2) * 4) * exp(-pow($y - 0.5, 2) * 4);
// Root size distribution - if the face is bigger, it's better,
// but it doesn't matter beyond a certain point, especially 256px ;)
$sizeScore = pow($w * 100, 1/4) * pow($h * 100, 1/4);
// Combine scores
$p["score"] = $positionScore * $sizeScore;
}
// Sort previews by score descending
usort($previews, function($a, $b) {
return $b["score"] <=> $a["score"];
});
return $previews;
}
}

74
src/components/frame/Tag.vue
View File

@ -16,7 +16,6 @@
:class="{ 'error': info.flag & c.FLAG_LOAD_FAIL }"
:key="'fpreview-' + info.fileid"
:src="getPreviewUrl(info.fileid, info.etag)"
:style="getCoverStyle(info)"
@error="info.flag |= c.FLAG_LOAD_FAIL" />
</div>
</div>
@ -34,13 +33,6 @@ import { NcCounterBubble } from '@nextcloud/vue'
import GlobalMixin from '../../mixins/GlobalMixin';
import { constants } from '../../services/Utils';
interface IFaceDetection extends IPhoto {
x: number;
y: number;
width: number;
height: number;
}
@Component({
components: {
NcCounterBubble,
@ -67,7 +59,7 @@ export default class Tag extends Mixins(GlobalMixin) {
getPreviewUrl(fileid: number, etag: string) {
if (this.isFace) {
return getPreviewUrl(fileid, etag, false, 2048);
return generateUrl('/apps/memories/api/faces/preview/' + this.data.fileid);
}
return getPreviewUrl(fileid, etag, true, 256);
}
@ -80,24 +72,24 @@ export default class Tag extends Mixins(GlobalMixin) {
// Reset state
this.error = false;
// Look for previews
if (!this.data.previews) {
// Add dummy preview if face
if (this.isFace) {
this.previews = [{ fileid: 0, etag: '', flag: 0 }];
return;
}
// Look for previews
if (!this.data.previews) return;
// Reset flag
this.data.previews.forEach((p) => p.flag = 0);
if (this.isFace) {
const face = this.chooseFaceDetection(this.data.previews as IFaceDetection[]);
this.previews = [face];
} else {
// Get 4 or 1 preview(s)
let data = this.data.previews;
if (data.length < 4) {
data = data.slice(0, 1);
}
this.previews = data;
}
this.error = this.previews.length === 0;
}
@ -117,56 +109,6 @@ export default class Tag extends Mixins(GlobalMixin) {
this.$router.push({ name: 'tags', params: { name: this.data.name }});
}
}
/** Choose the most appropriate face detection */
private chooseFaceDetection(detections: IFaceDetection[]) {
const scoreFacePosition = (faceDetection: IFaceDetection) => {
return Math.max(0, -1 * (faceDetection.x - faceDetection.width * 0.5))
+ Math.max(0, -1 * (faceDetection.y - faceDetection.height * 0.5))
+ Math.max(0, -1 * (1 - (faceDetection.x + faceDetection.width) - faceDetection.width * 0.5))
+ Math.max(0, -1 * (1 - (faceDetection.y + faceDetection.height) - faceDetection.height * 0.5))
}
const scoreFace = (faceDetection: IFaceDetection) => {
return (1 - faceDetection.width * faceDetection.height) + scoreFacePosition(faceDetection);
}
return detections.sort((a, b) => scoreFace(a) - scoreFace(b))[0];
}
/**
* This will produce an inline style to apply to images
* to zoom toward the detected face
*/
getCoverStyle(photo: IPhoto) {
if (!this.isFace) {
return {};
}
// Pass the same thing
const detection = photo as IFaceDetection;
// Zoom into the picture so that the face fills the --photos-face-width box nicely
// if the face is larger than the image, we don't zoom out (reason for the Math.max)
const zoom = Math.max(1, (1 / detection.width) * 0.4)
// Get center coordinate in percent
const horizontalCenterOfFace = (detection.x + detection.width / 2) * 100
const verticalCenterOfFace = (detection.y + detection.height / 2) * 100
// Get preview element dimensions
const elem = this.$refs.previews as HTMLElement;
const elemWidth = elem.clientWidth;
const elemHeight = elem.clientHeight;
return {
// we translate the image so that the center of the detected face is in the center
// and add the zoom
transform: `translate(calc(${elemWidth}px/2 - ${horizontalCenterOfFace}% ), calc(${elemHeight}px/2 - ${verticalCenterOfFace}% )) scale(${zoom})`,
// this is necessary for the zoom to zoom toward the center of the face
transformOrigin: `${horizontalCenterOfFace}% ${verticalCenterOfFace}%`,
}
}
}
</script>