faces: crop with imagick (#72)
Varun Patil
parent
c3719a3049
commit
5d813b27cc
|
|
@ -26,6 +26,7 @@ return [
|
|||
['name' => 'api#day', 'url' => '/api/days/{id}', 'verb' => 'GET'],
|
||||
['name' => 'api#tags', 'url' => '/api/tags', 'verb' => 'GET'],
|
||||
['name' => 'api#faces', 'url' => '/api/faces', 'verb' => 'GET'],
|
||||
['name' => 'api#facePreview', 'url' => '/api/faces/preview/{id}', 'verb' => 'GET'],
|
||||
['name' => 'api#imageInfo', 'url' => '/api/info/{id}', 'verb' => 'GET'],
|
||||
['name' => 'api#imageEdit', 'url' => '/api/edit/{id}', 'verb' => 'PATCH'],
|
||||
['name' => 'api#archive', 'url' => '/api/archive/{id}', 'verb' => 'PATCH'],
|
||||
|
|
|
|||
|
|
@ -35,13 +35,16 @@ use OCP\AppFramework\Http;
|
|||
use OCP\AppFramework\Http\JSONResponse;
|
||||
use OCP\AppFramework\Http\StreamResponse;
|
||||
use OCP\AppFramework\Http\ContentSecurityPolicy;
|
||||
use OCP\AppFramework\Http\DataResponse;
|
||||
use OCP\AppFramework\Http\DataDisplayResponse;
|
||||
use OCP\Files\IRootFolder;
|
||||
use OCP\Files\FileInfo;
|
||||
use OCP\Files\Folder;
|
||||
use OCP\IConfig;
|
||||
use OCP\IDBConnection;
|
||||
use OCP\IRequest;
|
||||
use OCP\IUserSession;
|
||||
use OCP\Files\FileInfo;
|
||||
use OCP\Files\Folder;
|
||||
use OCP\IPreview;
|
||||
|
||||
class ApiController extends Controller {
|
||||
private IConfig $config;
|
||||
|
|
@ -51,6 +54,7 @@ class ApiController extends Controller {
|
|||
private IAppManager $appManager;
|
||||
private TimelineQuery $timelineQuery;
|
||||
private TimelineWrite $timelineWrite;
|
||||
private IPreview $previewManager;
|
||||
|
||||
public function __construct(
|
||||
IRequest $request,
|
||||
|
|
@ -58,7 +62,8 @@ class ApiController extends Controller {
|
|||
IUserSession $userSession,
|
||||
IDBConnection $connection,
|
||||
IRootFolder $rootFolder,
|
||||
IAppManager $appManager) {
|
||||
IAppManager $appManager,
|
||||
IPreview $previewManager) {
|
||||
|
||||
parent::__construct(Application::APPNAME, $request);
|
||||
|
||||
|
|
@ -67,6 +72,7 @@ class ApiController extends Controller {
|
|||
$this->connection = $connection;
|
||||
$this->rootFolder = $rootFolder;
|
||||
$this->appManager = $appManager;
|
||||
$this->previewManager = $previewManager;
|
||||
$this->timelineQuery = new TimelineQuery($this->connection);
|
||||
$this->timelineWrite = new TimelineWrite($connection);
|
||||
}
|
||||
|
|
@ -385,33 +391,91 @@ class ApiController extends Controller {
|
|||
$folder,
|
||||
);
|
||||
|
||||
// Preload all face previews
|
||||
$previews = $this->timelineQuery->getFacePreviews($folder);
|
||||
|
||||
// Convert to map with key as cluster_id
|
||||
$previews_map = [];
|
||||
foreach ($previews as &$preview) {
|
||||
$key = $preview["cluster_id"];
|
||||
if (!array_key_exists($key, $previews_map)) {
|
||||
$previews_map[$key] = [];
|
||||
}
|
||||
unset($preview["cluster_id"]);
|
||||
$previews_map[$key][] = $preview;
|
||||
}
|
||||
|
||||
// Add all previews to list
|
||||
foreach ($list as &$face) {
|
||||
$key = $face["id"];
|
||||
if (array_key_exists($key, $previews_map)) {
|
||||
$face["previews"] = $previews_map[$key];
|
||||
} else {
|
||||
$face["previews"] = [];
|
||||
}
|
||||
}
|
||||
|
||||
return new JSONResponse($list, Http::STATUS_OK);
|
||||
}
|
||||
|
||||
/**
|
||||
* @NoAdminRequired
|
||||
* @NoCSRFRequired
|
||||
*
|
||||
* Get face preview image cropped with imagick
|
||||
* @return DataResponse
|
||||
*/
|
||||
public function facePreview(string $id): Http\Response {
|
||||
$user = $this->userSession->getUser();
|
||||
if (is_null($user)) {
|
||||
return new DataResponse([], Http::STATUS_PRECONDITION_FAILED);
|
||||
}
|
||||
|
||||
// Check faces enabled for this user
|
||||
if (!$this->recognizeIsEnabled()) {
|
||||
return new DataResponse([], Http::STATUS_PRECONDITION_FAILED);
|
||||
}
|
||||
|
||||
// Get folder to search for
|
||||
$folder = $this->getRequestFolder();
|
||||
if (is_null($folder)) {
|
||||
return new JSONResponse([], Http::STATUS_NOT_FOUND);
|
||||
}
|
||||
|
||||
// Run actual query
|
||||
$detections = $this->timelineQuery->getFacePreviewDetection($folder, intval($id));
|
||||
if (is_null($detections) || count($detections) == 0) {
|
||||
return new DataResponse([], Http::STATUS_NOT_FOUND);
|
||||
}
|
||||
|
||||
// Find the first detection that has a preview
|
||||
$preview = null;
|
||||
foreach ($detections as &$detection) {
|
||||
// Get the file (also checks permissions)
|
||||
$files = $folder->getById($detection["file_id"]);
|
||||
if (count($files) == 0 || $files[0]->getType() != FileInfo::TYPE_FILE) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get (hopefully cached) preview image
|
||||
try {
|
||||
$preview = $this->previewManager->getPreview($files[0], 2048, 2048, false);
|
||||
} catch (\Exception $e) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Got the preview
|
||||
break;
|
||||
}
|
||||
|
||||
// Make sure the preview is valid
|
||||
if (is_null($preview)) {
|
||||
return new DataResponse([], Http::STATUS_NOT_FOUND);
|
||||
}
|
||||
|
||||
// Crop image
|
||||
$image = new \Imagick();
|
||||
$image->readImageBlob($preview->getContent());
|
||||
$iw = $image->getImageWidth();
|
||||
$ih = $image->getImageHeight();
|
||||
$dw = floatval($detection["width"]);
|
||||
$dh = floatval($detection["height"]);
|
||||
$dcx = floatval($detection["x"]) + floatval($detection["width"]) / 2;
|
||||
$dcy = floatval($detection["y"]) + floatval($detection["height"]) / 2;
|
||||
$faceDim = max($dw * $iw, $dh * $ih) * 1.5;
|
||||
$image->cropImage(
|
||||
intval($faceDim),
|
||||
intval($faceDim),
|
||||
intval($dcx * $iw - $faceDim / 2),
|
||||
intval($dcy * $ih - $faceDim / 2),
|
||||
);
|
||||
$image->resizeImage(256, 256, \Imagick::FILTER_LANCZOS, 1);
|
||||
$blob = $image->getImageBlob();
|
||||
|
||||
// Create and send response
|
||||
$response = new DataDisplayResponse($blob, Http::STATUS_OK, [
|
||||
'Content-Type' => $image->getImageMimeType(),
|
||||
]);
|
||||
$response->cacheFor(3600 * 24, false, false);
|
||||
return $response;
|
||||
}
|
||||
|
||||
/**
|
||||
* @NoAdminRequired
|
||||
*
|
||||
|
|
|
|||
|
|
@ -73,21 +73,17 @@ trait TimelineQueryFaces {
|
|||
return $faces;
|
||||
}
|
||||
|
||||
public function getFacePreviews(Folder $folder) {
|
||||
public function getFacePreviewDetection(Folder &$folder, int $id) {
|
||||
$query = $this->connection->getQueryBuilder();
|
||||
|
||||
// Windowing
|
||||
$rowNumber = $query->createFunction('ROW_NUMBER() OVER (PARTITION BY rfd.cluster_id) as n');
|
||||
|
||||
// SELECT face detections for ID
|
||||
$query->select(
|
||||
'rfd.cluster_id',
|
||||
'rfd.file_id',
|
||||
'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height',
|
||||
'f.etag',
|
||||
$rowNumber,
|
||||
'rfd.file_id', // Needed to get the actual file
|
||||
'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height', // Image cropping
|
||||
'm.w as image_width', 'm.h as image_height', // Scoring
|
||||
'm.fileid', 'm.datetaken', // Just in case, for postgres
|
||||
)->from('recognize_face_detections', 'rfd');
|
||||
$query->where($query->expr()->isNotNull('rfd.cluster_id'));
|
||||
$query->where($query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)));
|
||||
|
||||
// WHERE these photos are memories indexed
|
||||
$query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id'));
|
||||
|
|
@ -95,33 +91,47 @@ trait TimelineQueryFaces {
|
|||
// WHERE these photos are in the user's requested folder recursively
|
||||
$query->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, true, false));
|
||||
|
||||
// Make this a sub query
|
||||
$fun = $query->createFunction('(' . $query->getSQL() . ')');
|
||||
// LIMIT results
|
||||
$query->setMaxResults(15);
|
||||
|
||||
// Create outer query
|
||||
$outerQuery = $this->connection->getQueryBuilder();
|
||||
$outerQuery->setParameters($query->getParameters());
|
||||
$outerQuery->select('*')->from($fun, 't');
|
||||
$outerQuery->where($query->expr()->lte('t.n', $outerQuery->createParameter('nc')));
|
||||
$outerQuery->setParameter('nc', 4, IQueryBuilder::PARAM_INT);
|
||||
// Sort by date taken so we get recent photos
|
||||
$query->orderBy('m.datetaken', 'DESC');
|
||||
$query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
|
||||
|
||||
// FETCH all face detections
|
||||
$previews = $outerQuery->executeQuery()->fetchAll();
|
||||
|
||||
// Post-process, everthing is a number
|
||||
foreach($previews as &$row) {
|
||||
$row["cluster_id"] = intval($row["cluster_id"]);
|
||||
$row["fileid"] = intval($row["file_id"]);
|
||||
$row["x"] = floatval($row["x"]);
|
||||
$row["y"] = floatval($row["y"]);
|
||||
$row["width"] = floatval($row["width"]);
|
||||
$row["height"] = floatval($row["height"]);
|
||||
|
||||
// remove stale
|
||||
unset($row["file_id"]);
|
||||
unset($row["n"]);
|
||||
// FETCH face detections
|
||||
$previews = $query->executeQuery()->fetchAll();
|
||||
if (empty($previews)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Score the face detections
|
||||
foreach ($previews as &$p) {
|
||||
// Get actual pixel size of face
|
||||
$iw = min(intval($p["image_width"] ?: 512), 2048);
|
||||
$ih = min(intval($p["image_height"] ?: 512), 2048);
|
||||
$w = floatval($p["width"]) * $iw;
|
||||
$h = floatval($p["height"]) * $ih;
|
||||
|
||||
// Get center of face
|
||||
$x = floatval($p["x"]) + floatval($p["width"]) / 2;
|
||||
$y = floatval($p["y"]) + floatval($p["height"]) / 2;
|
||||
|
||||
// 3D normal distribution - if the face is closer to the center, it's better
|
||||
$positionScore = exp(-pow($x - 0.5, 2) * 4) * exp(-pow($y - 0.5, 2) * 4);
|
||||
|
||||
// Root size distribution - if the face is bigger, it's better,
|
||||
// but it doesn't matter beyond a certain point, especially 256px ;)
|
||||
$sizeScore = pow($w * 100, 1/4) * pow($h * 100, 1/4);
|
||||
|
||||
// Combine scores
|
||||
$p["score"] = $positionScore * $sizeScore;
|
||||
}
|
||||
|
||||
// Sort previews by score descending
|
||||
usort($previews, function($a, $b) {
|
||||
return $b["score"] <=> $a["score"];
|
||||
});
|
||||
|
||||
return $previews;
|
||||
}
|
||||
}
|
||||
|
|
@ -16,7 +16,6 @@
|
|||
:class="{ 'error': info.flag & c.FLAG_LOAD_FAIL }"
|
||||
:key="'fpreview-' + info.fileid"
|
||||
:src="getPreviewUrl(info.fileid, info.etag)"
|
||||
:style="getCoverStyle(info)"
|
||||
@error="info.flag |= c.FLAG_LOAD_FAIL" />
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -34,13 +33,6 @@ import { NcCounterBubble } from '@nextcloud/vue'
|
|||
import GlobalMixin from '../../mixins/GlobalMixin';
|
||||
import { constants } from '../../services/Utils';
|
||||
|
||||
interface IFaceDetection extends IPhoto {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
@Component({
|
||||
components: {
|
||||
NcCounterBubble,
|
||||
|
|
@ -67,7 +59,7 @@ export default class Tag extends Mixins(GlobalMixin) {
|
|||
|
||||
getPreviewUrl(fileid: number, etag: string) {
|
||||
if (this.isFace) {
|
||||
return getPreviewUrl(fileid, etag, false, 2048);
|
||||
return generateUrl('/apps/memories/api/faces/preview/' + this.data.fileid);
|
||||
}
|
||||
return getPreviewUrl(fileid, etag, true, 256);
|
||||
}
|
||||
|
|
@ -80,24 +72,24 @@ export default class Tag extends Mixins(GlobalMixin) {
|
|||
// Reset state
|
||||
this.error = false;
|
||||
|
||||
// Look for previews
|
||||
if (!this.data.previews) {
|
||||
// Add dummy preview if face
|
||||
if (this.isFace) {
|
||||
this.previews = [{ fileid: 0, etag: '', flag: 0 }];
|
||||
return;
|
||||
}
|
||||
|
||||
// Look for previews
|
||||
if (!this.data.previews) return;
|
||||
|
||||
// Reset flag
|
||||
this.data.previews.forEach((p) => p.flag = 0);
|
||||
|
||||
if (this.isFace) {
|
||||
const face = this.chooseFaceDetection(this.data.previews as IFaceDetection[]);
|
||||
this.previews = [face];
|
||||
} else {
|
||||
let data = this.data.previews;
|
||||
if (data.length < 4) {
|
||||
data = data.slice(0, 1);
|
||||
}
|
||||
this.previews = data;
|
||||
// Get 4 or 1 preview(s)
|
||||
let data = this.data.previews;
|
||||
if (data.length < 4) {
|
||||
data = data.slice(0, 1);
|
||||
}
|
||||
this.previews = data;
|
||||
|
||||
this.error = this.previews.length === 0;
|
||||
}
|
||||
|
|
@ -117,56 +109,6 @@ export default class Tag extends Mixins(GlobalMixin) {
|
|||
this.$router.push({ name: 'tags', params: { name: this.data.name }});
|
||||
}
|
||||
}
|
||||
|
||||
/** Choose the most appropriate face detection */
|
||||
private chooseFaceDetection(detections: IFaceDetection[]) {
|
||||
const scoreFacePosition = (faceDetection: IFaceDetection) => {
|
||||
return Math.max(0, -1 * (faceDetection.x - faceDetection.width * 0.5))
|
||||
+ Math.max(0, -1 * (faceDetection.y - faceDetection.height * 0.5))
|
||||
+ Math.max(0, -1 * (1 - (faceDetection.x + faceDetection.width) - faceDetection.width * 0.5))
|
||||
+ Math.max(0, -1 * (1 - (faceDetection.y + faceDetection.height) - faceDetection.height * 0.5))
|
||||
}
|
||||
|
||||
const scoreFace = (faceDetection: IFaceDetection) => {
|
||||
return (1 - faceDetection.width * faceDetection.height) + scoreFacePosition(faceDetection);
|
||||
}
|
||||
|
||||
return detections.sort((a, b) => scoreFace(a) - scoreFace(b))[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* This will produce an inline style to apply to images
|
||||
* to zoom toward the detected face
|
||||
*/
|
||||
getCoverStyle(photo: IPhoto) {
|
||||
if (!this.isFace) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Pass the same thing
|
||||
const detection = photo as IFaceDetection;
|
||||
|
||||
// Zoom into the picture so that the face fills the --photos-face-width box nicely
|
||||
// if the face is larger than the image, we don't zoom out (reason for the Math.max)
|
||||
const zoom = Math.max(1, (1 / detection.width) * 0.4)
|
||||
|
||||
// Get center coordinate in percent
|
||||
const horizontalCenterOfFace = (detection.x + detection.width / 2) * 100
|
||||
const verticalCenterOfFace = (detection.y + detection.height / 2) * 100
|
||||
|
||||
// Get preview element dimensions
|
||||
const elem = this.$refs.previews as HTMLElement;
|
||||
const elemWidth = elem.clientWidth;
|
||||
const elemHeight = elem.clientHeight;
|
||||
|
||||
return {
|
||||
// we translate the image so that the center of the detected face is in the center
|
||||
// and add the zoom
|
||||
transform: `translate(calc(${elemWidth}px/2 - ${horizontalCenterOfFace}% ), calc(${elemHeight}px/2 - ${verticalCenterOfFace}% )) scale(${zoom})`,
|
||||
// this is necessary for the zoom to zoom toward the center of the face
|
||||
transformOrigin: `${horizontalCenterOfFace}% ${verticalCenterOfFace}%`,
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue