297 lines
11 KiB
PHP
297 lines
11 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace OCA\Memories\Db;
|
|
|
|
use OCP\DB\QueryBuilder\IQueryBuilder;
|
|
use OCP\Files\Folder;
|
|
use OCP\IDBConnection;
|
|
|
|
trait TimelineQueryPeopleFaceRecognition
|
|
{
|
|
protected IDBConnection $connection;
|
|
|
|
public function transformPeopleFaceRecognitionFilter(IQueryBuilder &$query, string $userId, int $currentModel, string $personStr)
|
|
{
|
|
// Get title and uid of face user
|
|
$personNames = explode('/', $personStr);
|
|
if (2 !== \count($personNames)) {
|
|
throw new \Exception('Invalid person query');
|
|
}
|
|
|
|
$personUid = $personNames[0];
|
|
$personName = $personNames[1];
|
|
|
|
// Join with images
|
|
$query->innerJoin('m', 'facerecog_images', 'fri', $query->expr()->andX(
|
|
$query->expr()->eq('fri.file', 'm.fileid'),
|
|
$query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)),
|
|
));
|
|
|
|
// Join with faces
|
|
$query->innerJoin(
|
|
'fri',
|
|
'facerecog_faces',
|
|
'frf',
|
|
$query->expr()->eq('frf.image', 'fri.id')
|
|
);
|
|
|
|
// Join with persons
|
|
$nameField = is_numeric($personName) ? 'frp.id' : 'frp.name';
|
|
$query->innerJoin('frf', 'facerecog_persons', 'frp', $query->expr()->andX(
|
|
$query->expr()->eq('frf.person', 'frp.id'),
|
|
$query->expr()->eq('frp.user', $query->createNamedParameter($personUid)),
|
|
$query->expr()->eq($nameField, $query->createNamedParameter($personName)),
|
|
));
|
|
}
|
|
|
|
public function transformPeopleFaceRecognitionRect(IQueryBuilder &$query, string $userId)
|
|
{
|
|
// Include detection params in response
|
|
$query->addSelect(
|
|
'frf.x AS face_x',
|
|
'frf.y AS face_y',
|
|
'frf.width AS face_width',
|
|
'frf.height AS face_height',
|
|
'm.w AS image_width',
|
|
'm.h AS image_height',
|
|
);
|
|
}
|
|
|
|
public function getPeopleFaceRecognition(TimelineRoot &$root, int $currentModel, bool $show_clusters = false, bool $show_singles = false, bool $show_hidden = false)
|
|
{
|
|
if ($show_clusters) {
|
|
return $this->getFaceRecognitionClusters($root, $currentModel, $show_singles, $show_hidden);
|
|
}
|
|
|
|
return $this->getFaceRecognitionPersons($root, $currentModel);
|
|
}
|
|
|
|
public function getFaceRecognitionPreview(TimelineRoot &$root, $currentModel, $previewId)
|
|
{
|
|
$query = $this->connection->getQueryBuilder();
|
|
|
|
// SELECT face detections
|
|
$query->select(
|
|
'fri.file as file_id', // Get actual file
|
|
'frf.x', // Image cropping
|
|
'frf.y',
|
|
'frf.width',
|
|
'frf.height',
|
|
'm.w as image_width', // Scoring
|
|
'm.h as image_height',
|
|
'frf.confidence',
|
|
'm.fileid',
|
|
'm.datetaken', // Just in case, for postgres
|
|
)->from('facerecog_faces', 'frf');
|
|
|
|
// WHERE faces are from images and current model.
|
|
$query->innerJoin('frf', 'facerecog_images', 'fri', $query->expr()->andX(
|
|
$query->expr()->eq('fri.id', 'frf.image'),
|
|
$query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)),
|
|
));
|
|
|
|
// WHERE these photos are memories indexed
|
|
$query->innerJoin('fri', 'memories', 'm', $query->expr()->eq('m.fileid', 'fri.file'));
|
|
|
|
$query->innerJoin('frf', 'facerecog_persons', 'frp', $query->expr()->eq('frp.id', 'frf.person'));
|
|
if (is_numeric($previewId)) {
|
|
// WHERE faces are from id persons (a cluster).
|
|
$query->where($query->expr()->eq('frp.id', $query->createNamedParameter($previewId)));
|
|
} else {
|
|
// WHERE faces are from name on persons.
|
|
$query->where($query->expr()->eq('frp.name', $query->createNamedParameter($previewId)));
|
|
}
|
|
|
|
// WHERE these photos are in the user's requested folder recursively
|
|
$query = $this->joinFilecache($query, $root, true, false);
|
|
|
|
// LIMIT results
|
|
$query->setMaxResults(15);
|
|
|
|
// Sort by date taken so we get recent photos
|
|
$query->orderBy('m.datetaken', 'DESC');
|
|
$query->addOrderBy('m.fileid', 'DESC'); // tie-breaker
|
|
|
|
// FETCH face detections
|
|
$cursor = $this->executeQueryWithCTEs($query);
|
|
$previews = $cursor->fetchAll();
|
|
if (empty($previews)) {
|
|
return null;
|
|
}
|
|
|
|
// Score the face detections
|
|
foreach ($previews as &$p) {
|
|
// Get actual pixel size of face
|
|
$iw = min((int) ($p['image_width'] ?: 512), 2048);
|
|
$ih = min((int) ($p['image_height'] ?: 512), 2048);
|
|
|
|
// Get percentage position and size
|
|
$p['x'] = (float) $p['x'] / $p['image_width'];
|
|
$p['y'] = (float) $p['y'] / $p['image_height'];
|
|
$p['width'] = (float) $p['width'] / $p['image_width'];
|
|
$p['height'] = (float) $p['height'] / $p['image_height'];
|
|
|
|
$w = (float) $p['width'];
|
|
$h = (float) $p['height'];
|
|
|
|
// Get center of face
|
|
$x = (float) $p['x'] + (float) $p['width'] / 2;
|
|
$y = (float) $p['y'] + (float) $p['height'] / 2;
|
|
|
|
// 3D normal distribution - if the face is closer to the center, it's better
|
|
$positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4);
|
|
|
|
// Root size distribution - if the image is bigger, it's better,
|
|
// but it doesn't matter beyond a certain point
|
|
$imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2);
|
|
|
|
// Faces occupying too much of the image don't look particularly good
|
|
$faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h);
|
|
|
|
// Combine scores
|
|
$p['score'] = $positionScore * $imgSizeScore * $faceSizeScore * $p['confidence'];
|
|
}
|
|
|
|
// Sort previews by score descending
|
|
usort($previews, function ($a, $b) {
|
|
return $b['score'] <=> $a['score'];
|
|
});
|
|
|
|
return $previews;
|
|
}
|
|
|
|
private function getFaceRecognitionClusters(TimelineRoot &$root, int $currentModel, bool $show_singles = false, bool $show_hidden = false)
|
|
{
|
|
$query = $this->connection->getQueryBuilder();
|
|
|
|
// SELECT all face clusters
|
|
$count = $query->func()->count($query->createFunction('DISTINCT m.fileid'));
|
|
$query->select('frp.id')->from('facerecog_persons', 'frp');
|
|
$query->selectAlias($count, 'count');
|
|
$query->selectAlias('frp.user', 'user_id');
|
|
|
|
// WHERE there are faces with this cluster
|
|
$query->innerJoin('frp', 'facerecog_faces', 'frf', $query->expr()->eq('frp.id', 'frf.person'));
|
|
|
|
// WHERE faces are from images.
|
|
$query->innerJoin('frf', 'facerecog_images', 'fri', $query->expr()->eq('fri.id', 'frf.image'));
|
|
|
|
// WHERE these items are memories indexed photos
|
|
$query->innerJoin('fri', 'memories', 'm', $query->expr()->andX(
|
|
$query->expr()->eq('fri.file', 'm.fileid'),
|
|
$query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)),
|
|
));
|
|
|
|
// WHERE these photos are in the user's requested folder recursively
|
|
$query = $this->joinFilecache($query, $root, true, false);
|
|
|
|
// GROUP by ID of face cluster
|
|
$query->groupBy('frp.id');
|
|
$query->addGroupBy('frp.user');
|
|
$query->where($query->expr()->isNull('frp.name'));
|
|
|
|
// By default hides individual faces when they have no name.
|
|
if (!$show_singles) {
|
|
$query->having($count, $query->createNamedParameter(1));
|
|
}
|
|
|
|
// By default it shows the people who were not hidden
|
|
if (!$show_hidden) {
|
|
$query->andWhere($query->expr()->eq('frp.is_visible', $query->createNamedParameter(true)));
|
|
}
|
|
|
|
// ORDER by number of faces in cluster and id for response stability.
|
|
$query->orderBy('count', 'DESC');
|
|
$query->addOrderBy('frp.id', 'DESC');
|
|
|
|
// It is not worth displaying all unnamed clusters. We show 15 to name them progressively,
|
|
$query->setMaxResults(15);
|
|
|
|
// FETCH all faces
|
|
$cursor = $this->executeQueryWithCTEs($query);
|
|
$faces = $cursor->fetchAll();
|
|
|
|
// Post process
|
|
foreach ($faces as &$row) {
|
|
$row['id'] = (int) $row['id'];
|
|
$row['count'] = (int) $row['count'];
|
|
}
|
|
|
|
return $faces;
|
|
}
|
|
|
|
private function getFaceRecognitionPersons(TimelineRoot &$root, int $currentModel)
|
|
{
|
|
$query = $this->connection->getQueryBuilder();
|
|
|
|
// SELECT all face clusters
|
|
$count = $query->func()->count($query->createFunction('DISTINCT m.fileid'));
|
|
$query->select('frp.name')->from('facerecog_persons', 'frp');
|
|
$query->selectAlias($count, 'count');
|
|
$query->selectAlias('frp.user', 'user_id');
|
|
|
|
// WHERE there are faces with this cluster
|
|
$query->innerJoin('frp', 'facerecog_faces', 'frf', $query->expr()->eq('frp.id', 'frf.person'));
|
|
|
|
// WHERE faces are from images.
|
|
$query->innerJoin('frf', 'facerecog_images', 'fri', $query->expr()->eq('fri.id', 'frf.image'));
|
|
|
|
// WHERE these items are memories indexed photos
|
|
$query->innerJoin('fri', 'memories', 'm', $query->expr()->andX(
|
|
$query->expr()->eq('fri.file', 'm.fileid'),
|
|
$query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)),
|
|
));
|
|
|
|
// WHERE these photos are in the user's requested folder recursively
|
|
$query = $this->joinFilecache($query, $root, true, false);
|
|
|
|
// GROUP by name of face clusters
|
|
$query->where($query->expr()->isNotNull('frp.name'));
|
|
$query->groupBy('frp.user');
|
|
$query->addGroupBy('frp.name');
|
|
|
|
// ORDER by number of faces in cluster
|
|
$query->orderBy('count', 'DESC');
|
|
$query->addOrderBy('frp.name', 'ASC');
|
|
|
|
// FETCH all faces
|
|
$cursor = $this->executeQueryWithCTEs($query);
|
|
$faces = $cursor->fetchAll();
|
|
|
|
// Post process
|
|
foreach ($faces as &$row) {
|
|
$row['id'] = $row['name'];
|
|
$row['count'] = (int) $row['count'];
|
|
}
|
|
|
|
return $faces;
|
|
}
|
|
|
|
/** Convert face fields to object */
|
|
private function processFaceRecognitionDetection(&$row, $days = false)
|
|
{
|
|
if (!isset($row)) {
|
|
return;
|
|
}
|
|
|
|
// Differentiate Recognize queries from Face Recognition
|
|
if (!isset($row['face_width']) || !isset($row['image_width'])) {
|
|
return;
|
|
}
|
|
|
|
if (!$days) {
|
|
$row['facerect'] = [
|
|
// Get percentage position and size
|
|
'w' => (float) $row['face_width'] / $row['image_width'],
|
|
'h' => (float) $row['face_height'] / $row['image_height'],
|
|
'x' => (float) $row['face_x'] / $row['image_width'],
|
|
'y' => (float) $row['face_y'] / $row['image_height'],
|
|
];
|
|
}
|
|
|
|
unset($row['face_x'], $row['face_y'], $row['face_w'], $row['face_h'], $row['image_height'], $row['image_width']);
|
|
}
|
|
}
|