innerJoin('m', 'facerecog_images', 'fri', $query->expr()->andX( $query->expr()->eq('fri.file', 'm.fileid'), $query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)), )); // Join with faces $query->innerJoin( 'fri', 'facerecog_faces', 'frf', $query->expr()->eq('frf.image', 'fri.id') ); // Join with persons $nameField = is_numeric($personName) ? 'frp.id' : 'frp.name'; $query->innerJoin('frf', 'facerecog_persons', 'frp', $query->expr()->andX( $query->expr()->eq('frf.person', 'frp.id'), $query->expr()->eq('frp.user', $query->createNamedParameter($personUid)), $query->expr()->eq($nameField, $query->createNamedParameter($personName)), )); } public function transformPeopleFaceRecognitionRect(IQueryBuilder &$query, string $userId) { // Include detection params in response $query->addSelect( 'frf.x AS face_x', 'frf.y AS face_y', 'frf.width AS face_width', 'frf.height AS face_height', 'm.w AS image_width', 'm.h AS image_height', ); } public function getPeopleFaceRecognition(TimelineRoot &$root, int $currentModel, bool $show_clusters = false, bool $show_singles = false, bool $show_hidden = false) { if ($show_clusters) { return $this->getFaceRecognitionClusters($root, $currentModel, $show_singles, $show_hidden); } return $this->getFaceRecognitionPersons($root, $currentModel); } public function getFaceRecognitionPreview(TimelineRoot &$root, $currentModel, $previewId) { $query = $this->connection->getQueryBuilder(); // SELECT face detections $query->select( 'fri.file as file_id', // Get actual file 'frf.x', // Image cropping 'frf.y', 'frf.width', 'frf.height', 'm.w as image_width', // Scoring 'm.h as image_height', 'frf.confidence', 'm.fileid', 'm.datetaken', // Just in case, for postgres )->from('facerecog_faces', 'frf'); // WHERE faces are from images and current model. $query->innerJoin('frf', 'facerecog_images', 'fri', $query->expr()->andX( $query->expr()->eq('fri.id', 'frf.image'), $query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)), )); // WHERE these photos are memories indexed $query->innerJoin('fri', 'memories', 'm', $query->expr()->eq('m.fileid', 'fri.file')); $query->innerJoin('frf', 'facerecog_persons', 'frp', $query->expr()->eq('frp.id', 'frf.person')); if (is_numeric($previewId)) { // WHERE faces are from id persons (a cluster). $query->where($query->expr()->eq('frp.id', $query->createNamedParameter($previewId))); } else { // WHERE faces are from name on persons. $query->where($query->expr()->eq('frp.name', $query->createNamedParameter($previewId))); } // WHERE these photos are in the user's requested folder recursively $query = $this->joinFilecache($query, $root, true, false); // LIMIT results $query->setMaxResults(15); // Sort by date taken so we get recent photos $query->orderBy('m.datetaken', 'DESC'); $query->addOrderBy('m.fileid', 'DESC'); // tie-breaker // FETCH face detections $cursor = $this->executeQueryWithCTEs($query); $previews = $cursor->fetchAll(); if (empty($previews)) { return null; } // Score the face detections foreach ($previews as &$p) { // Get actual pixel size of face $iw = min((int) ($p['image_width'] ?: 512), 2048); $ih = min((int) ($p['image_height'] ?: 512), 2048); // Get percentage position and size $p['x'] = (float) $p['x'] / $p['image_width']; $p['y'] = (float) $p['y'] / $p['image_height']; $p['width'] = (float) $p['width'] / $p['image_width']; $p['height'] = (float) $p['height'] / $p['image_height']; $w = (float) $p['width']; $h = (float) $p['height']; // Get center of face $x = (float) $p['x'] + (float) $p['width'] / 2; $y = (float) $p['y'] + (float) $p['height'] / 2; // 3D normal distribution - if the face is closer to the center, it's better $positionScore = exp(-($x - 0.5) ** 2 * 4) * exp(-($y - 0.5) ** 2 * 4); // Root size distribution - if the image is bigger, it's better, // but it doesn't matter beyond a certain point $imgSizeScore = ($iw * 100) ** (1 / 2) * ($ih * 100) ** (1 / 2); // Faces occupying too much of the image don't look particularly good $faceSizeScore = (-$w ** 2 + $w) * (-$h ** 2 + $h); // Combine scores $p['score'] = $positionScore * $imgSizeScore * $faceSizeScore * $p['confidence']; } // Sort previews by score descending usort($previews, function ($a, $b) { return $b['score'] <=> $a['score']; }); return $previews; } private function getFaceRecognitionClusters(TimelineRoot &$root, int $currentModel, bool $show_singles = false, bool $show_hidden = false) { $query = $this->connection->getQueryBuilder(); // SELECT all face clusters $count = $query->func()->count($query->createFunction('DISTINCT m.fileid')); $query->select('frp.id')->from('facerecog_persons', 'frp'); $query->selectAlias($count, 'count'); $query->selectAlias('frp.user', 'user_id'); // WHERE there are faces with this cluster $query->innerJoin('frp', 'facerecog_faces', 'frf', $query->expr()->eq('frp.id', 'frf.person')); // WHERE faces are from images. $query->innerJoin('frf', 'facerecog_images', 'fri', $query->expr()->eq('fri.id', 'frf.image')); // WHERE these items are memories indexed photos $query->innerJoin('fri', 'memories', 'm', $query->expr()->andX( $query->expr()->eq('fri.file', 'm.fileid'), $query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)), )); // WHERE these photos are in the user's requested folder recursively $query = $this->joinFilecache($query, $root, true, false); // GROUP by ID of face cluster $query->groupBy('frp.id'); $query->addGroupBy('frp.user'); $query->where($query->expr()->isNull('frp.name')); // By default hides individual faces when they have no name. if (!$show_singles) { $query->having($count, $query->createNamedParameter(1)); } // By default it shows the people who were not hidden if (!$show_hidden) { $query->andWhere($query->expr()->eq('frp.is_visible', $query->createNamedParameter(true))); } // ORDER by number of faces in cluster and id for response stability. $query->orderBy('count', 'DESC'); $query->addOrderBy('frp.id', 'DESC'); // It is not worth displaying all unnamed clusters. We show 15 to name them progressively, $query->setMaxResults(15); // FETCH all faces $cursor = $this->executeQueryWithCTEs($query); $faces = $cursor->fetchAll(); // Post process foreach ($faces as &$row) { $row['id'] = (int) $row['id']; $row['count'] = (int) $row['count']; } return $faces; } private function getFaceRecognitionPersons(TimelineRoot &$root, int $currentModel) { $query = $this->connection->getQueryBuilder(); // SELECT all face clusters $count = $query->func()->count($query->createFunction('DISTINCT m.fileid')); $query->select('frp.name')->from('facerecog_persons', 'frp'); $query->selectAlias($count, 'count'); $query->selectAlias('frp.user', 'user_id'); // WHERE there are faces with this cluster $query->innerJoin('frp', 'facerecog_faces', 'frf', $query->expr()->eq('frp.id', 'frf.person')); // WHERE faces are from images. $query->innerJoin('frf', 'facerecog_images', 'fri', $query->expr()->eq('fri.id', 'frf.image')); // WHERE these items are memories indexed photos $query->innerJoin('fri', 'memories', 'm', $query->expr()->andX( $query->expr()->eq('fri.file', 'm.fileid'), $query->expr()->eq('fri.model', $query->createNamedParameter($currentModel)), )); // WHERE these photos are in the user's requested folder recursively $query = $this->joinFilecache($query, $root, true, false); // GROUP by name of face clusters $query->where($query->expr()->isNotNull('frp.name')); $query->groupBy('frp.user'); $query->addGroupBy('frp.name'); // ORDER by number of faces in cluster $query->orderBy('count', 'DESC'); $query->addOrderBy('frp.name', 'ASC'); // FETCH all faces $cursor = $this->executeQueryWithCTEs($query); $faces = $cursor->fetchAll(); // Post process foreach ($faces as &$row) { $row['id'] = $row['name']; $row['count'] = (int) $row['count']; } return $faces; } /** Convert face fields to object */ private function processFaceRecognitionDetection(&$row, $days = false) { if (!isset($row)) { return; } // Differentiate Recognize queries from Face Recognition if (!isset($row['face_width']) || !isset($row['image_width'])) { return; } if (!$days) { $row['facerect'] = [ // Get percentage position and size 'w' => (float) $row['face_width'] / $row['image_width'], 'h' => (float) $row['face_height'] / $row['image_height'], 'x' => (float) $row['face_x'] / $row['image_width'], 'y' => (float) $row['face_y'] / $row['image_height'], ]; } unset($row['face_x'], $row['face_y'], $row['face_w'], $row['face_h'], $row['image_height'], $row['image_width']); } }