RPJosh
/
memories
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement people tab for recognize 3 (fix #43)

cache
Varun Patil 2022-10-07 12:28:39 -07:00
parent 9dc4ae20cb
commit d1e9205a54
13 changed files with 338 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
appinfo/routes.php
View File

@ -4,24 +4,20 @@ return [
// Days and folder API
['name' => 'page#main', 'url' => '/', 'verb' => 'GET'],
['name' => 'page#folder', 'url' => '/folders/{path}', 'verb' => 'GET',
'requirements' => [
'path' => '.*',
],
'defaults' => [
'path' => '',
]
'requirements' => [ 'path' => '.*' ],
'defaults' => [ 'path' => '' ]
],
['name' => 'page#favorites', 'url' => '/favorites', 'verb' => 'GET'],
['name' => 'page#videos', 'url' => '/videos', 'verb' => 'GET'],
['name' => 'page#archive', 'url' => '/archive', 'verb' => 'GET'],
['name' => 'page#thisday', 'url' => '/thisday', 'verb' => 'GET'],
['name' => 'page#people', 'url' => '/people/{name}', 'verb' => 'GET',
'requirements' => [ 'name' => '.*' ],
'defaults' => [ 'name' => '' ]
],
['name' => 'page#tags', 'url' => '/tags/{name}', 'verb' => 'GET',
'requirements' => [
'name' => '.*',
],
'defaults' => [
'name' => '',
]
'requirements' => [ 'name' => '.*' ],
'defaults' => [ 'name' => '' ]
],
// API
@ -29,6 +25,8 @@ return [
['name' => 'api#dayPost', 'url' => '/api/days', 'verb' => 'POST'],
['name' => 'api#day', 'url' => '/api/days/{id}', 'verb' => 'GET'],
['name' => 'api#tags', 'url' => '/api/tags', 'verb' => 'GET'],
['name' => 'api#faces', 'url' => '/api/faces', 'verb' => 'GET'],
['name' => 'api#facePreviews', 'url' => '/api/face-previews/{id}', 'verb' => 'GET'],
['name' => 'api#imageInfo', 'url' => '/api/info/{id}', 'verb' => 'GET'],
['name' => 'api#imageEdit', 'url' => '/api/edit/{id}', 'verb' => 'PATCH'],
['name' => 'api#archive', 'url' => '/api/archive/{id}', 'verb' => 'PATCH'],

57
lib/Controller/ApiController.php
View File

@ -81,6 +81,12 @@ class ApiController extends Controller {
$transforms[] = array($this->timelineQuery, 'transformVideoFilter');
}
// Filter only for one face
$faceId = $this->request->getParam('face');
if ($faceId) {
$transforms[] = array($this->timelineQuery, 'transformFaceFilter', intval($faceId));
}
// Filter only for one tag
$tagName = $this->request->getParam('tag');
if ($tagName) {
@ -303,6 +309,57 @@ class ApiController extends Controller {
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* @NoAdminRequired
*
* Get list of faces with counts of images
* @return JSONResponse
*/
public function faces(): JSONResponse {
$user = $this->userSession->getUser();
if (is_null($user)) {
return new JSONResponse([], Http::STATUS_PRECONDITION_FAILED);
}
// If this isn't the timeline folder then things aren't going to work
$folder = $this->getRequestFolder();
if (is_null($folder)) {
return new JSONResponse([], Http::STATUS_NOT_FOUND);
}
// Run actual query
$list = $this->timelineQuery->getFaces(
$folder,
);
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* @NoAdminRequired
* @NoCSRFRequired
*
* Get preview objects for a face ID
* @return JSONResponse
*/
public function facePreviews(string $id): JSONResponse {
$user = $this->userSession->getUser();
if (is_null($user)) {
return new JSONResponse([], Http::STATUS_PRECONDITION_FAILED);
}
// If this isn't the timeline folder then things aren't going to work
$folder = $this->getRequestFolder();
if (is_null($folder)) {
return new JSONResponse([], Http::STATUS_NOT_FOUND);
}
// Run actual query
$list = $this->timelineQuery->getFacePreviews(
$folder, intval($id),
);
return new JSONResponse($list, Http::STATUS_OK);
}
/**
* @NoAdminRequired
*

9
lib/Controller/PageController.php
View File

@ -68,6 +68,7 @@ class PageController extends Controller {
// Apps enabled
$this->initialState->provideInitialState('systemtags', $this->appManager->isEnabledForUser('systemtags') === true);
$this->initialState->provideInitialState('recognize', $this->appManager->isEnabledForUser('recognize') === true);
$response = new TemplateResponse($this->appName, 'main');
return $response;
@ -113,6 +114,14 @@ class PageController extends Controller {
return $this->main();
}
/**
* @NoAdminRequired
* @NoCSRFRequired
*/
public function people() {
return $this->main();
}
/**
* @NoAdminRequired
* @NoCSRFRequired

1
lib/Db/TimelineQuery.php
View File

@ -9,6 +9,7 @@ class TimelineQuery {
use TimelineQueryDays;
use TimelineQueryFilters;
use TimelineQueryTags;
use TimelineQueryFaces;
protected IDBConnection $connection;

86
lib/Db/TimelineQueryFaces.php 100644
View File

@ -0,0 +1,86 @@
<?php
declare(strict_types=1);
namespace OCA\Memories\Db;
use OCP\IDBConnection;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Folder;
trait TimelineQueryFaces {
protected IDBConnection $connection;
public function transformFaceFilter(IQueryBuilder &$query, string $userId, int $faceId) {
$query->innerJoin('m', 'recognize_face_detections', 'rfd', $query->expr()->andX(
$query->expr()->eq('rfd.file_id', 'm.fileid'),
$query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($faceId)),
));
}
public function getFaces(Folder $folder) {
$query = $this->connection->getQueryBuilder();
// SELECT all face clusters
$count = $query->func()->count($query->createFunction('DISTINCT m.fileid'), 'count');
$query->select('rfc.id', 'rfc.title', $count)->from('recognize_face_clusters', 'rfc');
// WHERE there are faces with this cluster
$query->innerJoin('rfc', 'recognize_face_detections', 'rfd', $query->expr()->eq('rfc.id', 'rfd.cluster_id'));
// WHERE these items are memories indexed photos
$query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id'));
// WHERE these photos are in the user's requested folder recursively
$query->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, true, false));
// GROUP by ID of face cluster
$query->groupBy('rfc.id');
// ORDER by number of faces in cluster
$query->orderBy('count', 'DESC');
// FETCH all faces
$faces = $query->executeQuery()->fetchAll();
// Post process
foreach($faces as &$row) {
$row["name"] = $row["title"];
unset($row["title"]);
$row["count"] = intval($row["count"]);
}
return $faces;
}
public function getFacePreviews(Folder $folder, int $faceId) {
$query = $this->connection->getQueryBuilder();
// SELECT face detections for ID
$query->select('rfd.file_id', 'rfd.x', 'rfd.y', 'rfd.width', 'rfd.height', 'f.etag')->from('recognize_face_detections', 'rfd');
$query->where($query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($faceId)));
// WHERE these photos are memories indexed
$query->innerJoin('rfd', 'memories', 'm', $query->expr()->eq('m.fileid', 'rfd.file_id'));
// WHERE these photos are in the user's requested folder recursively
$query->innerJoin('m', 'filecache', 'f', $this->getFilecacheJoinQuery($query, $folder, true, false));
// MAX 4 results
$query->setMaxResults(4);
// FETCH all face detections
$previews = $query->executeQuery()->fetchAll();
// Post-process, everthing is a number
foreach($previews as &$row) {
$row["fileid"] = intval($row["file_id"]);
unset($row["file_id"]);
$row["x"] = floatval($row["x"]);
$row["y"] = floatval($row["y"]);
$row["width"] = floatval($row["width"]);
$row["height"] = floatval($row["height"]);
}
return $previews;
}
}

6
src/App.vue
View File

@ -19,6 +19,10 @@
:title="t('memories', 'Videos')">
<Video slot="icon" :size="20" />
</NcAppNavigationItem>
<NcAppNavigationItem :to="{name: 'people'}"
:title="t('memories', 'People')">
<PeopleIcon slot="icon" :size="20" />
</NcAppNavigationItem>
<NcAppNavigationItem :to="{name: 'archive'}"
:title="t('memories', 'Archive')">
<ArchiveIcon slot="icon" :size="20" />
@ -79,6 +83,7 @@ import Star from 'vue-material-design-icons/Star.vue'
import Video from 'vue-material-design-icons/Video.vue'
import ArchiveIcon from 'vue-material-design-icons/PackageDown.vue';
import CalendarIcon from 'vue-material-design-icons/Calendar.vue';
import PeopleIcon from 'vue-material-design-icons/AccountBoxMultiple.vue';
import TagsIcon from 'vue-material-design-icons/Tag.vue';
@Component({
@ -98,6 +103,7 @@ import TagsIcon from 'vue-material-design-icons/Tag.vue';
Video,
ArchiveIcon,
CalendarIcon,
PeopleIcon,
TagsIcon,
},
})

108
src/components/Tag.vue
View File

@ -3,8 +3,9 @@
hasPreview: previews.length > 0,
onePreview: previews.length === 1,
hasError: error,
isFace: isFace,
}"
@click="openTag(data)"
@click="openTag()"
v-bind:style="{
width: rowHeight + 'px',
height: rowHeight + 'px',
@ -22,6 +23,7 @@
'p-loading': !(info.flag & c.FLAG_LOADED),
'p-load-fail': info.flag & c.FLAG_LOAD_FAIL,
}"
:style="extraStyles"
@load="info.flag |= c.FLAG_LOADED"
@error="info.flag |= c.FLAG_LOAD_FAIL" />
</div>
@ -39,6 +41,14 @@ import NcCounterBubble from '@nextcloud/vue/dist/Components/NcCounterBubble'
import axios from '@nextcloud/axios'
import GlobalMixin from '../mixins/GlobalMixin';
import { constants } from '../services/Utils';
interface IFaceDetection extends IPhoto {
x: number;
y: number;
width: number;
height: number;
}
@Component({
components: {
@ -55,8 +65,8 @@ export default class Tag extends Mixins(GlobalMixin) {
// Error occured fetching thumbs
private error = false;
/** Passthrough */
private getPreviewUrl = getPreviewUrl;
/** Extra styles to apply on image */
private extraStyles = {};
mounted() {
this.refreshPreviews();
@ -67,12 +77,31 @@ export default class Tag extends Mixins(GlobalMixin) {
this.refreshPreviews();
}
/** Refresh previews */
getPreviewUrl(fileid: number, etag: string) {
if (this.isFace) {
return generateUrl(`/core/preview?fileId=${fileid}&c=${etag}&x=2048&y=2048&forceIcon=0&a=1`);
}
return getPreviewUrl(fileid, etag);
}
get isFace() {
return this.data.flag & constants.c.FLAG_IS_FACE;
}
async refreshPreviews() {
// Reset state
this.error = false;
this.extraStyles = {};
// Get previews
if (this.isFace) {
await this.refreshPreviewsFace();
} else {
await this.refreshPreviewsTag();
}
}
/** Refresh previews for tag */
async refreshPreviewsTag() {
const url = `/apps/memories/api/days/*?limit=4&tag=${this.data.name}`;
try {
const res = await axios.get<IPhoto[]>(generateUrl(url));
@ -87,9 +116,66 @@ export default class Tag extends Mixins(GlobalMixin) {
}
}
/** Refresh previews for face */
async refreshPreviewsFace() {
const url = `/apps/memories/api/face-previews/${this.data.faceid}`;
try {
const res = await axios.get<IFaceDetection[]>(generateUrl(url));
res.data.forEach((p) => p.flag = 0);
const face = this.chooseFaceDetection(res.data);
this.previews = [face];
this.extraStyles = this.getCoverStyle(face);
} catch (e) {
this.error = true;
console.error(e);
}
}
/** Open tag */
openTag(tag: ITag) {
this.$router.push({ name: 'tags', params: { name: tag.name }});
openTag() {
if (this.isFace) {
this.$router.push({ name: 'people', params: { name: this.data.faceid.toString() }});
} else {
this.$router.push({ name: 'tags', params: { name: this.data.name }});
}
}
/** Choose the most appropriate face detection */
private chooseFaceDetection(detections: IFaceDetection[]) {
const scoreFacePosition = (faceDetection: IFaceDetection) => {
return Math.max(0, -1 * (faceDetection.x - faceDetection.width * 0.5))
+ Math.max(0, -1 * (faceDetection.y - faceDetection.height * 0.5))
+ Math.max(0, -1 * (1 - (faceDetection.x + faceDetection.width) - faceDetection.width * 0.5))
+ Math.max(0, -1 * (1 - (faceDetection.y + faceDetection.height) - faceDetection.height * 0.5))
}
return detections.sort((a, b) =>
scoreFacePosition(a)
- scoreFacePosition(b)
)[0];
}
/**
* This will produce an inline style to apply to images
* to zoom toward the detected face
*
* @return {{}|{transform: string, width: string, transformOrigin: string}}
*/
getCoverStyle(detection: IFaceDetection) {
// Zoom into the picture so that the face fills the --photos-face-width box nicely
// if the face is larger than the image, we don't zoom out (reason for the Math.max)
const zoom = Math.max(1, (1 / detection.width) * 0.4)
const horizontalCenterOfFace = (detection.x + detection.width / 2) * 100
const verticalCenterOfFace = (detection.y + detection.height / 2) * 100
return {
// we translate the image so that the center of the detected face is in the center
// and add the zoom
transform: `translate(calc(${this.rowHeight}px/2 - ${horizontalCenterOfFace}% ), calc(${this.rowHeight}px/2 - ${verticalCenterOfFace}% )) scale(${zoom})`,
// this is necessary for the zoom to zoom toward the center of the face
transformOrigin: `${horizontalCenterOfFace}% ${verticalCenterOfFace}%`,
}
}
}
</script>
@ -112,6 +198,12 @@ export default class Tag extends Mixins(GlobalMixin) {
word-wrap: break-word;
text-overflow: ellipsis;
line-height: 1em;
.isFace > & {
top: unset;
bottom: 10%;
transform: unset;
}
}
.bbl {
@ -134,6 +226,7 @@ export default class Tag extends Mixins(GlobalMixin) {
margin: 0;
width: 50%;
height: 50%;
overflow: hidden;
display: inline-block;
.tag.onePreview > & {
@ -143,7 +236,6 @@ export default class Tag extends Mixins(GlobalMixin) {
> img {
padding: 0;
width: 100%;
height: 100%;
filter: brightness(50%);
cursor: pointer;

19
src/components/Timeline.vue
View File

@ -514,6 +514,11 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
query.set('archive', '1');
}
// People
if (this.$route.name === 'people' && this.$route.params.name) {
query.set('face', this.$route.params.name);
}
// Tags
if (this.$route.name === 'tags' && this.$route.params.name) {
query.set('tag', this.$route.params.name);
@ -550,11 +555,11 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
// Special headers
if (head.dayId === this.TagDayID.FOLDERS) {
head.name = this.t("memories", "Folders");
return head.name;
return (head.name = this.t("memories", "Folders"));
} else if (head.dayId === this.TagDayID.TAGS) {
head.name = this.t("memories", "Tags");
return head.name;
return (head.name = this.t("memories", "Tags"));
} else if (head.dayId === this.TagDayID.FACES) {
return (head.name = this.t("memories", "People"));
}
// Make date string
@ -582,6 +587,8 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
data = await dav.getOnThisDayData();
} else if (this.$route.name === 'tags' && !this.$route.params.name) {
data = await dav.getTagsData();
} else if (this.$route.name === 'people' && !this.$route.params.name) {
data = await dav.getPeopleData();
} else {
data = (await axios.get<IDay[]>(generateUrl(this.appendQuery(url), params))).data;
}
@ -895,6 +902,10 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
photo.flag |= this.c.FLAG_IS_FOLDER;
delete photo.isfolder;
}
if (photo.isface) {
photo.flag |= this.c.FLAG_IS_FACE;
delete photo.isface;
}
if (photo.istag) {
photo.flag |= this.c.FLAG_IS_TAG;
delete photo.istag;

1
src/mixins/UserConfig.ts
View File

@ -33,6 +33,7 @@ export default class UserConfig extends Vue {
config_timelinePath = loadState('memories', 'timelinePath') || '';
config_showHidden = loadState('memories', 'showHidden') === "true";
config_tagsEnabled = loadState('memories', 'systemtags');
config_recognizeEnabled = loadState('memories', 'recognize');
created() {
subscribe(eventName, this.updateLocalSetting)

9
src/router.ts
View File

@ -100,6 +100,15 @@
}),
},
{
path: '/people/:name*',
component: Timeline,
name: 'people',
props: route => ({
rootTitle: t('memories', 'People'),
}),
},
{
path: '/tags/:name*',
component: Timeline,

34
src/services/DavRequests.ts
View File

@ -430,7 +430,7 @@ export async function getOnThisDayData(): Promise<IDay[]> {
/**
* Get list of tags and convert to Days response
*/
export async function getTagsData(): Promise<IDay[]> {
export async function getTagsData(): Promise<IDay[]> {
// Query for photos
let data: {
count: number;
@ -455,4 +455,36 @@ export async function getOnThisDayData(): Promise<IDay[]> {
istag: true,
} as ITag)),
}]
}
/**
* Get list of tags and convert to Days response
*/
export async function getPeopleData(): Promise<IDay[]> {
// Query for photos
let data: {
id: number;
count: number;
name: string;
}[] = [];
try {
const res = await axios.get<typeof data>(generateUrl('/apps/memories/api/faces'));
data = res.data;
} catch (e) {
throw e;
}
// Convert to days response
return [{
dayid: constants.TagDayID.FACES,
count: data.length,
detail: data.map((face) => ({
name: face.name,
count: face.count,
fileid: hashCode(face.name),
faceid: face.id,
istag: true,
isface: true,
} as any)),
}]
}

12
src/services/Utils.ts
View File

@ -52,16 +52,18 @@ export const constants = {
FLAG_IS_FAVORITE: 1 << 4,
FLAG_IS_FOLDER: 1 << 5,
FLAG_IS_TAG: 1 << 6,
FLAG_SELECTED: 1 << 7,
FLAG_LEAVING: 1 << 8,
FLAG_EXIT_LEFT: 1 << 9,
FLAG_ENTER_RIGHT: 1 << 10,
FLAG_FORCE_RELOAD: 1 << 11,
FLAG_IS_FACE: 1 << 7,
FLAG_SELECTED: 1 << 8,
FLAG_LEAVING: 1 << 9,
FLAG_EXIT_LEFT: 1 << 10,
FLAG_ENTER_RIGHT: 1 << 11,
FLAG_FORCE_RELOAD: 1 << 12,
},
TagDayID: {
START: -(1 << 30),
FOLDERS: -(1 << 30) + 1,
TAGS: -(1 << 30) + 2,
FACES: -(1 << 30) + 3,
},
}

4
src/types.ts
View File

@ -49,6 +49,8 @@ export type IPhoto = {
isfolder?: boolean;
/** Is this a tag */
istag?: boolean;
/** Is this a face */
isface?: boolean;
/** Optional datetaken epoch */
datetaken?: number;
}
@ -67,6 +69,8 @@ export interface ITag extends IPhoto {
name: string;
/** Number of images in this tag */
count: number;
/** ID of face if this is a face */
faceid?: number;
}
export type IRow = {