sync refactor

Signed-off-by: Varun Patil <radialapps@gmail.com>

package-lock.json

@@ -13,6 +13,7 @@
         "@nextcloud/paths": "^2.1.0",
         "@nextcloud/sharing": "^0.1.0",
         "@nextcloud/vue": "7.12.1",
+        "dexie": "^3.2.4",
         "filerobot-image-editor": "^4.5.1",
         "fuse.js": "^6.6.2",
         "hammerjs": "^2.0.8",
@@ -4552,6 +4553,14 @@
       "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
       "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="
     },
+    "node_modules/dexie": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.2.4.tgz",
+      "integrity": "sha512-VKoTQRSv7+RnffpOJ3Dh6ozknBqzWw/F3iqMdsZg958R0AS8AnY9x9d1lbwENr0gzeGJHXKcGhAMRaqys6SxqA==",
+      "engines": {
+        "node": ">=6.0"
+      }
+    },
     "node_modules/diff": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",
@@ -15026,6 +15035,11 @@
       "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
       "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="
     },
+    "dexie": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.2.4.tgz",
+      "integrity": "sha512-VKoTQRSv7+RnffpOJ3Dh6ozknBqzWw/F3iqMdsZg958R0AS8AnY9x9d1lbwENr0gzeGJHXKcGhAMRaqys6SxqA=="
+    },
     "diff": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",

package.json

@@ -34,6 +34,7 @@
     "@nextcloud/paths": "^2.1.0",
     "@nextcloud/sharing": "^0.1.0",
     "@nextcloud/vue": "7.12.1",
+    "dexie": "^3.2.4",
     "filerobot-image-editor": "^4.5.1",
     "fuse.js": "^6.6.2",
     "hammerjs": "^2.0.8",

src/components/Settings.vue

@@ -160,6 +160,7 @@ import { defineComponent } from 'vue';
 import UserConfig from '../mixins/UserConfig';
 import * as utils from '../services/utils';
 import * as nativex from '../native';
+import * as nativeSync from '../native/sync';
 
 import NcButton from '@nextcloud/vue/dist/Components/NcButton';
 const NcAppSettingsDialog = () => import('@nextcloud/vue/dist/Components/NcAppSettingsDialog');
@@ -169,6 +170,7 @@ const NcCheckboxRadioSwitch = () => import('@nextcloud/vue/dist/Components/NcChe
 import MultiPathSelectionModal from './modal/MultiPathSelectionModal.vue';
 
 import type { IConfig } from '../types';
+import type { LocalFolderConfig } from '../native/types';
 
 export default defineComponent({
   name: 'Settings',
@@ -184,7 +186,7 @@ export default defineComponent({
   mixins: [UserConfig],
 
   data: () => ({
-    localFolders: [] as nativex.LocalFolderConfig[],
+    localFolders: [] as LocalFolderConfig[],
   }),
 
   props: {
@@ -289,11 +291,11 @@ export default defineComponent({
 
     // --------------- Native APIs start -----------------------------
     async refreshNativeConfig() {
-      this.localFolders = await nativex.getLocalFolders();
+      this.localFolders = await nativeSync.getLocalFolders();
     },
 
     async updateDeviceFolders() {
-      await nativex.setLocalFolders(this.localFolders);
+      await nativeSync.setLocalFolders(this.localFolders);
     },
 
     async logout() {

src/native/helpers.ts

@@ -0,0 +1,10 @@
+/**
+ * Add current origin to URL if doesn't have any protocol or origin.
+ */
+export function addOrigin(url: string) {
+  return url.match(/^(https?:)?\/\//)
+    ? url
+    : url.startsWith('/')
+    ? `${location.origin}${url}`
+    : `${location.origin}/${url}`;
+}

src/native/index.ts

@@ -1,7 +1,13 @@
 import axios from '@nextcloud/axios';
 import { generateUrl } from '@nextcloud/router';
-import type { IDay, IPhoto } from './types';
-import { API as SAPI } from './services/API';
+
+import type { IDay, IPhoto } from '../types';
+import { API as SAPI } from '../services/API';
+import { addOrigin } from './helpers';
+
+import type { LocalFolderConfig, ISystemImage } from './types';
+import * as sync from './sync';
+
 const euc = encodeURIComponent;
 
 /** Access NativeX over localhost */
@@ -9,20 +15,6 @@ const BASE_URL = 'http://127.0.0.1';
 
 /** NativeX asynchronous API */
 export const API = {
-  /**
-   * Local days API.
-   * @regex ^/api/days$
-   * @returns {IDay[]} for all locally available days.
-   */
-  DAYS: () => `${BASE_URL}/api/days`,
-  /**
-   * Local photos API.
-   * @regex ^/api/days/\d+$
-   * @param dayId Day ID to fetch photos for
-   * @returns {IPhoto[]} for all locally available photos for this day.
-   */
-  DAY: (dayId: number) => `${BASE_URL}/api/days/${dayId}`,
-
   /**
    * Local photo metadata API.
    * @regex ^/api/image/info/\d+$
@@ -81,13 +73,6 @@ export const API = {
    * @returns {void}
    */
   SHARE_LOCAL: (auid: number) => `${BASE_URL}/api/share/local/${auid}`,
-
-  /**
-   * Get list of local folders configuration.
-   * @regex ^/api/config/local-folders$
-   * @returns {LocalFolderConfig[]} List of local folders configuration
-   */
-  CONFIG_LOCAL_FOLDERS: () => `${BASE_URL}/api/config/local-folders`,
 };
 
 /** NativeX synchronous API. */
@@ -142,12 +127,6 @@ export type NativeX = {
    */
   destroyVideo: (fileid: string) => void;
 
-  /**
-   * Set the local folders configuration to show in the timeline.
-   * @param json JSON-encoded array of LocalFolderConfig
-   */
-  configSetLocalFolders: (json: string) => void;
-
   /**
    * Start the login process
    * @param baseUrl Base URL of the Nextcloud instance
@@ -164,13 +143,17 @@ export type NativeX = {
    * Reload the app.
    */
   reload: () => void;
-};
 
-/** Setting of whether a local folder is enabled */
-export type LocalFolderConfig = {
-  id: string;
-  name: string;
-  enabled: boolean;
+  /**
+   * Initialize local sync.
+   */
+  localSyncInit: () => void;
+
+  /**
+   * Get the next batch of local sync.
+   * @returns {ISystemImage[]} Batch of photos to add
+   */
+  localSyncNext: () => string;
 };
 
 /** The native interface is a global object that is injected by the native app. */
@@ -270,9 +253,7 @@ export async function extendDaysWithLocal(days: IDay[]) {
   if (!has()) return;
 
   // Query native part
-  const res = await fetch(API.DAYS());
-  if (!res.ok) return;
-  const local: IDay[] = await res.json();
+  const local: IDay[] = await sync.getDaysLocal();
   const remoteMap = new Map(days.map((d) => [d.dayid, d]));
 
   // Merge local days into remote days
@@ -301,12 +282,10 @@ export async function extendDaysWithLocal(days: IDay[]) {
 export async function extendDayWithLocal(dayId: number, photos: IPhoto[]) {
   if (!has()) return;
 
-  // Query native part
-  const res = await fetch(API.DAY(dayId));
-  if (!res.ok) return;
-
   // Merge local photos into remote photos
-  const localPhotos: IPhoto[] = await res.json();
+  const localPhotos: IPhoto[] = await sync.getDayLocal(dayId);
+  console.log('nativex', dayId, JSON.stringify(localPhotos));
+
   const serverAUIDs = new Set(photos.map((p) => p.auid));
 
   // Filter out files that are only available locally
@@ -332,21 +311,6 @@ export async function deleteLocalPhotos(photos: IPhoto[], dry: boolean = false):
   return res.data.confirms ? res.data.count : 0;
 }
 
-/**
- * Get list of local folders configuration.
- * Should be called only if NativeX is available.
- */
-export async function getLocalFolders() {
-  return (await axios.get<LocalFolderConfig[]>(API.CONFIG_LOCAL_FOLDERS())).data;
-}
-
-/**
- * Set list of local folders configuration.
- */
-export async function setLocalFolders(config: LocalFolderConfig[]) {
-  nativex?.configSetLocalFolders(JSON.stringify(config));
-}
-
 /**
  * Log out from Nextcloud and pass ahead.
  */
@@ -357,12 +321,28 @@ export async function logout() {
 }
 
 /**
- * Add current origin to URL if doesn't have any protocol or origin.
+ * Iterate the local files on the device.
  */
-function addOrigin(url: string) {
-  return url.match(/^(https?:)?\/\//)
-    ? url
-    : url.startsWith('/')
-    ? `${location.origin}${url}`
-    : `${location.origin}/${url}`;
+export async function* localSyncIter() {
+  if (!has()) return;
+
+  // Initialize and iterate
+  nativex.localSyncInit();
+
+  while (true) {
+    const next = nativex.localSyncNext();
+    if (!next) break;
+
+    let photos: ISystemImage[];
+    try {
+      photos = JSON.parse(next);
+    } catch (e) {
+      console.error('Failed to parse local sync JSON', e);
+      continue;
+    }
+
+    for (const photo of photos) {
+      yield photo;
+    }
+  }
 }

src/native/sync.ts

@@ -0,0 +1,161 @@
+import Dexie from 'dexie';
+import { getBuilder } from '@nextcloud/browser-storage';
+import { localSyncIter } from '.';
+
+import type { IDay, IPhoto } from '../types';
+import { ISystemImage, LocalFolderConfig } from './types';
+
+class MemoriesDatabase extends Dexie {
+  local!: Dexie.Table<ILocalFile, number>;
+
+  constructor() {
+    super('MemoriesDatabase');
+    this.version(1).stores({
+      local: '++id, fileid, auid, dayid, flag, [bucket_id+dayid]',
+    });
+  }
+}
+
+/** Table for a locally stored file. */
+interface ILocalFile extends ISystemImage {
+  id?: number;
+  dayid: number;
+  flag: number;
+}
+
+/** Open database */
+const db = new MemoriesDatabase();
+
+/** Preferences database */
+const storage = getBuilder('memories_sync').clearOnLogout(false).persist().build();
+const STORAGE_LOCAL_FOLDERS = 'local_folders';
+
+// Cache for preferences
+let _enabledBucketIds: number[] | null = null;
+
+/**
+ * Trigger the synchronization process at the local database.
+ */
+export async function go() {
+  // Clear local database
+  await db.local.clear();
+
+  // Mark all entries
+  await db.local.where({ flag: 0 }).modify({ flag: 1 });
+
+  for await (const sysImg of localSyncIter()) {
+    await syncImage(sysImg);
+  }
+
+  // Delete all entries with flag=1
+  await db.local.where({ flag: 1 }).delete();
+}
+
+async function syncImage(sysImg: ISystemImage) {
+  // Check if file already exists with same mtime
+  const entry = await db.local.where({ fileid: sysImg.fileid, mtime: sysImg.mtime }).first();
+  if (entry) {
+    // Unmark this entry
+    await db.local.where({ id: entry.id }).modify({ flag: 0 });
+    return;
+  }
+
+  // Insert new file
+  await db.transaction('rw', db.local, async () => {
+    await db.local.where({ fileid: sysImg.fileid }).delete();
+    await db.local.add({
+      ...sysImg,
+      dayid: Math.floor(sysImg.datetaken / 86400),
+      flag: 0,
+    });
+  });
+
+  console.log('nativex, synced file', sysImg.basename);
+}
+
+/**
+ * Get the local days list.
+ * @param dayid Day ID
+ */
+export async function getDaysLocal(): Promise<IDay[]> {
+  if (!getEnabledBucketIds()?.length) return [];
+
+  const days: IDay[] = [];
+
+  // Get all day IDs
+  const dayIds = await db.local.orderBy('dayid').uniqueKeys();
+
+  // For each unique username, call usePostCountSince():
+  await Promise.all(
+    dayIds.map(async (dayid: number) => {
+      days.push({
+        dayid: dayid,
+        count: await db.local
+          .where(['bucket_id', 'dayid'])
+          .anyOf(getEnabledBucketIds().map((bucket_id) => [bucket_id, dayid]))
+          .count(),
+      });
+    })
+  );
+
+  return days.filter((d) => d.count > 0);
+}
+
+/**
+ * Get the local photos for a given day.
+ * @param dayid Day ID
+ */
+export async function getDayLocal(dayid: number): Promise<IPhoto[]> {
+  if (!getEnabledBucketIds()?.length) return [];
+
+  return (
+    await db.local
+      .where(['bucket_id', 'dayid'])
+      .anyOf(getEnabledBucketIds().map((bucket_id) => [bucket_id, dayid]))
+      .toArray()
+  ).map((file) => ({
+    ...file,
+    bucket_id: undefined,
+    bucket_name: undefined,
+    flag: undefined as unknown as number,
+  }));
+}
+
+/** Get the enabled bucket ids list */
+export function getEnabledBucketIds(): number[] {
+  if (_enabledBucketIds) return _enabledBucketIds;
+
+  try {
+    _enabledBucketIds = JSON.parse(storage.getItem(STORAGE_LOCAL_FOLDERS) || '[]') as number[];
+  } catch (e) {
+    _enabledBucketIds = [];
+  }
+
+  return _enabledBucketIds;
+}
+
+/** Get the active bucket list. */
+export async function getLocalFolders(): Promise<LocalFolderConfig[]> {
+  const bucketIds = await db.local.orderBy('bucket_id').uniqueKeys();
+  const enabledIds = getEnabledBucketIds();
+
+  const buckets = await Promise.all(
+    bucketIds.map(async (bucket_id: number) => {
+      const file = await db.local.where({ bucket_id }).first();
+      if (!file) return null;
+
+      return {
+        id: file.bucket_id,
+        name: file.bucket_name,
+        enabled: enabledIds.includes(file.bucket_id),
+      };
+    })
+  );
+
+  return buckets.filter((b) => b !== null) as LocalFolderConfig[];
+}
+
+export async function setLocalFolders(config: LocalFolderConfig[]) {
+  _enabledBucketIds = config.filter((f) => f.enabled).map((f) => f.id);
+  storage.setItem(STORAGE_LOCAL_FOLDERS, JSON.stringify(_enabledBucketIds));
+}

src/native/types.ts

@@ -0,0 +1,23 @@
+/** Local file type */
+export type ISystemImage = {
+  fileid: number;
+  basename: string;
+  mimetype: string;
+  h: number;
+  w: number;
+  size: number;
+  etag: string;
+  mtime: number;
+  epoch: number;
+  auid: number;
+  bucket_id: number;
+  bucket_name: string;
+  datetaken: number;
+};
+
+/** Setting of whether a local folder is enabled */
+export type LocalFolderConfig = {
+  id: number;
+  name: string;
+  enabled: boolean;
+};