Merge branch 'master' into stable24

CHANGELOG.md

@@ -4,6 +4,9 @@ This file is manually updated. Please file an issue if something is missing.
 
 ## v4.8.0, v3.8.0
 
+- **Feature**: Support for Live Photos ([#124](https://github.com/pulsejet/memories/issues/124))
+  - You need to run `occ memories:index --clear` to reindex live photos
+  - Only JPEG (iOS with MOV, Google, Samsung) is supported. HEIC is not supported.
 - **Feature**: Timeline path now scans recursively for mounted volumes / shares inside it
 - **Feature**: Multiple timeline paths can be specified ([#178](https://github.com/pulsejet/memories/issues/178))
 - Support for server-side encrypted storage ([#99](https://github.com/pulsejet/memories/issues/99))

appinfo/routes.php

@@ -59,6 +59,7 @@ return [
         ['name' => 'Archive#archive', 'url' => '/api/archive/{id}', 'verb' => 'PATCH'],
 
         ['name' => 'Video#transcode', 'url' => '/api/video/transcode/{client}/{fileid}/{profile}', 'verb' => 'GET'],
+        ['name' => 'Video#livephoto', 'url' => '/api/video/livephoto/{fileid}', 'verb' => 'GET'],
 
         // Config API
         ['name' => 'Other#setUserConfig', 'url' => '/api/config/{key}', 'verb' => 'PUT'],

lib/Command/Index.php

@@ -268,6 +268,11 @@ class Index extends Command
                 return;
             }
 
+            // check path contains IMDB then skip
+            if (false !== strpos($folderPath, 'IMDB')) {
+                return;
+            }
+
             $nodes = $folder->getDirectoryListing();
 
             foreach ($nodes as $i => &$node) {
@@ -287,16 +292,21 @@ class Index extends Command
         }
     }
 
-    private function parseFile(File &$file, bool &$refresh): void
+    private function parseFile(File &$file, bool &$refresh): bool
     {
         // Process the file
         $res = $this->timelineWrite->processFile($file, $refresh);
         if (2 === $res) {
             ++$this->nProcessed;
-        } elseif (1 === $res) {
+
+            return true;
+        }
+        if (1 === $res) {
             ++$this->nSkipped;
         } else {
             ++$this->nInvalid;
         }
+
+        return false;
     }
 }

lib/Controller/VideoController.php

@@ -23,9 +23,11 @@ declare(strict_types=1);
 
 namespace OCA\Memories\Controller;
 
+use OCA\Memories\Exif;
 use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\DataDisplayResponse;
 use OCP\AppFramework\Http\JSONResponse;
+use OCP\Files\File;
 
 class VideoController extends ApiBase
 {
@@ -35,8 +37,6 @@ class VideoController extends ApiBase
      * @NoCSRFRequired
      *
      * Transcode a video to HLS by proxy
-     *
-     * @return JSONResponse an empty JSONResponse with respective http status code
      */
     public function transcode(string $client, string $fileid, string $profile): Http\Response
     {
@@ -137,6 +137,102 @@ class VideoController extends ApiBase
         return $response;
     }
 
+    /**
+     * @NoAdminRequired
+     *
+     * @NoCSRFRequired
+     *
+     * Return the live video part of a live photo
+     */
+    public function livephoto(string $fileid)
+    {
+        $fileid = (int) $fileid;
+        $files = $this->rootFolder->getById($fileid);
+        if (0 === \count($files)) {
+            return new JSONResponse(['message' => 'File not found'], Http::STATUS_NOT_FOUND);
+        }
+        $file = $files[0];
+
+        // Check file etag
+        $etag = $file->getEtag();
+        if ($etag !== $this->request->getParam('etag')) {
+            return new JSONResponse(['message' => 'File changed'], Http::STATUS_PRECONDITION_FAILED);
+        }
+
+        // Check file liveid
+        $liveid = $this->request->getParam('liveid');
+        if (!$liveid) {
+            return new JSONResponse(['message' => 'Live ID not provided'], Http::STATUS_BAD_REQUEST);
+        }
+
+        // Response data
+        $name = '';
+        $blob = null;
+        $mime = '';
+
+        // Video is inside the file
+        $path = null;
+        if (str_starts_with($liveid, 'self__')) {
+            $path = $file->getStorage()->getLocalFile($file->getInternalPath());
+            $mime = 'video/mp4';
+            $name = $file->getName().'.mp4';
+        }
+
+        // Different manufacurers have different formats
+        if ('self__trailer' === $liveid) {
+            try { // Get trailer
+                $blob = Exif::getBinaryExifProp($path, '-trailer');
+            } catch (\Exception $e) {
+                return new JSONResponse(['message' => 'Trailer not found'], Http::STATUS_NOT_FOUND);
+            }
+        } elseif ('self__embeddedvideo' === $liveid) {
+            try { // Get embedded video file
+                $blob = Exif::getBinaryExifProp($path, '-EmbeddedVideoFile');
+            } catch (\Exception $e) {
+                return new JSONResponse(['message' => 'Embedded video not found'], Http::STATUS_NOT_FOUND);
+            }
+        } else {
+            // Get stored video file (Apple MOV)
+            $lp = $this->timelineQuery->getLivePhoto($fileid);
+            if (!$lp || $lp['liveid'] !== $liveid) {
+                return new JSONResponse(['message' => 'Live ID not found'], Http::STATUS_NOT_FOUND);
+            }
+
+            // Get and return file
+            $liveFileId = (int) $lp['fileid'];
+            $files = $this->rootFolder->getById($liveFileId);
+            if (0 === \count($files)) {
+                return new JSONResponse(['message' => 'Live file not found'], Http::STATUS_NOT_FOUND);
+            }
+            $liveFile = $files[0];
+
+            if ($liveFile instanceof File) {
+                // Requested only JSON info
+                if ('json' === $this->request->getParam('format')) {
+                    return new JSONResponse($lp);
+                }
+
+                $name = $liveFile->getName();
+                $blob = $liveFile->getContent();
+                $mime = $liveFile->getMimeType();
+            }
+        }
+
+        // Make and send response
+        if ($blob) {
+            $response = new DataDisplayResponse($blob, Http::STATUS_OK, []);
+            $response->setHeaders([
+                'Content-Type' => $mime,
+                'Content-Disposition' => "attachment; filename=\"{$name}\"",
+            ]);
+            $response->cacheFor(3600 * 24, false, false);
+
+            return $response;
+        }
+
+        return new JSONResponse(['message' => 'Live file not found'], Http::STATUS_NOT_FOUND);
+    }
+
     private function getUpstream($client, $path, $profile)
     {
         $path = rawurlencode($path);

lib/Db/LivePhoto.php

@@ -0,0 +1,96 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\Memories\Db;
+
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\File;
+use OCP\IDBConnection;
+
+class LivePhoto
+{
+    protected IDBConnection $connection;
+
+    public function __construct(IDBConnection $connection)
+    {
+        $this->connection = $connection;
+    }
+
+    /** Check if a given Exif data is the video part of a live photo */
+    public function isVideoPart(array &$exif)
+    {
+        return 'video/quicktime' === $exif['MIMEType'] && \array_key_exists('ContentIdentifier', $exif);
+    }
+
+    /** Get liveid from photo part */
+    public function getLivePhotoId(array &$exif)
+    {
+        // Apple JPEG (MOV has ContentIdentifier)
+        if (\array_key_exists('MediaGroupUUID', $exif)) {
+            return $exif['MediaGroupUUID'];
+        }
+
+        // Samsung JPEG
+        if (\array_key_exists('EmbeddedVideoType', $exif) && str_contains($exif['EmbeddedVideoType'], 'MotionPhoto')) {
+            return 'self__embeddedvideo';
+        }
+
+        // Google JPEG and Samsung HEIC (Apple?)
+        if (\array_key_exists('MotionPhoto', $exif)) {
+            if ('image/jpeg' === $exif['MIMEType']) {
+                // Google JPEG -- image should hopefully be in trailer
+                return 'self__trailer';
+            }
+            if ('image/heic' === $exif['MIMEType']) {
+                // Samsung HEIC -- no way to get this out yet
+                return '';
+            }
+        }
+
+        return '';
+    }
+
+    public function processVideoPart(File &$file, array &$exif)
+    {
+        $fileId = $file->getId();
+        $mtime = $file->getMTime();
+        $liveid = $exif['ContentIdentifier'];
+        if (empty($liveid)) {
+            return;
+        }
+
+        $query = $this->connection->getQueryBuilder();
+        $query->select('fileid')
+            ->from('memories_livephoto')
+            ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
+        ;
+        $cursor = $query->executeQuery();
+        $prevRow = $cursor->fetch();
+        $cursor->closeCursor();
+
+        if ($prevRow) {
+            // Update existing row
+            $query->update('memories_livephoto')
+                ->set('liveid', $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR))
+                ->set('mtime', $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT))
+                ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
+            ;
+            $query->executeStatement();
+        } else {
+            // Try to create new row
+            try {
+                $query->insert('memories_livephoto')
+                    ->values([
+                        'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
+                        'mtime' => $query->createNamedParameter($mtime, IQueryBuilder::PARAM_INT),
+                        'fileid' => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
+                    ])
+                ;
+                $query->executeStatement();
+            } catch (\Exception $ex) {
+                error_log('Failed to create memories_livephoto record: '.$ex->getMessage());
+            }
+        }
+    }
+}

lib/Db/TimelineQuery.php

@@ -14,6 +14,7 @@ class TimelineQuery
     use TimelineQueryFaces;
     use TimelineQueryFilters;
     use TimelineQueryFolders;
+    use TimelineQueryLivePhoto;
     use TimelineQueryTags;
 
     protected IDBConnection $connection;

lib/Db/TimelineQueryDays.php

@@ -152,7 +152,7 @@ trait TimelineQueryDays
         // We don't actually use m.datetaken here, but postgres
         // needs that all fields in ORDER BY are also in SELECT
         // when using DISTINCT on selected fields
-        $query->select($fileid, 'm.isvideo', 'm.video_duration', 'm.datetaken', 'm.dayid', 'm.w', 'm.h')
+        $query->select($fileid, 'm.isvideo', 'm.video_duration', 'm.datetaken', 'm.dayid', 'm.w', 'm.h', 'm.liveid')
             ->from('memories', 'm')
         ;
 
@@ -282,6 +282,9 @@ trait TimelineQueryDays
                 $row['isfavorite'] = 1;
             }
             unset($row['categoryid']);
+            if (!$row['liveid']) {
+                unset($row['liveid']);
+            }
 
             // Check if path exists and starts with basePath and remove
             if (isset($row['path']) && !empty($row['path'])) {

lib/Db/TimelineQueryLivePhoto.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\Memories\Db;
+
+trait TimelineQueryLivePhoto
+{
+    public function getLivePhoto(int $fileid)
+    {
+        $qb = $this->connection->getQueryBuilder();
+        $qb->select('lp.fileid', 'lp.liveid')
+            ->from('memories', 'm')
+            ->where($qb->expr()->eq('m.fileid', $qb->createNamedParameter($fileid)))
+            ->innerJoin('m', 'memories_livephoto', 'lp', $qb->expr()->andX(
+                $qb->expr()->eq('lp.liveid', 'm.liveid'),
+            ))
+        ;
+        $result = $qb->executeQuery();
+        $row = $result->fetch();
+        $result->closeCursor();
+
+        return $row;
+    }
+}

lib/Db/TimelineWrite.php

@@ -15,11 +15,13 @@ class TimelineWrite
 {
     protected IDBConnection $connection;
     protected IPreview $preview;
+    protected LivePhoto $livePhoto;
 
     public function __construct(IDBConnection $connection, IPreview &$preview)
     {
         $this->connection = $connection;
         $this->preview = $preview;
+        $this->livePhoto = new LivePhoto($connection);
     }
 
     /**
@@ -79,6 +81,19 @@ class TimelineWrite
         $cursor = $query->executeQuery();
         $prevRow = $cursor->fetch();
         $cursor->closeCursor();
+
+        // Check in live-photo table in case this is a video part of a live photo
+        if (!$prevRow) {
+            $query = $this->connection->getQueryBuilder();
+            $query->select('fileid', 'mtime')
+                ->from('memories_livephoto')
+                ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
+            ;
+            $cursor = $query->executeQuery();
+            $prevRow = $cursor->fetch();
+            $cursor->closeCursor();
+        }
+
         if ($prevRow && !$force && (int) $prevRow['mtime'] === $mtime) {
             return 1;
         }
@@ -91,11 +106,19 @@ class TimelineWrite
         } catch (\Exception $e) {
         }
 
+        // Hand off if live photo video part
+        if ($isvideo && $this->livePhoto->isVideoPart($exif)) {
+            $this->livePhoto->processVideoPart($file, $exif);
+
+            return 2;
+        }
+
         // Get more parameters
         $dateTaken = Exif::getDateTaken($file, $exif);
         $dayId = floor($dateTaken / 86400);
         $dateTaken = gmdate('Y-m-d H:i:s', $dateTaken);
         [$w, $h] = Exif::getDimensions($exif);
+        $liveid = $this->livePhoto->getLivePhotoId($exif);
 
         // Video parameters
         $videoDuration = 0;
@@ -103,11 +126,17 @@ class TimelineWrite
             $videoDuration = round($exif['Duration'] ?? $exif['TrackDuration'] ?? 0);
         }
 
-        // Truncate any fields >2048 chars
+        // Clean up EXIF to keep only useful metadata
         foreach ($exif as $key => &$value) {
+            // Truncate any fields > 2048 chars
             if (\is_string($value) && \strlen($value) > 2048) {
                 $exif[$key] = substr($value, 0, 2048);
             }
+
+            // These are huge and not needed
+            if (str_starts_with($key, 'Nikon') || str_starts_with($key, 'QuickTime')) {
+                unset($exif[$key]);
+            }
         }
 
         // Store JSON string
@@ -134,6 +163,7 @@ class TimelineWrite
                 ->set('w', $query->createNamedParameter($w, IQueryBuilder::PARAM_INT))
                 ->set('h', $query->createNamedParameter($h, IQueryBuilder::PARAM_INT))
                 ->set('exif', $query->createNamedParameter($exifJson, IQueryBuilder::PARAM_STR))
+                ->set('liveid', $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR))
                 ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
             ;
             $query->executeStatement();
@@ -152,6 +182,7 @@ class TimelineWrite
                         'w' => $query->createNamedParameter($w, IQueryBuilder::PARAM_INT),
                         'h' => $query->createNamedParameter($h, IQueryBuilder::PARAM_INT),
                         'exif' => $query->createNamedParameter($exifJson, IQueryBuilder::PARAM_STR),
+                        'liveid' => $query->createNamedParameter($liveid, IQueryBuilder::PARAM_STR),
                     ])
                 ;
                 $query->executeStatement();

lib/Exif.php

@@ -262,6 +262,28 @@ class Exif
         }
     }
 
+    public static function getBinaryExifProp(string $path, string $prop)
+    {
+        $pipes = [];
+        $proc = proc_open(array_merge(self::getExiftool(), [$prop, '-n', '-b', $path]), [
+            1 => ['pipe', 'w'],
+            2 => ['pipe', 'w'],
+        ], $pipes);
+        stream_set_blocking($pipes[1], false);
+
+        try {
+            return self::readOrTimeout($pipes[1], 5000);
+        } catch (\Exception $ex) {
+            error_log("Exiftool timeout: [{$path}]");
+
+            throw new \Exception('Could not read from Exiftool');
+        } finally {
+            fclose($pipes[1]);
+            fclose($pipes[2]);
+            proc_terminate($proc);
+        }
+    }
+
     /** Get path to exiftool binary */
     private static function getExiftool()
     {

lib/Migration/Version400800Date20221122105007.php

@@ -0,0 +1,98 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2022 Your name <your@email.com>
+ * @author Your name <your@email.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Memories\Migration;
+
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version400800Date20221122105007 extends SimpleMigrationStep
+{
+    /**
+     * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+     */
+    public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+    }
+
+    /**
+     * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+     */
+    public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options): ?ISchemaWrapper
+    {
+        /** @var ISchemaWrapper $schema */
+        $schema = $schemaClosure();
+
+        $table = $schema->getTable('memories');
+
+        if (!$table->hasColumn('liveid')) {
+            $table->addColumn('liveid', 'string', [
+                'notnull' => false,
+                'length' => 256,
+                'default' => '',
+            ]);
+        }
+
+        // Live photos table
+        if (!$schema->hasTable('memories_livephoto')) {
+            $table = $schema->createTable('memories_livephoto');
+
+            $table->addColumn('id', 'integer', [
+                'autoincrement' => true,
+                'notnull' => true,
+            ]);
+
+            $table->addColumn('liveid', 'string', [
+                'notnull' => true,
+                'length' => 256,
+            ]);
+
+            $table->addColumn('fileid', Types::BIGINT, [
+                'notnull' => true,
+                'length' => 20,
+            ]);
+
+            $table->addColumn('mtime', Types::INTEGER, [
+                'notnull' => true,
+            ]);
+
+            $table->setPrimaryKey(['id']);
+            $table->addIndex(['liveid'], 'memories_lp_liveid_index');
+            $table->addUniqueIndex(['fileid'], 'memories_lp_fileid_index');
+        }
+
+        return $schema;
+    }
+
+    /**
+     * @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+     */
+    public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void
+    {
+    }
+}

scripts/get-exiftool.sh

@@ -17,7 +17,7 @@ mv "exiftool-$exifver" exiftool
 rm -rf *.zip exiftool/t exiftool/html
 chmod 755 exiftool/exiftool
 
-govod="0.0.15"
+govod="0.0.16"
 wget -q "https://github.com/pulsejet/go-vod/releases/download/$govod/go-vod-amd64"
 wget -q "https://github.com/pulsejet/go-vod/releases/download/$govod/go-vod-aarch64"
 chmod 755 go-vod-*

src/App.vue

@@ -356,4 +356,39 @@ aside.app-sidebar {
   height: 100%;
   display: block;
 }
+
+:root {
+  --livephoto-img-transition: opacity 0.4s linear, transform 0.3s ease-in-out;
+}
+
+// Live photo transitions
+.memories-livephoto {
+  position: relative;
+  overflow: hidden;
+  contain: strict;
+
+  img,
+  video {
+    position: absolute;
+    padding: inherit;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    display: block;
+    transition: var(--livephoto-img-transition);
+  }
+
+  video,
+  &.playing.canplay img {
+    opacity: 0;
+  }
+  img,
+  &.playing.canplay video {
+    opacity: 1;
+  }
+  &.playing.canplay img {
+    transform: scale(1.05);
+  }
+}
 </style>

src/components/PsLivePhoto.ts

@@ -0,0 +1,76 @@
+import PhotoSwipe from "photoswipe";
+import * as utils from "../services/Utils";
+
+function isLiveContent(content): boolean {
+  return Boolean(content?.data?.photo?.liveid);
+}
+
+class LivePhotoContentSetup {
+  constructor(lightbox: PhotoSwipe, private options) {
+    this.initLightboxEvents(lightbox);
+  }
+
+  initLightboxEvents(lightbox: PhotoSwipe) {
+    lightbox.on("contentLoad", this.onContentLoad.bind(this));
+    lightbox.on("contentActivate", this.onContentActivate.bind(this));
+    lightbox.on("contentDeactivate", this.onContentDeactivate.bind(this));
+    lightbox.on("contentAppend", this.onContentAppend.bind(this));
+  }
+
+  onContentLoad(e) {
+    const content = e.content;
+    if (!isLiveContent(content)) return;
+
+    e.preventDefault();
+    if (content.element) return;
+
+    const photo = content?.data?.photo;
+
+    const video = document.createElement("video");
+    video.muted = true;
+    video.autoplay = false;
+    video.playsInline = true;
+    video.preload = "none";
+    video.src = utils.getLivePhotoVideoUrl(photo);
+
+    const div = document.createElement("div");
+    div.className = "memories-livephoto";
+    div.appendChild(video);
+    content.element = div;
+
+    utils.setupLivePhotoHooks(video);
+
+    const img = document.createElement("img");
+    img.src = content.data.src;
+    img.onload = () => content.onLoaded();
+    div.appendChild(img);
+
+    content.element = div;
+  }
+
+  onContentActivate({ content }) {
+    if (isLiveContent(content) && content.element) {
+      const video = content.element.querySelector("video");
+      if (video) {
+        video.currentTime = 0;
+        video.play();
+      }
+    }
+  }
+
+  onContentDeactivate({ content }) {
+    if (isLiveContent(content) && content.element) {
+      content.element.querySelector("video")?.pause();
+    }
+  }
+
+  onContentAppend(e) {
+    if (isLiveContent(e.content)) {
+      e.preventDefault();
+      e.content.isAttached = true;
+      e.content.appendImage();
+    }
+  }
+}
+
+export default LivePhotoContentSetup;

src/components/Timeline.vue

@@ -1213,7 +1213,7 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
    */
   async deleteFromViewWithAnimation(delPhotos: IPhoto[]) {
     // Only keep photos with day
-    delPhotos = delPhotos.filter((p) => p.d);
+    delPhotos = delPhotos.filter((p) => p?.d);
     if (delPhotos.length === 0) return;
 
     // Get all days that need to be updatd

src/components/Viewer.vue

@@ -13,7 +13,12 @@
       @close="editorOpen = false"
     />
 
-    <div class="inner" ref="inner" v-show="!editorOpen">
+    <div
+      class="inner"
+      ref="inner"
+      v-show="!editorOpen"
+      @pointermove.passive="setUiVisible"
+    >
       <div class="top-bar" v-if="photoswipe" :class="{ showControls }">
         <NcActions
           :inline="numInlineActions"
@@ -81,6 +86,17 @@
               <DownloadIcon :size="24" />
             </template>
           </NcActionButton>
+          <NcActionButton
+            v-if="currentPhoto?.liveid"
+            :aria-label="t('memories', 'Download Video')"
+            @click="downloadCurrentLiveVideo"
+            :close-after-click="true"
+          >
+            {{ t("memories", "Download Video") }}
+            <template #icon>
+              <DownloadIcon :size="24" />
+            </template>
+          </NcActionButton>
           <NcActionButton
             v-if="!routeIsPublic"
             :aria-label="t('memories', 'View in folder')"
@@ -120,6 +136,7 @@ import PhotoSwipe, { PhotoSwipeOptions } from "photoswipe";
 import "photoswipe/style.css";
 
 import PsVideo from "./PsVideo";
+import PsLivePhoto from "./PsLivePhoto";
 
 import ShareIcon from "vue-material-design-icons/ShareVariant.vue";
 import DeleteIcon from "vue-material-design-icons/TrashCanOutline.vue";
@@ -161,6 +178,9 @@ export default class Viewer extends Mixins(GlobalMixin) {
   private sidebarWidth = 400;
   private outerWidth = "100vw";
 
+  /** User interaction detection */
+  private activityTimer = 0;
+
   /** Base dialog */
   private photoswipe: PhotoSwipe | null = null;
 
@@ -237,13 +257,40 @@ export default class Viewer extends Mixins(GlobalMixin) {
   }
 
   /** Event on file changed */
-  handleFileUpdated({ fileid }: { fileid: number }) {
+  private handleFileUpdated({ fileid }: { fileid: number }) {
     if (this.currentPhoto && this.currentPhoto.fileid === fileid) {
       this.currentPhoto.etag += "_";
       this.photoswipe.refreshSlideContent(this.currIndex);
     }
   }
 
+  /** User interacted with the page with mouse */
+  private setUiVisible(evt: any) {
+    clearTimeout(this.activityTimer);
+    if (evt) {
+      // If directly triggered, always update ui visibility
+      // If triggered through a pointer event, only update if this is not
+      // a touch event (i.e. a mouse move).
+      // On touch devices, tapAction directly handles the ui visibility
+      // through Photoswipe.
+      const isPointer = evt instanceof PointerEvent;
+      const isMouse = isPointer && evt.pointerType !== "touch";
+      if (this.isOpen && (!isPointer || isMouse)) {
+        this.photoswipe?.template?.classList.add("pswp--ui-visible");
+
+        if (isMouse) {
+          this.activityTimer = window.setTimeout(() => {
+            if (this.isOpen) {
+              this.photoswipe?.template?.classList.remove("pswp--ui-visible");
+            }
+          }, 2000);
+        }
+      }
+    } else {
+      this.photoswipe?.template?.classList.remove("pswp--ui-visible");
+    }
+  }
+
   /** Create the base photoswipe object */
   private async createBase(args: PhotoSwipeOptions) {
     this.show = true;
@@ -320,7 +367,6 @@ export default class Viewer extends Mixins(GlobalMixin) {
     this.photoswipe.on("openingAnimationStart", () => {
       this.isOpen = true;
       this.fullyOpened = false;
-      this.showControls = true;
       if (this.sidebarOpen) {
         this.openSidebar();
       }
@@ -331,7 +377,7 @@ export default class Viewer extends Mixins(GlobalMixin) {
     this.photoswipe.on("close", () => {
       this.isOpen = false;
       this.fullyOpened = false;
-      this.showControls = false;
+      this.setUiVisible(false);
       this.hideSidebar();
       this.setRouteHash(undefined);
       this.updateTitle(undefined);
@@ -344,7 +390,6 @@ export default class Viewer extends Mixins(GlobalMixin) {
       this.show = false;
       this.isOpen = false;
       this.fullyOpened = false;
-      this.showControls = false;
       this.photoswipe = null;
       this.list = [];
       this.days.clear();
@@ -353,11 +398,6 @@ export default class Viewer extends Mixins(GlobalMixin) {
       this.globalAnchor = -1;
     });
 
-    // toggle-controls
-    this.photoswipe.on("tapAction", () => {
-      this.showControls = !this.showControls;
-    });
-
     // Update vue route for deep linking
     this.photoswipe.on("slideActivate", (e) => {
       this.currIndex = this.photoswipe.currIndex;
@@ -367,6 +407,22 @@ export default class Viewer extends Mixins(GlobalMixin) {
       globalThis.currentViewerPhoto = photo;
     });
 
+    // Show and hide controls
+    this.photoswipe.on("uiRegister", (e) => {
+      if (this.photoswipe?.template) {
+        new MutationObserver((mutations) => {
+          mutations.forEach((mutationRecord) => {
+            this.showControls = (<HTMLElement>(
+              mutationRecord.target
+            ))?.classList.contains("pswp--ui-visible");
+          });
+        }).observe(this.photoswipe.template, {
+          attributes: true,
+          attributeFilter: ["class"],
+        });
+      }
+    });
+
     // Video support
     new PsVideo(this.photoswipe, {
       videoAttributes: { controls: "", playsinline: "", preload: "none" },
@@ -374,6 +430,9 @@ export default class Viewer extends Mixins(GlobalMixin) {
       preventDragOffset: 40,
     });
 
+    // Live photo support
+    new PsLivePhoto(this.photoswipe, {});
+
     return this.photoswipe;
   }
 
@@ -609,7 +668,10 @@ export default class Viewer extends Mixins(GlobalMixin) {
   }
 
   get canEdit() {
-    return this.currentPhoto?.mimetype?.startsWith("image/");
+    return (
+      this.currentPhoto?.mimetype?.startsWith("image/") &&
+      !this.currentPhoto.liveid
+    );
   }
 
   private openEditor() {
@@ -747,6 +809,13 @@ export default class Viewer extends Mixins(GlobalMixin) {
     dav.downloadFilesByPhotos([photo]);
   }
 
+  /** Download live part of current video */
+  private async downloadCurrentLiveVideo() {
+    const photo = this.currentPhoto;
+    if (!photo) return;
+    window.location.href = utils.getLivePhotoVideoUrl(photo);
+  }
+
   /** Open the sidebar */
   private async openSidebar(photo?: IPhoto) {
     const fInfo = await dav.getFiles([photo || this.currentPhoto]);
@@ -852,6 +921,10 @@ export default class Viewer extends Mixins(GlobalMixin) {
 .inner,
 .inner :deep .pswp {
   width: inherit;
+
+  .pswp__top-bar {
+    background: linear-gradient(0deg, transparent, rgba(0, 0, 0, 0.3));
+  }
 }
 
 :deep .video-js .vjs-big-play-button {

src/components/frame/Photo.vue

@@ -22,10 +22,21 @@
       <Video :size="22" />
     </div>
 
+    <div
+      class="livephoto"
+      @mouseenter.passive="playVideo"
+      @mouseleave.passive="stopVideo"
+    >
+      <LivePhoto :size="22" v-if="data.liveid" />
+    </div>
+
     <Star :size="22" v-if="data.flag & c.FLAG_IS_FAVORITE" />
 
     <div
       class="img-outer fill-block"
+      :class="{
+        'memories-livephoto': data.liveid,
+      }"
       @contextmenu="contextmenu"
       @pointerdown.passive="$emit('pointerdown', $event)"
       @touchstart.passive="$emit('touchstart', $event)"
@@ -42,7 +53,15 @@
         @load="load"
         @error="error"
       />
-      <div class="overlay" />
+      <video
+        ref="video"
+        v-if="videoUrl"
+        :src="videoUrl"
+        preload="none"
+        muted
+        playsinline
+      />
+      <div class="overlay fill-block" />
     </div>
   </div>
 </template>
@@ -59,12 +78,14 @@ import errorsvg from "../../assets/error.svg";
 import CheckCircle from "vue-material-design-icons/CheckCircle.vue";
 import Star from "vue-material-design-icons/Star.vue";
 import Video from "vue-material-design-icons/PlayCircleOutline.vue";
+import LivePhoto from "vue-material-design-icons/MotionPlayOutline.vue";
 
 @Component({
   components: {
     CheckCircle,
     Video,
     Star,
+    LivePhoto,
   },
 })
 export default class Photo extends Mixins(GlobalMixin) {
@@ -95,6 +116,12 @@ export default class Photo extends Mixins(GlobalMixin) {
   mounted() {
     this.hasFaceRect = false;
     this.refresh();
+
+    // Setup video hooks
+    const video = this.$refs.video as HTMLVideoElement;
+    if (video) {
+      utils.setupLivePhotoHooks(video);
+    }
   }
 
   get videoDuration() {
@@ -104,6 +131,12 @@ export default class Photo extends Mixins(GlobalMixin) {
     return null;
   }
 
+  get videoUrl() {
+    if (this.data.liveid) {
+      return utils.getLivePhotoVideoUrl(this.data);
+    }
+  }
+
   async refresh() {
     this.src = await this.getSrc();
   }
@@ -204,6 +237,23 @@ export default class Photo extends Mixins(GlobalMixin) {
       e.stopPropagation();
     }
   }
+
+  /** Start preview video */
+  playVideo() {
+    if (this.$refs.video && !(this.data.flag & this.c.FLAG_SELECTED)) {
+      const video = this.$refs.video as HTMLVideoElement;
+      video.currentTime = 0;
+      video.play();
+    }
+  }
+
+  /** Stop preview video */
+  stopVideo() {
+    if (this.$refs.video) {
+      const video = this.$refs.video as HTMLVideoElement;
+      video.pause();
+    }
+  }
 }
 </script>
 
@@ -273,14 +323,16 @@ $icon-size: $icon-half-size * 2;
   }
 }
 .video,
-.star-icon {
+.star-icon,
+.livephoto {
   position: absolute;
   z-index: 100;
   pointer-events: none;
   transition: transform 0.15s ease;
   filter: invert(1) brightness(100);
 }
-.video {
+.video,
+.livephoto {
   position: absolute;
   top: var(--icon-dist);
   right: var(--icon-dist);
@@ -298,6 +350,9 @@ $icon-size: $icon-half-size * 2;
     margin-right: 3px;
   }
 }
+.livephoto {
+  pointer-events: auto;
+}
 .star-icon {
   bottom: var(--icon-dist);
   left: var(--icon-dist);
@@ -308,6 +363,7 @@ $icon-size: $icon-half-size * 2;
 
 /* Actual image */
 div.img-outer {
+  position: relative;
   box-sizing: border-box;
   padding: 0;
 
@@ -331,7 +387,7 @@ div.img-outer {
     -webkit-tap-highlight-color: transparent;
     -webkit-touch-callout: none;
     user-select: none;
-    transition: border-radius 0.1s ease-in;
+    transition: border-radius 0.1s ease-in, var(--livephoto-img-transition);
 
     .p-outer.placeholder > & {
       display: none;
@@ -341,11 +397,16 @@ div.img-outer {
     }
   }
 
-  & > .overlay {
+  > video {
     pointer-events: none;
-    width: 100%;
+    object-fit: cover;
-    height: 100%;
+  }
-    transform: translateY(-100%); // very weird stuff
+
+  > .overlay {
+    pointer-events: none;
+    position: absolute;
+    top: 0;
+    left: 0;
     background: linear-gradient(180deg, rgba(0, 0, 0, 0.2) 0%, transparent 30%);
 
     display: none;

src/services/Utils.ts

@@ -1,5 +1,6 @@
 import { getCanonicalLocale } from "@nextcloud/l10n";
 import { getCurrentUser } from "@nextcloud/auth";
+import { generateUrl } from "@nextcloud/router";
 import { loadState } from "@nextcloud/initial-state";
 import { IPhoto } from "../types";
 import moment from "moment";
@@ -236,6 +237,32 @@ export function getFolderRoutePath(basePath: string) {
   return path;
 }
 
+/**
+ * Get URL to live photo video part
+ */
+export function getLivePhotoVideoUrl(p: IPhoto) {
+  return generateUrl(
+    `/apps/memories/api/video/livephoto/${p.fileid}?etag=${p.etag}&liveid=${p.liveid}`
+  );
+}
+
+/**
+ * Set up hooks to set classes on parent element for live photo
+ * @param video Video element
+ */
+export function setupLivePhotoHooks(video: HTMLVideoElement) {
+  const div = video.closest(".memories-livephoto") as HTMLDivElement;
+  video.onplay = () => {
+    div.classList.add("playing");
+  };
+  video.oncanplay = () => {
+    div.classList.add("canplay");
+  };
+  video.onended = video.onpause = () => {
+    div.classList.remove("playing");
+  };
+}
+
 /**
  * Get route hash for viewer for photo
  */

src/services/dav/base.ts

@@ -1,10 +1,13 @@
 import { getCurrentUser } from "@nextcloud/auth";
 import { showError } from "@nextcloud/dialogs";
 import { translate as t } from "@nextcloud/l10n";
+import axios from "@nextcloud/axios";
+
 import { IFileInfo, IPhoto } from "../../types";
-import client from "../DavClient";
 import { genFileInfo } from "../FileUtils";
 import { getAlbumFileInfos } from "./albums";
+import * as utils from "../Utils";
+import client from "../DavClient";
 
 export const props = `
     <oc:fileid />
@@ -195,10 +198,32 @@ export async function* deletePhotos(photos: IPhoto[]) {
 
   const fileIdsSet = new Set(photos.map((p) => p.fileid));
 
+  // Get live photo data
+  const livePhotos = (
+    await Promise.all(
+      photos
+        .filter((p) => p.liveid && !p.liveid.startsWith("self__"))
+        .map(async (p) => {
+          const url = utils.getLivePhotoVideoUrl(p) + "&format=json";
+          try {
+            const response = await axios.get(url);
+            const data = response.data;
+            fileIdsSet.add(data.fileid);
+            return {
+              fileid: data.fileid,
+            } as IPhoto;
+          } catch (error) {
+            console.error(error);
+            return null;
+          }
+        })
+    )
+  ).filter((p) => p !== null) as IPhoto[];
+
   // Get files data
   let fileInfos: IFileInfo[] = [];
   try {
-    fileInfos = await getFiles(photos);
+    fileInfos = await getFiles(photos.concat(livePhotos));
   } catch (e) {
     console.error("Failed to get file info for files to delete", photos, e);
     showError(t("memories", "Failed to delete files."));

src/types.ts

@@ -57,6 +57,8 @@ export type IPhoto = {
   w?: number;
   /** Height of full image */
   h?: number;
+  /** Live photo identifier */
+  liveid?: string;
 
   /** Grid display width px */
   dispW?: number;