From ef068ac2b30bd5f9ed0f299dbc75eb3ace000042 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 29 Aug 2021 05:20:23 +0200 Subject: [PATCH] Support animated images fixes #461 --- CMakeLists.txt | 2 + resources/qml/delegates/ImageMessage.qml | 69 ++++---- resources/qml/delegates/MessageDelegate.qml | 2 + .../qml/delegates/PlayableMediaMessage.qml | 4 +- src/timeline/TimelineViewManager.cpp | 2 + src/ui/MxcAnimatedImage.cpp | 164 ++++++++++++++++++ src/ui/MxcAnimatedImage.h | 79 +++++++++ src/ui/MxcMediaProxy.cpp | 4 +- 8 files changed, 293 insertions(+), 33 deletions(-) create mode 100644 src/ui/MxcAnimatedImage.cpp create mode 100644 src/ui/MxcAnimatedImage.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f24cffef..fb83ae2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -311,6 +311,7 @@ set(SRC_FILES src/ui/InfoMessage.cpp src/ui/Label.cpp src/ui/LoadingIndicator.cpp + src/ui/MxcAnimatedImage.cpp src/ui/MxcMediaProxy.cpp src/ui/NhekoCursorShape.cpp src/ui/NhekoDropArea.cpp @@ -522,6 +523,7 @@ qt5_wrap_cpp(MOC_HEADERS src/ui/InfoMessage.h src/ui/Label.h src/ui/LoadingIndicator.h + src/ui/MxcAnimatedImage.h src/ui/MxcMediaProxy.h src/ui/Menu.h src/ui/NhekoCursorShape.h diff --git a/resources/qml/delegates/ImageMessage.qml b/resources/qml/delegates/ImageMessage.qml index b432018c..f39176b3 100644 --- a/resources/qml/delegates/ImageMessage.qml +++ b/resources/qml/delegates/ImageMessage.qml @@ -14,6 +14,7 @@ Item { required property string body required property string filename required property bool isReply + required property string eventId property double tempWidth: Math.min(parent ? parent.width : undefined, originalWidth < 1 ? 200 : originalWidth) property double tempHeight: tempWidth * proportionalHeight property double divisor: isReply ? 5 : 3 @@ -37,6 +38,7 @@ Item { Image { id: img + visible: !mxcimage.loaded anchors.fill: parent source: url.replace("mxc://", "image://MxcImage/") asynchronous: true @@ -53,38 +55,47 @@ Item { gesturePolicy: TapHandler.ReleaseWithinBounds } - HoverHandler { - id: mouseArea + } + + MxcAnimatedImage { + id: mxcimage + + visible: loaded + anchors.fill: parent + roomm: room + eventId: parent.eventId + } + + HoverHandler { + id: mouseArea + } + + Item { + id: overlay + + anchors.fill: parent + visible: mouseArea.hovered + + Rectangle { + id: container + + width: parent.width + implicitHeight: imgcaption.implicitHeight + anchors.bottom: overlay.bottom + color: Nheko.colors.window + opacity: 0.75 } - Item { - id: overlay - - anchors.fill: parent - visible: mouseArea.hovered - - Rectangle { - id: container - - width: parent.width - implicitHeight: imgcaption.implicitHeight - anchors.bottom: overlay.bottom - color: Nheko.colors.window - opacity: 0.75 - } - - Text { - id: imgcaption - - anchors.fill: container - elide: Text.ElideMiddle - horizontalAlignment: Text.AlignHCenter - verticalAlignment: Text.AlignVCenter - // See this MSC: https://github.com/matrix-org/matrix-doc/pull/2530 - text: filename ? filename : body - color: Nheko.colors.text - } + Text { + id: imgcaption + anchors.fill: container + elide: Text.ElideMiddle + horizontalAlignment: Text.AlignHCenter + verticalAlignment: Text.AlignVCenter + // See this MSC: https://github.com/matrix-org/matrix-doc/pull/2530 + text: filename ? filename : body + color: Nheko.colors.text } } diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml index fde43559..4086a1a8 100644 --- a/resources/qml/delegates/MessageDelegate.qml +++ b/resources/qml/delegates/MessageDelegate.qml @@ -102,6 +102,7 @@ Item { body: d.body filename: d.filename isReply: d.isReply + eventId: d.eventId } } @@ -118,6 +119,7 @@ Item { body: d.body filename: d.filename isReply: d.isReply + eventId: d.eventId } } diff --git a/resources/qml/delegates/PlayableMediaMessage.qml b/resources/qml/delegates/PlayableMediaMessage.qml index 94c058ab..eb6db291 100644 --- a/resources/qml/delegates/PlayableMediaMessage.qml +++ b/resources/qml/delegates/PlayableMediaMessage.qml @@ -185,9 +185,9 @@ Rectangle { } } onStateChanged: { - if (state == MxcMedia.StoppedState) { + if (state == MxcMedia.StoppedState) button.state = "stopped"; - } + } } diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp index f42ec02f..681cbe09 100644 --- a/src/timeline/TimelineViewManager.cpp +++ b/src/timeline/TimelineViewManager.cpp @@ -35,6 +35,7 @@ #include "dialogs/ImageOverlay.h" #include "emoji/EmojiModel.h" #include "emoji/Provider.h" +#include "ui/MxcAnimatedImage.h" #include "ui/MxcMediaProxy.h" #include "ui/NhekoCursorShape.h" #include "ui/NhekoDropArea.h" @@ -177,6 +178,7 @@ TimelineViewManager::TimelineViewManager(CallManager *callManager, ChatPage *par qmlRegisterType("im.nheko", 1, 0, "DelegateChooser"); qmlRegisterType("im.nheko", 1, 0, "NhekoDropArea"); qmlRegisterType("im.nheko", 1, 0, "CursorShape"); + qmlRegisterType("im.nheko", 1, 0, "MxcAnimatedImage"); qmlRegisterType("im.nheko", 1, 0, "MxcMedia"); qmlRegisterUncreatableType( "im.nheko", 1, 0, "DeviceVerificationFlow", "Can't create verification flow from QML!"); diff --git a/src/ui/MxcAnimatedImage.cpp b/src/ui/MxcAnimatedImage.cpp new file mode 100644 index 00000000..cfc03827 --- /dev/null +++ b/src/ui/MxcAnimatedImage.cpp @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "MxcAnimatedImage.h" + +#include +#include +#include +#include +#include +#include + +#include "EventAccessors.h" +#include "Logging.h" +#include "MatrixClient.h" +#include "timeline/TimelineModel.h" + +void +MxcAnimatedImage::startDownload() +{ + if (!room_) + return; + if (eventId_.isEmpty()) + return; + + auto event = room_->eventById(eventId_); + if (!event) { + nhlog::ui()->error("Failed to load media for event {}, event not found.", + eventId_.toStdString()); + return; + } + + QByteArray mimeType = QString::fromStdString(mtx::accessors::mimetype(*event)).toUtf8(); + + animatable_ = QMovie::supportedFormats().contains(mimeType.split('/').back()); + animatableChanged(); + + if (!animatable_) + return; + + QString mxcUrl = QString::fromStdString(mtx::accessors::url(*event)); + QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event)); + + auto encryptionInfo = mtx::accessors::file(*event); + + // If the message is a link to a non mxcUrl, don't download it + if (!mxcUrl.startsWith("mxc://")) { + return; + } + + QString suffix = QMimeDatabase().mimeTypeForName(mimeType).preferredSuffix(); + + const auto url = mxcUrl.toStdString(); + const auto name = QString(mxcUrl).remove("mxc://"); + QFileInfo filename(QString("%1/media_cache/media/%2.%3") + .arg(QStandardPaths::writableLocation(QStandardPaths::CacheLocation)) + .arg(name) + .arg(suffix)); + if (QDir::cleanPath(name) != name) { + nhlog::net()->warn("mxcUrl '{}' is not safe, not downloading file", url); + return; + } + + QDir().mkpath(filename.path()); + + QPointer self = this; + + auto processBuffer = [this, mimeType, encryptionInfo, self](QIODevice &device) { + if (!self) + return; + + if (buffer.isOpen()) { + movie.stop(); + movie.setDevice(nullptr); + buffer.close(); + } + + if (encryptionInfo) { + QByteArray ba = device.readAll(); + std::string temp(ba.constData(), ba.size()); + temp = mtx::crypto::to_string( + mtx::crypto::decrypt_file(temp, encryptionInfo.value())); + buffer.setData(temp.data(), temp.size()); + } else { + buffer.setData(device.readAll()); + } + buffer.open(QIODevice::ReadOnly); + buffer.reset(); + + QTimer::singleShot(0, this, [this, mimeType] { + nhlog::ui()->info("Playing movie with size: {}, {}", + buffer.bytesAvailable(), + buffer.isOpen()); + movie.setFormat(mimeType); + movie.setDevice(&buffer); + movie.start(); + emit loadedChanged(); + }); + }; + + if (filename.isReadable()) { + QFile f(filename.filePath()); + if (f.open(QIODevice::ReadOnly)) { + processBuffer(f); + return; + } + } + + http::client()->download( + url, + [filename, url, processBuffer](const std::string &data, + const std::string &, + const std::string &, + mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to retrieve media {}: {} {}", + url, + err->matrix_error.error, + static_cast(err->status_code)); + return; + } + + try { + QFile file(filename.filePath()); + + if (!file.open(QIODevice::WriteOnly)) + return; + + QByteArray ba(data.data(), (int)data.size()); + file.write(ba); + file.close(); + + QBuffer buf(&ba); + buf.open(QBuffer::ReadOnly); + processBuffer(buf); + } catch (const std::exception &e) { + nhlog::ui()->warn("Error while saving file to: {}", e.what()); + } + }); +} + +QSGNode * +MxcAnimatedImage::updatePaintNode(QSGNode *oldNode, QQuickItem::UpdatePaintNodeData *) +{ + imageDirty = false; + QSGImageNode *n = static_cast(oldNode); + if (!n) + n = window()->createImageNode(); + + // n->setTexture(nullptr); + auto img = movie.currentImage(); + if (!img.isNull()) + n->setTexture(window()->createTextureFromImage(img)); + else + return nullptr; + + n->setSourceRect(img.rect()); + n->setRect(QRect(0, 0, width(), height())); + n->setFiltering(QSGTexture::Linear); + n->setMipmapFiltering(QSGTexture::Linear); + + return n; +} diff --git a/src/ui/MxcAnimatedImage.h b/src/ui/MxcAnimatedImage.h new file mode 100644 index 00000000..7b9502e0 --- /dev/null +++ b/src/ui/MxcAnimatedImage.h @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include + +class TimelineModel; + +// This is an AnimatedImage, that can draw encrypted images +class MxcAnimatedImage : public QQuickItem +{ + Q_OBJECT + Q_PROPERTY(TimelineModel *roomm READ room WRITE setRoom NOTIFY roomChanged REQUIRED) + Q_PROPERTY(QString eventId READ eventId WRITE setEventId NOTIFY eventIdChanged) + Q_PROPERTY(bool animatable READ animatable NOTIFY animatableChanged) + Q_PROPERTY(bool loaded READ loaded NOTIFY loadedChanged) +public: + MxcAnimatedImage(QQuickItem *parent = nullptr) + : QQuickItem(parent) + { + connect(this, &MxcAnimatedImage::eventIdChanged, &MxcAnimatedImage::startDownload); + connect(this, &MxcAnimatedImage::roomChanged, &MxcAnimatedImage::startDownload); + connect(&movie, &QMovie::frameChanged, this, &MxcAnimatedImage::newFrame); + setFlag(QQuickItem::ItemHasContents); + // setAcceptHoverEvents(true); + } + + bool animatable() const { return animatable_; } + bool loaded() const { return buffer.size() > 0; } + QString eventId() const { return eventId_; } + TimelineModel *room() const { return room_; } + void setEventId(QString newEventId) + { + if (eventId_ != newEventId) { + eventId_ = newEventId; + emit eventIdChanged(); + } + } + void setRoom(TimelineModel *room) + { + if (room_ != room) { + room_ = room; + emit roomChanged(); + } + } + + QSGNode *updatePaintNode(QSGNode *oldNode, + QQuickItem::UpdatePaintNodeData *updatePaintNodeData) override; + +signals: + void roomChanged(); + void eventIdChanged(); + void animatableChanged(); + void loadedChanged(); + +private slots: + void startDownload(); + void newFrame(int frame) + { + currentFrame = frame; + imageDirty = true; + update(); + } + +private: + TimelineModel *room_ = nullptr; + QString eventId_; + QString filename_; + bool animatable_ = false; + QBuffer buffer; + QMovie movie; + int currentFrame = 0; + bool imageDirty = true; +}; diff --git a/src/ui/MxcMediaProxy.cpp b/src/ui/MxcMediaProxy.cpp index c1de2c31..dc65de7c 100644 --- a/src/ui/MxcMediaProxy.cpp +++ b/src/ui/MxcMediaProxy.cpp @@ -91,11 +91,11 @@ MxcMediaProxy::startDownload() buffer.open(QIODevice::ReadOnly); buffer.reset(); - QTimer::singleShot(0, this, [this, self, filename] { + QTimer::singleShot(0, this, [this, filename] { nhlog::ui()->info("Playing buffer with size: {}, {}", buffer.bytesAvailable(), buffer.isOpen()); - self->setMedia(QMediaContent(filename.fileName()), &buffer); + this->setMedia(QMediaContent(filename.fileName()), &buffer); emit loadedChanged(); }); };