Hardware accelerated video decoding

.github/workflows/pipeline.yml

@@ -144,7 +144,7 @@ jobs:
     - name: Install Dependencies Linux
       run: |
         sudo apt-get update
-        sudo apt-get install nasm libx11-dev libxft-dev libxext-dev libwayland-dev libxkbcommon-dev libegl1-mesa-dev libibus-1.0-dev libxrandr-dev libltdl-dev mono-complete autoconf autoconf-archive automake libtool
+        sudo apt-get install nasm libx11-dev libxft-dev libxext-dev libwayland-dev libxkbcommon-dev libegl1-mesa-dev libibus-1.0-dev libxrandr-dev libltdl-dev mono-complete autoconf autoconf-archive automake libtool libdrm-dev
       if: matrix.os_name == 'linux' || matrix.os_name == 'android'
     - name: Install Dependencies Mac
       run: brew install nasm mono

CMakeLists.txt

@@ -407,6 +407,7 @@ set(Impacto_Header
     src/renderer/renderer.h
     src/renderer/window.h
     src/renderer/yuvframe.h
+    src/renderer/nv12frame.h
 
     src/data/savesystem.h
     src/data/tipssystem.h
@@ -1058,13 +1059,24 @@ if (NOT DEFINED IMPACTO_DISABLE_LIBASS)
     list(APPEND Impacto_Src src/subtitle/ass/subtitlerenderer.cpp)
     list(APPEND Impacto_Header src/subtitle/ass/subtitlerenderer.h)
 
-    pkg_check_modules(LIBASS REQUIRED IMPORTED_TARGET GLOBAL libass)
-    list(APPEND Impacto_Include_Dirs ${LIBASS_INCLUDE_DIRS})
+    pkg_check_modules(ass REQUIRED IMPORTED_TARGET GLOBAL libass)
+    # Libass links libc++ which overrides the static stllib linkage 
+    if(ANDROID)
+        get_target_property(ASS_LINK_LIBS PkgConfig::ass INTERFACE_LINK_LIBRARIES)
+        set(CLEANED_ASS_LINK_LIBS "")
+        foreach(LIB_PATH IN LISTS ASS_LINK_LIBS)
+            get_filename_component(LIB_NAME "${LIB_PATH}" NAME)
+
+            # Check if the filename starts with "libc++." or "libc++_"
+            if(NOT ("${LIB_NAME}" MATCHES "^libc\\+\\+\\." OR "${LIB_NAME}" MATCHES "^libc\\+\\+_"))
+                list(APPEND CLEANED_ASS_LINK_LIBS "${LIB_PATH}")
+            endif()
+        endforeach()
+
+        set_target_properties(PkgConfig::ass PROPERTIES INTERFACE_LINK_LIBRARIES "${CLEANED_ASS_LINK_LIBS}")
+    endif()
     list(APPEND Impacto_Libs
-        ${LIBASS_LINK_LIBRARIES}
-    )
-    list(APPEND Impacto_Compile_Options
-        ${LIBASS_CFLAGS_OTHER}
+        PkgConfig::ass
     )
 
 endif()
@@ -1124,6 +1136,7 @@ if (NOT DEFINED IMPACTO_DISABLE_OPENGL)
         src/renderer/opengl/shader.cpp
         src/renderer/opengl/glc.cpp
         src/renderer/opengl/yuvframe.cpp
+        src/renderer/opengl/nv12frame.cpp
         src/renderer/opengl/3d/renderable3d.cpp
         src/renderer/opengl/3d/scene.cpp
 
@@ -1135,6 +1148,7 @@ if (NOT DEFINED IMPACTO_DISABLE_OPENGL)
         src/renderer/opengl/shader.h
         src/renderer/opengl/glc.h
         src/renderer/opengl/yuvframe.h
+        src/renderer/opengl/nv12frame.h
         src/renderer/opengl/3d/renderable3d.h
         src/renderer/opengl/3d/scene.h
     )
@@ -1154,19 +1168,21 @@ if (NOT DEFINED IMPACTO_DISABLE_FFMPEG)
     )
 
     if (NOT VCPKG_TOOLCHAIN)
-        set(AVCPP_PATCH git apply ${CMAKE_CURRENT_SOURCE_DIR}/vendor/patches/avcpp.patch)
         if(NOT BUILD_SHARED_LIBS)
             set(AV_ENABLE_SHARED OFF CACHE BOOL "Enable shared library build (Off)" FORCE)            
         endif()
         set(AV_DISABLE_AVDEVICE ON CACHE INTERNAL "Disable FFMPEG AVDevice")
         set(AV_DISABLE_AVFILTER ON CACHE INTERNAL "Disable FFMPEG AVFilter")
         set(AV_BUILD_EXAMPLES OFF CACHE INTERNAL "Build AVCPP Examples")
+        set(AVCPP_PATCH git apply ${CMAKE_CURRENT_SOURCE_DIR}/portfiles/avcpp/0001-codeccontext.patch)
         FetchContent_Declare(
             avcpp
             GIT_REPOSITORY "https://github.com/h4tr3d/avcpp"
             GIT_TAG "00e29ea0c0f7036423b80553aaccacd70f90aace"
             SYSTEM
             GIT_SUBMODULES ""
+            PATCH_COMMAND ${AVCPP_PATCH}
+            UPDATE_DISCONNECTED 1
         )
 
         FetchContent_MakeAvailable(avcpp)
@@ -1219,6 +1235,7 @@ if (NOT DEFINED IMPACTO_DISABLE_VULKAN)
         src/renderer/vulkan/pipeline.cpp
         src/renderer/vulkan/utils.cpp
         src/renderer/vulkan/yuvframe.cpp
+        src/renderer/vulkan/nv12frame.cpp
         src/renderer/vulkan/3d/renderable3d.cpp
         src/renderer/vulkan/3d/scene.cpp
     )
@@ -1227,7 +1244,7 @@ if (NOT DEFINED IMPACTO_DISABLE_VULKAN)
         src/renderer/vulkan/renderer.h
         src/renderer/vulkan/pipeline.h
         src/renderer/vulkan/utils.h
-        src/renderer/vulkan/yuvframe.h
+        src/renderer/vulkan/nv12frame.h
         src/renderer/vulkan/3d/renderable3d.h
         src/renderer/vulkan/3d/scene.h
     )
@@ -1246,6 +1263,7 @@ if (NOT DEFINED IMPACTO_DISABLE_DX9)
         src/renderer/dx9/window.cpp
         src/renderer/dx9/shader.cpp
         src/renderer/dx9/yuvframe.cpp
+        src/renderer/dx9/nv12frame.cpp
         src/renderer/dx9/3d/scene.cpp
         src/renderer/dx9/3d/renderable3d.cpp
     )
@@ -1255,6 +1273,7 @@ if (NOT DEFINED IMPACTO_DISABLE_DX9)
         src/renderer/dx9/window.h
         src/renderer/dx9/shader.h
         src/renderer/dx9/yuvframe.h
+        src/renderer/dx9/nv12frame.h
         src/renderer/dx9/3d/scene.h
         src/renderer/dx9/3d/renderable3d.h
     )

CMakePresets.json

@@ -54,6 +54,7 @@
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/triplets",
+        "VCPKG_OVERLAY_PORTS": "${sourceDir}/portfiles",
         "IMPACTO_WARNINGS": "ON"
       }
     },

android/app/src/main/AndroidManifest.xml

@@ -35,14 +35,15 @@
 
     <application
         android:allowBackup="true"
+        android:appCategory="game"
         android:label="@string/app_name"
         android:icon="@mipmap/ic_launcher"
         android:roundIcon="@mipmap/ic_launcher_round"
         android:supportsRtl="true"
         android:theme="@style/AppTheme">
         <activity
             android:name=".ImpactoActivity"
-            android:configChanges="keyboardHidden|orientation"
+            android:configChanges="keyboardHidden|orientation|screenSize"
 			android:screenOrientation="landscape"
             android:label="@string/app_name"
             android:exported="true">

portfiles/avcpp/0001-codeccontext.patch

@@ -0,0 +1,157 @@
+diff --git a/src/avcpp/codeccontext.cpp b/src/avcpp/codeccontext.cpp
+index f67807e..6f3783f 100644
+--- a/src/avcpp/codeccontext.cpp
++++ b/src/avcpp/codeccontext.cpp
+@@ -90,7 +90,7 @@ int decode(AVCodecContext *avctx,
+     }
+
+     ret = avcodec_receive_frame(avctx, picture);
+-    if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
++    if (ret < 0 && ret != AVERROR_EOF)
+         return ret;
+     if (ret >= 0 && got_picture_ptr)
+         *got_picture_ptr = 1;
+@@ -203,17 +203,17 @@ VideoDecoderContext &VideoDecoderContext::operator=(VideoDecoderContext&& other)
+     return moveOperator(std::move(other));
+ }
+
+-VideoFrame VideoDecoderContext::decode(const Packet &packet, OptionalErrorCode ec, bool autoAllocateFrame)
++VideoFrame VideoDecoderContext::decode(const Packet &packet, OptionalErrorCode ec, bool autoAllocateFrame, bool send_packet)
+ {
+-    return decodeVideo(ec, packet, 0, nullptr, autoAllocateFrame);
++    return decodeVideo(ec, packet, 0, nullptr, autoAllocateFrame, send_packet);
+ }
+
+ VideoFrame VideoDecoderContext::decode(const Packet &packet, size_t offset, size_t &decodedBytes, OptionalErrorCode ec, bool autoAllocateFrame)
+ {
+-    return decodeVideo(ec, packet, offset, &decodedBytes, autoAllocateFrame);
++    return decodeVideo(ec, packet, offset, &decodedBytes, autoAllocateFrame, true);
+ }
+
+-VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet &packet, size_t offset, size_t *decodedBytes, bool autoAllocateFrame)
++VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet &packet, size_t offset, size_t *decodedBytes, bool autoAllocateFrame, bool send_packet)
+ {
+     clear_if(ec);
+
+@@ -230,7 +230,7 @@ VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet &
+     }
+
+     int gotFrame = 0;
+-    auto st = decodeCommon(outFrame, packet, offset, gotFrame, avcodec_decode_video_legacy);
++    auto st = decodeCommon(outFrame, packet, offset, send_packet, gotFrame, avcodec_decode_video_legacy);
+
+     if (get<1>(st)) {
+         throws_if(ec, get<0>(st), *get<1>(st));
+@@ -882,7 +882,7 @@ void CodecContext2::open(const Codec &codec, AVDictionary **options, OptionalErr
+         throws_if(ec, stat, ffmpeg_category());
+ }
+
+-std::pair<int, const error_category *> CodecContext2::decodeCommon(AVFrame *outFrame, const Packet &inPacket, size_t offset, int &frameFinished, int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) noexcept
++std::pair<int, const error_category *> CodecContext2::decodeCommon(AVFrame *outFrame, const Packet &inPacket, size_t offset, bool send_packet, int &frameFinished, int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) noexcept
+ {
+     if (!isValid())
+         return make_error_pair(Errors::CodecInvalid);
+@@ -902,7 +902,7 @@ std::pair<int, const error_category *> CodecContext2::decodeCommon(AVFrame *outF
+     pkt.data += offset;
+     pkt.size -= offset;
+
+-    int decoded = decodeProc(m_raw, outFrame, &frameFinished, &pkt);
++    int decoded = decodeProc(m_raw, outFrame, &frameFinished, send_packet? &pkt: nullptr);
+     return make_error_pair(decoded);
+ }
+
+@@ -940,19 +940,19 @@ AudioDecoderContext &AudioDecoderContext::operator=(AudioDecoderContext &&other)
+     return moveOperator(std::move(other));
+ }
+
+-AudioSamples AudioDecoderContext::decode(const Packet &inPacket, OptionalErrorCode ec)
++AudioSamples AudioDecoderContext::decode(const Packet &inPacket, OptionalErrorCode ec, bool send_packet)
+ {
+-    return decode(inPacket, 0u, ec);
++    return decode(inPacket, 0u, ec, send_packet);
+ }
+
+-AudioSamples AudioDecoderContext::decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec)
++AudioSamples AudioDecoderContext::decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec, bool send_packet)
+ {
+     clear_if(ec);
+
+     AudioSamples outSamples;
+
+     int gotFrame = 0;
+-    auto st = decodeCommon(outSamples, inPacket, offset, gotFrame, avcodec_decode_audio_legacy);
++    auto st = decodeCommon(outSamples, inPacket, offset, send_packet, gotFrame, avcodec_decode_audio_legacy);
+     if (get<1>(st))
+     {
+         throws_if(ec, get<0>(st), *get<1>(st));
+@@ -1016,10 +1016,11 @@ std::pair<int, const std::error_category*>
+ CodecContext2::decodeCommon(T &outFrame,
+              const Packet &inPacket,
+              size_t offset,
++             bool send_packet,
+              int &frameFinished,
+              int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *))
+ {
+-    auto st = decodeCommon(outFrame.raw(), inPacket, offset, frameFinished, decodeProc);
++    auto st = decodeCommon(outFrame.raw(), inPacket, offset, send_packet, frameFinished, decodeProc);
+     if (std::get<1>(st))
+         return st;
+
+diff --git a/src/avcpp/codeccontext.h b/src/avcpp/codeccontext.h
+index 2e32904..3e8f7d0 100644
+--- a/src/avcpp/codeccontext.h
++++ b/src/avcpp/codeccontext.h
+@@ -150,7 +150,7 @@ protected:
+
+
+     std::pair<int, const std::error_category*>
+-    decodeCommon(AVFrame *outFrame, const class Packet &inPacket, size_t offset, int &frameFinished,
++    decodeCommon(AVFrame *outFrame, const class Packet &inPacket, size_t offset, bool send_packet, int &frameFinished,
+                  int (*decodeProc)(AVCodecContext*, AVFrame*,int *, const AVPacket *)) noexcept;
+
+     std::pair<int, const std::error_category*>
+@@ -163,6 +163,7 @@ public:
+     decodeCommon(T &outFrame,
+                  const class Packet &inPacket,
+                  size_t offset,
++                 bool send_packet,
+                  int &frameFinished,
+                  int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *));
+
+@@ -417,12 +418,14 @@ public:
+      *                       av#throws the function will throw on error instead
+      * @param autoAllocateFrame  it true - output will be allocated at the ffmpeg internal, otherwise
+      *                           it will be allocated before decode proc call.
++     * @param send_packets   whether to send the packet or not (for successive decodes)
+      * @return encoded video frame, if error: exception thrown or error code returns, in both cases
+      *         output undefined.
+      */
+     VideoFrame decode(const Packet    &packet,
+                       OptionalErrorCode ec = throws(),
+-                      bool             autoAllocateFrame = true);
++                      bool autoAllocateFrame = true,
++                      bool send_packet = true);
+
+     /**
+      * @brief decodeVideo - decode video packet with additional parameters
+@@ -449,7 +452,8 @@ private:
+                            const Packet &packet,
+                            size_t offset,
+                            size_t *decodedBytes,
+-                           bool    autoAllocateFrame);
++                           bool    autoAllocateFrame,
++                           bool    send_packet);
+
+ };
+
+@@ -598,8 +602,8 @@ public:
+
+     AudioDecoderContext& operator=(AudioDecoderContext&& other);
+
+-    AudioSamples decode(const Packet &inPacket, OptionalErrorCode ec = throws());
+-    AudioSamples decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec = throws());
++    AudioSamples decode(const Packet &inPacket, OptionalErrorCode ec = throws(), bool send_packet = true);
++    AudioSamples decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec = throws(), bool send_packet = true);
+
+ };
+

portfiles/avcpp/portfile.cmake

@@ -1,10 +1,16 @@
+if(VCPKG_TARGET_IS_WINDOWS)
+    # avcpp doesn't export any symbols
+    vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
+endif()
+
 vcpkg_from_github(
     OUT_SOURCE_PATH SOURCE_PATH
     REPO h4tr3d/avcpp
     REF "b56c4fd8d94e149c8caa14e8fc29820899c9d585"
     SHA512 7c36a90ed878f5addc1e3ebed21db66088f39f57605f73c6b7f260a3c0537d4a908f60fb17be0ec801e2e7eaeee6b3b701a9c7f99f2e0ba0a00eb97b62567cc4
     PATCHES
         0002-av_init_packet_deprecation.patch
+        0001-codeccontext.patch
 )
 
 string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" AVCPP_ENABLE_STATIC)

src/renderer/dx9/nv12frame.cpp

@@ -0,0 +1,52 @@
+#include "nv12frame.h"
+
+#include "../../log.h"
+
+namespace Impacto {
+namespace DirectX9 {
+
+DX9NV12Frame::DX9NV12Frame(IDirect3DDevice9* device) { Device = device; }
+
+void DX9NV12Frame::Init(float width, float height) {
+  Width = width;
+  Height = height;
+  auto err = Device->CreateTexture((UINT)Width, (UINT)Height, 1, 0, D3DFMT_A8,
+                                   D3DPOOL_MANAGED, &Luma, nullptr);
+  err = Device->CreateTexture((UINT)(Width / 2), (UINT)(Height / 2), 1, 0,
+                              D3DFMT_A8L8, D3DPOOL_MANAGED, &CbCr, nullptr);
+}
+
+void Impacto::DirectX9::DX9NV12Frame::Submit(const void* luma, int lumaStride,
+                                             const void* cbcr, int cbcrStride) {
+  D3DLOCKED_RECT lockRect;
+  auto err = Luma->LockRect(0, &lockRect, NULL, 0);
+  const uint8_t* src = (const uint8_t*)luma;
+  uint8_t* dst = (uint8_t*)lockRect.pBits;
+  for (int y = 0; y < (int)Height; y++) {
+    memcpy(dst, src, (int)Width);
+    src += lumaStride;
+    dst += lockRect.Pitch;
+  }
+
+  err = Luma->UnlockRect(0);
+
+  err = CbCr->LockRect(0, &lockRect, NULL, 0);
+
+  src = (const uint8_t*)cbcr;
+  dst = (uint8_t*)lockRect.pBits;
+  for (int y = 0; y < (int)Height / 2; y++) {
+    memcpy(dst, src, (int)Width);
+    src += cbcrStride;
+    dst += lockRect.Pitch;
+  }
+
+  err = CbCr->UnlockRect(0);
+}
+
+void DX9NV12Frame::Release() {
+  Luma->Release();
+  CbCr->Release();
+}
+
+}  // namespace DirectX9
+}  // namespace Impacto