diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index fef6e8303..812fd9cf6 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -144,7 +144,7 @@ jobs: - name: Install Dependencies Linux run: | sudo apt-get update - sudo apt-get install nasm libx11-dev libxft-dev libxext-dev libwayland-dev libxkbcommon-dev libegl1-mesa-dev libibus-1.0-dev libxrandr-dev libltdl-dev mono-complete autoconf autoconf-archive automake libtool + sudo apt-get install nasm libx11-dev libxft-dev libxext-dev libwayland-dev libxkbcommon-dev libegl1-mesa-dev libibus-1.0-dev libxrandr-dev libltdl-dev mono-complete autoconf autoconf-archive automake libtool libdrm-dev if: matrix.os_name == 'linux' || matrix.os_name == 'android' - name: Install Dependencies Mac run: brew install nasm mono diff --git a/CMakeLists.txt b/CMakeLists.txt index e14e4653b..4285bd12a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ set(Impacto_Header src/renderer/renderer.h src/renderer/window.h src/renderer/yuvframe.h + src/renderer/nv12frame.h src/data/savesystem.h src/data/tipssystem.h @@ -1145,6 +1146,7 @@ if (NOT DEFINED IMPACTO_DISABLE_OPENGL) src/renderer/opengl/shader.cpp src/renderer/opengl/glc.cpp src/renderer/opengl/yuvframe.cpp + src/renderer/opengl/nv12frame.cpp src/renderer/opengl/3d/renderable3d.cpp src/renderer/opengl/3d/scene.cpp @@ -1156,6 +1158,7 @@ if (NOT DEFINED IMPACTO_DISABLE_OPENGL) src/renderer/opengl/shader.h src/renderer/opengl/glc.h src/renderer/opengl/yuvframe.h + src/renderer/opengl/nv12frame.h src/renderer/opengl/3d/renderable3d.h src/renderer/opengl/3d/scene.h ) @@ -1175,19 +1178,21 @@ if (NOT DEFINED IMPACTO_DISABLE_FFMPEG) ) if (NOT VCPKG_TOOLCHAIN) - set(AVCPP_PATCH git apply ${CMAKE_CURRENT_SOURCE_DIR}/vendor/patches/avcpp.patch) if(NOT BUILD_SHARED_LIBS) set(AV_ENABLE_SHARED OFF CACHE BOOL "Enable shared library build (Off)" FORCE) endif() set(AV_DISABLE_AVDEVICE ON CACHE INTERNAL "Disable FFMPEG AVDevice") set(AV_DISABLE_AVFILTER ON CACHE INTERNAL "Disable FFMPEG AVFilter") set(AV_BUILD_EXAMPLES OFF CACHE INTERNAL "Build AVCPP Examples") + set(AVCPP_PATCH git apply ${CMAKE_CURRENT_SOURCE_DIR}/portfiles/avcpp/0001-codeccontext.patch) FetchContent_Declare( avcpp GIT_REPOSITORY "https://github.com/h4tr3d/avcpp" GIT_TAG "00e29ea0c0f7036423b80553aaccacd70f90aace" SYSTEM GIT_SUBMODULES "" + PATCH_COMMAND ${AVCPP_PATCH} + UPDATE_DISCONNECTED 1 ) FetchContent_MakeAvailable(avcpp) @@ -1204,6 +1209,10 @@ if (NOT DEFINED IMPACTO_DISABLE_FFMPEG) PkgConfig::FFmpeg ) else () + find_package(FFMPEG REQUIRED) + list(APPEND Impacto_Include_Dirs ${FFMPEG_INCLUDE_DIRS}) + list(APPEND Impacto_Libs ${FFMPEG_LIBRARIES}) + find_package(avcpp CONFIG REQUIRED) list(APPEND Impacto_Libs avcpp::avcpp @@ -1240,6 +1249,7 @@ if (NOT DEFINED IMPACTO_DISABLE_VULKAN) src/renderer/vulkan/pipeline.cpp src/renderer/vulkan/utils.cpp src/renderer/vulkan/yuvframe.cpp + src/renderer/vulkan/nv12frame.cpp src/renderer/vulkan/3d/renderable3d.cpp src/renderer/vulkan/3d/scene.cpp ) @@ -1248,7 +1258,7 @@ if (NOT DEFINED IMPACTO_DISABLE_VULKAN) src/renderer/vulkan/renderer.h src/renderer/vulkan/pipeline.h src/renderer/vulkan/utils.h - src/renderer/vulkan/yuvframe.h + src/renderer/vulkan/nv12frame.h src/renderer/vulkan/3d/renderable3d.h src/renderer/vulkan/3d/scene.h ) @@ -1267,6 +1277,7 @@ if (NOT DEFINED IMPACTO_DISABLE_DX9) src/renderer/dx9/window.cpp src/renderer/dx9/shader.cpp src/renderer/dx9/yuvframe.cpp + src/renderer/dx9/nv12frame.cpp src/renderer/dx9/3d/scene.cpp src/renderer/dx9/3d/renderable3d.cpp ) @@ -1276,6 +1287,7 @@ if (NOT DEFINED IMPACTO_DISABLE_DX9) src/renderer/dx9/window.h src/renderer/dx9/shader.h src/renderer/dx9/yuvframe.h + src/renderer/dx9/nv12frame.h src/renderer/dx9/3d/scene.h src/renderer/dx9/3d/renderable3d.h ) diff --git a/CMakePresets.json b/CMakePresets.json index 259fc3ded..3671b1fe2 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -54,6 +54,7 @@ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/triplets", + "VCPKG_OVERLAY_PORTS": "${sourceDir}/portfiles", "IMPACTO_WARNINGS": "ON" } }, diff --git a/portfiles/avcpp/0001-codeccontext.patch b/portfiles/avcpp/0001-codeccontext.patch new file mode 100644 index 000000000..88b33cc53 --- /dev/null +++ b/portfiles/avcpp/0001-codeccontext.patch @@ -0,0 +1,157 @@ +diff --git a/src/avcpp/codeccontext.cpp b/src/avcpp/codeccontext.cpp +index f67807e..6f3783f 100644 +--- a/src/avcpp/codeccontext.cpp ++++ b/src/avcpp/codeccontext.cpp +@@ -90,7 +90,7 @@ int decode(AVCodecContext *avctx, + } + + ret = avcodec_receive_frame(avctx, picture); +- if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) ++ if (ret < 0 && ret != AVERROR_EOF) + return ret; + if (ret >= 0 && got_picture_ptr) + *got_picture_ptr = 1; +@@ -203,17 +203,17 @@ VideoDecoderContext &VideoDecoderContext::operator=(VideoDecoderContext&& other) + return moveOperator(std::move(other)); + } + +-VideoFrame VideoDecoderContext::decode(const Packet &packet, OptionalErrorCode ec, bool autoAllocateFrame) ++VideoFrame VideoDecoderContext::decode(const Packet &packet, OptionalErrorCode ec, bool autoAllocateFrame, bool send_packet) + { +- return decodeVideo(ec, packet, 0, nullptr, autoAllocateFrame); ++ return decodeVideo(ec, packet, 0, nullptr, autoAllocateFrame, send_packet); + } + + VideoFrame VideoDecoderContext::decode(const Packet &packet, size_t offset, size_t &decodedBytes, OptionalErrorCode ec, bool autoAllocateFrame) + { +- return decodeVideo(ec, packet, offset, &decodedBytes, autoAllocateFrame); ++ return decodeVideo(ec, packet, offset, &decodedBytes, autoAllocateFrame, true); + } + +-VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet &packet, size_t offset, size_t *decodedBytes, bool autoAllocateFrame) ++VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet &packet, size_t offset, size_t *decodedBytes, bool autoAllocateFrame, bool send_packet) + { + clear_if(ec); + +@@ -230,7 +230,7 @@ VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet & + } + + int gotFrame = 0; +- auto st = decodeCommon(outFrame, packet, offset, gotFrame, avcodec_decode_video_legacy); ++ auto st = decodeCommon(outFrame, packet, offset, send_packet, gotFrame, avcodec_decode_video_legacy); + + if (get<1>(st)) { + throws_if(ec, get<0>(st), *get<1>(st)); +@@ -882,7 +882,7 @@ void CodecContext2::open(const Codec &codec, AVDictionary **options, OptionalErr + throws_if(ec, stat, ffmpeg_category()); + } + +-std::pair CodecContext2::decodeCommon(AVFrame *outFrame, const Packet &inPacket, size_t offset, int &frameFinished, int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) noexcept ++std::pair CodecContext2::decodeCommon(AVFrame *outFrame, const Packet &inPacket, size_t offset, bool send_packet, int &frameFinished, int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) noexcept + { + if (!isValid()) + return make_error_pair(Errors::CodecInvalid); +@@ -902,7 +902,7 @@ std::pair CodecContext2::decodeCommon(AVFrame *outF + pkt.data += offset; + pkt.size -= offset; + +- int decoded = decodeProc(m_raw, outFrame, &frameFinished, &pkt); ++ int decoded = decodeProc(m_raw, outFrame, &frameFinished, send_packet? &pkt: nullptr); + return make_error_pair(decoded); + } + +@@ -940,19 +940,19 @@ AudioDecoderContext &AudioDecoderContext::operator=(AudioDecoderContext &&other) + return moveOperator(std::move(other)); + } + +-AudioSamples AudioDecoderContext::decode(const Packet &inPacket, OptionalErrorCode ec) ++AudioSamples AudioDecoderContext::decode(const Packet &inPacket, OptionalErrorCode ec, bool send_packet) + { +- return decode(inPacket, 0u, ec); ++ return decode(inPacket, 0u, ec, send_packet); + } + +-AudioSamples AudioDecoderContext::decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec) ++AudioSamples AudioDecoderContext::decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec, bool send_packet) + { + clear_if(ec); + + AudioSamples outSamples; + + int gotFrame = 0; +- auto st = decodeCommon(outSamples, inPacket, offset, gotFrame, avcodec_decode_audio_legacy); ++ auto st = decodeCommon(outSamples, inPacket, offset, send_packet, gotFrame, avcodec_decode_audio_legacy); + if (get<1>(st)) + { + throws_if(ec, get<0>(st), *get<1>(st)); +@@ -1016,10 +1016,11 @@ std::pair + CodecContext2::decodeCommon(T &outFrame, + const Packet &inPacket, + size_t offset, ++ bool send_packet, + int &frameFinished, + int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) + { +- auto st = decodeCommon(outFrame.raw(), inPacket, offset, frameFinished, decodeProc); ++ auto st = decodeCommon(outFrame.raw(), inPacket, offset, send_packet, frameFinished, decodeProc); + if (std::get<1>(st)) + return st; + +diff --git a/src/avcpp/codeccontext.h b/src/avcpp/codeccontext.h +index 2e32904..3e8f7d0 100644 +--- a/src/avcpp/codeccontext.h ++++ b/src/avcpp/codeccontext.h +@@ -150,7 +150,7 @@ protected: + + + std::pair +- decodeCommon(AVFrame *outFrame, const class Packet &inPacket, size_t offset, int &frameFinished, ++ decodeCommon(AVFrame *outFrame, const class Packet &inPacket, size_t offset, bool send_packet, int &frameFinished, + int (*decodeProc)(AVCodecContext*, AVFrame*,int *, const AVPacket *)) noexcept; + + std::pair +@@ -163,6 +163,7 @@ public: + decodeCommon(T &outFrame, + const class Packet &inPacket, + size_t offset, ++ bool send_packet, + int &frameFinished, + int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)); + +@@ -417,12 +418,14 @@ public: + * av#throws the function will throw on error instead + * @param autoAllocateFrame it true - output will be allocated at the ffmpeg internal, otherwise + * it will be allocated before decode proc call. ++ * @param send_packets whether to send the packet or not (for successive decodes) + * @return encoded video frame, if error: exception thrown or error code returns, in both cases + * output undefined. + */ + VideoFrame decode(const Packet &packet, + OptionalErrorCode ec = throws(), +- bool autoAllocateFrame = true); ++ bool autoAllocateFrame = true, ++ bool send_packet = true); + + /** + * @brief decodeVideo - decode video packet with additional parameters +@@ -449,7 +452,8 @@ private: + const Packet &packet, + size_t offset, + size_t *decodedBytes, +- bool autoAllocateFrame); ++ bool autoAllocateFrame, ++ bool send_packet); + + }; + +@@ -598,8 +602,8 @@ public: + + AudioDecoderContext& operator=(AudioDecoderContext&& other); + +- AudioSamples decode(const Packet &inPacket, OptionalErrorCode ec = throws()); +- AudioSamples decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec = throws()); ++ AudioSamples decode(const Packet &inPacket, OptionalErrorCode ec = throws(), bool send_packet = true); ++ AudioSamples decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec = throws(), bool send_packet = true); + + }; + diff --git a/portfiles/avcpp/portfile.cmake b/portfiles/avcpp/portfile.cmake index d24755bd4..fcb234cc7 100644 --- a/portfiles/avcpp/portfile.cmake +++ b/portfiles/avcpp/portfile.cmake @@ -1,8 +1,13 @@ +if(VCPKG_TARGET_IS_WINDOWS) + # avcpp doesn't export any symbols + vcpkg_check_linkage(ONLY_STATIC_LIBRARY) +endif() + vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO h4tr3d/avcpp - REF "b56c4fd8d94e149c8caa14e8fc29820899c9d585" - SHA512 7c36a90ed878f5addc1e3ebed21db66088f39f57605f73c6b7f260a3c0537d4a908f60fb17be0ec801e2e7eaeee6b3b701a9c7f99f2e0ba0a00eb97b62567cc4 + REF "6b33527fa392eff2d2fb006f83e13bd2ba516b77" + SHA512 6085d3442efc6b73f1a2bb94aaa1c57cf7b4be48da6471fa42f694e4da1977f576816367c7882204ac9af2b6aac11037cf57acd1bc081a03c06019c809c48f37 PATCHES 0002-av_init_packet_deprecation.patch ) diff --git a/src/renderer/dx9/nv12frame.cpp b/src/renderer/dx9/nv12frame.cpp new file mode 100644 index 000000000..58b75f471 --- /dev/null +++ b/src/renderer/dx9/nv12frame.cpp @@ -0,0 +1,52 @@ +#include "nv12frame.h" + +#include "../../log.h" + +namespace Impacto { +namespace DirectX9 { + +DX9NV12Frame::DX9NV12Frame(IDirect3DDevice9* device) { Device = device; } + +void DX9NV12Frame::Init(float width, float height) { + Width = width; + Height = height; + auto err = Device->CreateTexture((UINT)Width, (UINT)Height, 1, 0, D3DFMT_A8, + D3DPOOL_MANAGED, &Luma, nullptr); + err = Device->CreateTexture((UINT)(Width / 2), (UINT)(Height / 2), 1, 0, + D3DFMT_A8L8, D3DPOOL_MANAGED, &CbCr, nullptr); +} + +void Impacto::DirectX9::DX9NV12Frame::Submit(const void* luma, int lumaStride, + const void* cbcr, int cbcrStride) { + D3DLOCKED_RECT lockRect; + auto err = Luma->LockRect(0, &lockRect, NULL, 0); + const uint8_t* src = (const uint8_t*)luma; + uint8_t* dst = (uint8_t*)lockRect.pBits; + for (int y = 0; y < (int)Height; y++) { + memcpy(dst, src, (int)Width); + src += lumaStride; + dst += lockRect.Pitch; + } + + err = Luma->UnlockRect(0); + + err = CbCr->LockRect(0, &lockRect, NULL, 0); + + src = (const uint8_t*)cbcr; + dst = (uint8_t*)lockRect.pBits; + for (int y = 0; y < (int)Height / 2; y++) { + memcpy(dst, src, (int)Width); + src += cbcrStride; + dst += lockRect.Pitch; + } + + err = CbCr->UnlockRect(0); +} + +void DX9NV12Frame::Release() { + Luma->Release(); + CbCr->Release(); +} + +} // namespace DirectX9 +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/dx9/nv12frame.h b/src/renderer/dx9/nv12frame.h new file mode 100644 index 000000000..da09e0897 --- /dev/null +++ b/src/renderer/dx9/nv12frame.h @@ -0,0 +1,30 @@ +#pragma once + +#include "../nv12frame.h" + +#include + +namespace Impacto { +namespace DirectX9 { + +class DX9NV12Frame : public NV12Frame { + friend class Renderer; + + public: + DX9NV12Frame(IDirect3DDevice9* device); + + void Init(float width, float height) override; + + void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) override; + void Release() override; + + protected: + IDirect3DDevice9* Device; + + IDirect3DTexture9* Luma; + IDirect3DTexture9* CbCr; +}; + +} // namespace DirectX9 +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/dx9/renderer.cpp b/src/renderer/dx9/renderer.cpp index 717b95e11..764ae7aa5 100644 --- a/src/renderer/dx9/renderer.cpp +++ b/src/renderer/dx9/renderer.cpp @@ -82,6 +82,8 @@ void Renderer::Init() { vertexDeclaration); ShaderYUVFrame = new Shader(); ShaderYUVFrame->Compile("YUVFrame", Device, vertexDeclaration); + ShaderNV12Frame = new Shader(); + ShaderNV12Frame->Compile("NV12Frame", Device, vertexDeclaration); ShaderCCMessageBox = new Shader(); ShaderCCMessageBox->Compile("CCMessageBoxSprite", Device, vertexDeclaration); ShaderCHLCCMenuBackground = new Shader(); @@ -258,9 +260,15 @@ void Renderer::FreeTexture(uint32_t id) { } YUVFrame* Renderer::CreateYUVFrame(float width, float height) { - VideoFrameInternal = new DX9YUVFrame(Device); - VideoFrameInternal->Init(width, height); - return (YUVFrame*)VideoFrameInternal; + VideoFrameInternalYUV = new DX9YUVFrame(Device); + VideoFrameInternalYUV->Init(width, height); + return (YUVFrame*)VideoFrameInternalYUV; +} + +NV12Frame* Renderer::CreateNV12Frame(float width, float height) { + VideoFrameInternalNV12 = new DX9NV12Frame(Device); + VideoFrameInternalNV12->Init(width, height); + return (NV12Frame*)VideoFrameInternalNV12; } void Renderer::DrawSprite(const Sprite& sprite, const CornersQuad& dest, @@ -687,9 +695,46 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, // Do we have space for one more sprite quad? EnsureSpaceAvailable(4, sizeof(VertexBufferSprites), 6); - auto err = Device->SetTexture(0, VideoFrameInternal->Luma); - err = Device->SetTexture(1, VideoFrameInternal->Cb); - err = Device->SetTexture(2, VideoFrameInternal->Cr); + auto err = Device->SetTexture(0, VideoFrameInternalYUV->Luma); + err = Device->SetTexture(1, VideoFrameInternalYUV->Cb); + err = Device->SetTexture(2, VideoFrameInternalYUV->Cr); + + // This is cursed man, idk + BOOL alphaVideoB = (BOOL)alphaVideo; + Device->SetPixelShaderConstantB(0, &alphaVideoB, 1); + + // OK, all good, make quad + + VertexBufferSprites* vertices = + (VertexBufferSprites*)(VertexBuffer + VertexBufferFill); + VertexBufferFill += 4 * sizeof(VertexBufferSprites); + + IndexBufferFill += 6; + + QuadSetUV(RectF(0.0f, 0.0f, frame.Width, frame.Height), + {frame.Width, frame.Height}, &vertices[0].UV, + sizeof(VertexBufferSprites)); + QuadSetPosition(dest, &vertices[0].Position, sizeof(VertexBufferSprites)); + + for (int i = 0; i < 4; i++) vertices[i].Tint = tint; +} + +void Renderer::DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + const glm::vec4 tint, const bool alphaVideo) { + if (!Drawing) { + ImpLog(LogLevel::Error, LogChannel::Render, + "Renderer->DrawVideoTexture() called before BeginFrame()\n"); + return; + } + + EnsureShader(ShaderNV12Frame); + CurrentTexture = std::numeric_limits::max(); + + // Do we have space for one more sprite quad? + EnsureSpaceAvailable(4, sizeof(VertexBufferSprites), 6); + + auto err = Device->SetTexture(0, VideoFrameInternalNV12->Luma); + err = Device->SetTexture(1, VideoFrameInternalNV12->CbCr); // This is cursed man, idk BOOL alphaVideoB = (BOOL)alphaVideo; diff --git a/src/renderer/dx9/renderer.h b/src/renderer/dx9/renderer.h index e9aa522ff..9e9be3039 100644 --- a/src/renderer/dx9/renderer.h +++ b/src/renderer/dx9/renderer.h @@ -4,6 +4,7 @@ #include "window.h" #include "shader.h" #include "yuvframe.h" +#include "nv12frame.h" #include @@ -33,6 +34,7 @@ class Renderer : public BaseRenderer { } void FreeTexture(uint32_t id) override; YUVFrame* CreateYUVFrame(float width, float height) override; + NV12Frame* CreateNV12Frame(float width, float height) override; void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, std::span tints, @@ -91,7 +93,8 @@ class Renderer : public BaseRenderer { void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo) override; - + void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo = false) override; void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, const glm::mat4 transformation, const glm::vec4 tint) override {}; // TODO: Implement @@ -134,13 +137,15 @@ class Renderer : public BaseRenderer { Shader* ShaderMaskedSprite; Shader* ShaderMaskedSpriteNoAlpha; Shader* ShaderYUVFrame; + Shader* ShaderNV12Frame; Shader* ShaderCCMessageBox; Shader* ShaderCHLCCMenuBackground; uint32_t CurrentTexture = 0; uint32_t NextTextureId = 1; - DX9YUVFrame* VideoFrameInternal; + DX9YUVFrame* VideoFrameInternalYUV; + DX9NV12Frame* VideoFrameInternalNV12; bool Drawing = false; diff --git a/src/renderer/nv12frame.h b/src/renderer/nv12frame.h new file mode 100644 index 000000000..116e3b437 --- /dev/null +++ b/src/renderer/nv12frame.h @@ -0,0 +1,21 @@ +#pragma once + +#include "../impacto.h" + +namespace Impacto { + +class NV12Frame { + public: + float Width; + float Height; + uint32_t LumaId; + uint32_t CbCrId; + + virtual void Init(float width, float height) = 0; + + virtual void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) = 0; + virtual void Release() = 0; +}; + +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/opengl/nv12frame.cpp b/src/renderer/opengl/nv12frame.cpp new file mode 100644 index 000000000..a0cabb7c2 --- /dev/null +++ b/src/renderer/opengl/nv12frame.cpp @@ -0,0 +1,43 @@ +#include "nv12frame.h" +#include "../../log.h" + +#include + +namespace Impacto { +namespace OpenGL { + +void GLNV12Frame::Init(float width, float height) { + Width = width; + Height = height; + GLuint yuv[2]; + + glGenTextures(2, yuv); + LumaId = yuv[0]; + CbCrId = yuv[1]; +} + +void Impacto::OpenGL::GLNV12Frame::Submit(const void* luma, int lumaStride, + const void* cbcr, int cbcrStride) { + glBindTexture(GL_TEXTURE_2D, LumaId); + glPixelStorei(GL_UNPACK_ROW_LENGTH, lumaStride); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, (GLsizei)Width, (GLsizei)Height, 0, + GL_RED, GL_UNSIGNED_BYTE, luma); + glGenerateMipmap(GL_TEXTURE_2D); + + glBindTexture(GL_TEXTURE_2D, CbCrId); + glPixelStorei(GL_UNPACK_ROW_LENGTH, cbcrStride / 2); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, (GLsizei)(Width / 2), + (GLsizei)(Height / 2), 0, GL_RG, GL_UNSIGNED_BYTE, cbcr); + glGenerateMipmap(GL_TEXTURE_2D); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); +} + +void GLNV12Frame::Release() { + GLuint yuv[2]; + yuv[0] = LumaId; + yuv[1] = CbCrId; + glDeleteTextures(2, yuv); +} + +} // namespace OpenGL +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/opengl/nv12frame.h b/src/renderer/opengl/nv12frame.h new file mode 100644 index 000000000..da4179bb1 --- /dev/null +++ b/src/renderer/opengl/nv12frame.h @@ -0,0 +1,18 @@ +#pragma once + +#include "../nv12frame.h" + +namespace Impacto { +namespace OpenGL { + +class GLNV12Frame : public NV12Frame { + public: + void Init(float width, float height) override; + + void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) override; + void Release() override; +}; + +} // namespace OpenGL +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/opengl/renderer.cpp b/src/renderer/opengl/renderer.cpp index b0428b0ee..d5ba2d3ed 100644 --- a/src/renderer/opengl/renderer.cpp +++ b/src/renderer/opengl/renderer.cpp @@ -8,6 +8,7 @@ #include "../../log.h" #include "3d/scene.h" #include "yuvframe.h" +#include "nv12frame.h" #ifndef IMPACTO_DISABLE_IMGUI #include @@ -100,6 +101,7 @@ void Renderer::Init() { GaussianBlurShaderProgram.emplace(Shaders.Compile("GaussianBlur")); MosaicShaderProgram.emplace(Shaders.Compile("Mosaic")); SubtitleGlyphShaderProgram.emplace(Shaders.Compile("SubtitleGlyph")); + NV12FrameShaderProgram.emplace(Shaders.Compile("NV12Frame")); glGenSamplers((GLsizei)Samplers.size(), Samplers.data()); for (size_t i = 0; i < TextureUnitCount; i++) { @@ -191,26 +193,27 @@ uint32_t Renderer::SubmitTexture(TexFmt format, uint8_t* buffer, int width, glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 16); // Load in data - const GLuint texFormat = [format]() { + const auto [internalFormat, + texFormat] = [format]() -> std::pair { switch (format) { case TexFmt_RGBA: glPixelStorei(GL_UNPACK_ALIGNMENT, 4); - return GL_RGBA; + return {GL_RGBA, GL_RGBA}; case TexFmt_RGB: glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - return GL_RGB; + return {GL_RGB, GL_RGB}; case TexFmt_U8: glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - return GL_RED; + return {GL_R8, GL_RED}; default: throw std::invalid_argument( fmt::format("Unimplemented texture format {}", (int)format)); } }(); - glTexImage2D(GL_TEXTURE_2D, 0, texFormat, width, height, 0, texFormat, + glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, width, height, 0, texFormat, GL_UNSIGNED_BYTE, buffer); // Build mip chain @@ -261,6 +264,12 @@ YUVFrame* Renderer::CreateYUVFrame(float width, float height) { return (YUVFrame*)frame; } +NV12Frame* Renderer::CreateNV12Frame(float width, float height) { + auto frame = new GLNV12Frame(); + frame->Init(width, height); + return (NV12Frame*)frame; +} + void Renderer::InsertVertices( const std::span vertices, const std::span indices) { @@ -1014,6 +1023,35 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, InsertVerticesQuad(dest, RectF(0.0f, 0.0f, 1.0f, 1.0f), tint); } +void Renderer::DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + const glm::vec4 tint, const bool alphaVideo) { + if (!Drawing) { + ImpLog(LogLevel::Error, LogChannel::Render, + "Renderer->DrawVideoTexture() called before BeginFrame()\n"); + return; + } + + EnsureTopologyMode(TopologyMode::Triangles); + + UseTextures(std::array, 2>{ + std::pair{frame.LumaId, 0}, + std::pair{frame.CbCrId, 1}, + }); + + NV12FrameUniforms uniforms{ + .Projection = Projection, + .Luma = 0, + .CbCr = 1, + .IsAlpha = alphaVideo, + }; + + UseShader(*NV12FrameShaderProgram, uniforms); + + // OK, all good, make quad + + InsertVerticesQuad(dest, RectF(0.0f, 0.0f, 1.0f, 1.0f), tint); +} + void Renderer::CaptureScreencap(Sprite& sprite) { Flush(); sprite.Sheet.IsScreenCap = true; diff --git a/src/renderer/opengl/renderer.h b/src/renderer/opengl/renderer.h index 570b0e581..d0b0d7308 100644 --- a/src/renderer/opengl/renderer.h +++ b/src/renderer/opengl/renderer.h @@ -35,6 +35,7 @@ class Renderer : public BaseRenderer { std::span outBuffer) override; void FreeTexture(uint32_t id) override; YUVFrame* CreateYUVFrame(float width, float height) override; + NV12Frame* CreateNV12Frame(float width, float height) override; void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, std::span tints, @@ -92,6 +93,8 @@ class Renderer : public BaseRenderer { void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo) override; + void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo) override; void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, const glm::mat4 transformation, const glm::vec4 tint) override; @@ -144,6 +147,7 @@ class Renderer : public BaseRenderer { std::optional GaussianBlurShaderProgram; std::optional MosaicShaderProgram; std::optional SubtitleGlyphShaderProgram; + std::optional NV12FrameShaderProgram; const void* CurrentShaderProgram = nullptr; diff --git a/src/renderer/opengl/shader.cpp b/src/renderer/opengl/shader.cpp index 1aec36f46..7ad6ad9a0 100644 --- a/src/renderer/opengl/shader.cpp +++ b/src/renderer/opengl/shader.cpp @@ -768,5 +768,24 @@ void SubtitleGlyphShader::UploadUniforms(SubtitleGlyphUniforms newUniforms) { UpdateVar(newUniforms.CoverageMap, Uniforms.CoverageMap, CoverageMapLocation); } +NV12FrameShader::NV12FrameShader(GLint programId) + : Shader(programId), + ProjectionLocation(glGetUniformLocation(programId, "Projection")), + LumaLocation(glGetUniformLocation(programId, "Luma")), + CbCrLocation(glGetUniformLocation(programId, "CbCr")), + IsAlphaLocation(glGetUniformLocation(programId, "IsAlpha")) { + UploadVar(Uniforms.Projection, ProjectionLocation); + UploadVar(Uniforms.Luma, LumaLocation); + UploadVar(Uniforms.CbCr, CbCrLocation); + UploadVar(Uniforms.IsAlpha, IsAlphaLocation); +} + +void NV12FrameShader::UploadUniforms(NV12FrameUniforms newUniforms) { + UpdateVar(newUniforms.Projection, Uniforms.Projection, ProjectionLocation); + UpdateVar(newUniforms.Luma, Uniforms.Luma, LumaLocation); + UpdateVar(newUniforms.CbCr, Uniforms.CbCr, CbCrLocation); + UpdateVar(newUniforms.IsAlpha, Uniforms.IsAlpha, IsAlphaLocation); +} + } // namespace OpenGL } // namespace Impacto diff --git a/src/renderer/opengl/shader.h b/src/renderer/opengl/shader.h index ac6b5fb87..52016e1a6 100644 --- a/src/renderer/opengl/shader.h +++ b/src/renderer/opengl/shader.h @@ -641,5 +641,29 @@ class SubtitleGlyphShader : public Shader { const GLint CoverageMapLocation; }; +struct NV12FrameUniforms { + bool operator==(const NV12FrameUniforms& other) const = default; + + glm::mat4 Projection; + + GLint Luma = 0; + GLint CbCr = 0; + bool IsAlpha = false; +}; + +class NV12FrameShader : public Shader { + public: + NV12FrameShader(GLint programId); + + void UploadUniforms(NV12FrameUniforms uniforms) override; + + private: + const GLint ProjectionLocation; + + const GLint LumaLocation; + const GLint CbCrLocation; + const GLint IsAlphaLocation; +}; + } // namespace OpenGL } // namespace Impacto \ No newline at end of file diff --git a/src/renderer/renderer.h b/src/renderer/renderer.h index 2b8b4254d..a969dcdf9 100644 --- a/src/renderer/renderer.h +++ b/src/renderer/renderer.h @@ -6,6 +6,7 @@ #include "../spritesheet.h" #include "../text/text.h" #include "yuvframe.h" +#include "nv12frame.h" #include namespace Impacto { @@ -92,6 +93,7 @@ class BaseRenderer { std::span outBuffer) = 0; virtual void FreeTexture(uint32_t id) = 0; virtual YUVFrame* CreateYUVFrame(float width, float height) = 0; + virtual NV12Frame* CreateNV12Frame(float width, float height) = 0; virtual void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, @@ -387,6 +389,8 @@ class BaseRenderer { virtual void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo = false) = 0; + virtual void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo = false) = 0; virtual void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, glm::vec4 tint) = 0; diff --git a/src/renderer/vulkan/nv12frame.cpp b/src/renderer/vulkan/nv12frame.cpp new file mode 100644 index 000000000..ca18ad347 --- /dev/null +++ b/src/renderer/vulkan/nv12frame.cpp @@ -0,0 +1,157 @@ +#include "nv12frame.h" + +namespace Impacto { +namespace Vulkan { + +void VkNV12Frame::Init(float width, float height) { + Width = width; + Height = height; + + VkDeviceSize imageSize = (VkDeviceSize)(width * height); + VkDeviceSize bufferSize = + imageSize + 2 * (((VkDeviceSize)width / 2) * ((VkDeviceSize)height / 2)); + + StagingBuffer = CreateBuffer(bufferSize, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, + VMA_MEMORY_USAGE_CPU_ONLY); + vmaMapMemory(Allocator, StagingBuffer.Allocation, &MappedStagingBuffer); + + VkExtent3D lumaImageExtent; + lumaImageExtent.width = static_cast(width); + lumaImageExtent.height = static_cast(height); + lumaImageExtent.depth = 1; + + VkExtent3D cbCrImageExtent; + cbCrImageExtent.width = static_cast(width / 2); + cbCrImageExtent.height = static_cast(height / 2); + cbCrImageExtent.depth = 1; + + auto lumaImageInfo = + GetImageCreateInfo(VK_FORMAT_R8_UNORM, lumaImageExtent, + VkSampleCountFlagBits::VK_SAMPLE_COUNT_1_BIT); + VmaAllocationCreateInfo dimgAllocinfo = {}; + dimgAllocinfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + vmaCreateImage(Allocator, &lumaImageInfo, &dimgAllocinfo, + &LumaImage.Image.Image, &LumaImage.Image.Allocation, nullptr); + auto cbcrImageInfo = + GetImageCreateInfo(VK_FORMAT_R8G8_UNORM, cbCrImageExtent, + VkSampleCountFlagBits::VK_SAMPLE_COUNT_1_BIT); + vmaCreateImage(Allocator, &cbcrImageInfo, &dimgAllocinfo, + &CbCrImage.Image.Image, &CbCrImage.Image.Allocation, nullptr); + + VkImageViewCreateInfo imageInfo = {}; + imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + imageInfo.pNext = nullptr; + imageInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + imageInfo.image = LumaImage.Image.Image; + imageInfo.format = VK_FORMAT_R8_UNORM; + imageInfo.subresourceRange.baseMipLevel = 0; + imageInfo.subresourceRange.levelCount = 1; + imageInfo.subresourceRange.baseArrayLayer = 0; + imageInfo.subresourceRange.layerCount = 1; + imageInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + + vkCreateImageView(MainUploadContext.Device, &imageInfo, nullptr, + &LumaImage.ImageView); + + imageInfo.image = CbCrImage.Image.Image; + imageInfo.format = VK_FORMAT_R8G8_UNORM; + vkCreateImageView(MainUploadContext.Device, &imageInfo, nullptr, + &CbCrImage.ImageView); +} + +void Impacto::Vulkan::VkNV12Frame::Submit(const void* luma, int lumaStride, + const void* cbcr, int cbcrStride) { + const int cbcrOffset = (int)(Width * Height); + + const uint8_t* src = (const uint8_t*)luma; + uint8_t* dst = (uint8_t*)MappedStagingBuffer; + for (int y = 0; y < Height; y++) { + memcpy(dst, src, (int)Width); + src += lumaStride; + dst += (int)Width; + } + + src = (const uint8_t*)cbcr; + dst = (uint8_t*)MappedStagingBuffer + cbcrOffset; + for (int y = 0; y < (int)Height / 2; y++) { + memcpy(dst, src, (int)Width); + src += cbcrStride; + dst += (int)Width; + } + + ImmediateSubmit([&](VkCommandBuffer cmd) { + VkExtent3D lumaImageExtent; + lumaImageExtent.width = static_cast(Width); + lumaImageExtent.height = static_cast(Height); + lumaImageExtent.depth = 1; + VkExtent3D cbCrImageExtent; + cbCrImageExtent.width = static_cast(Width / 2); + cbCrImageExtent.height = static_cast(Height / 2); + cbCrImageExtent.depth = 1; + + VkImageSubresourceRange range; + range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + range.baseMipLevel = 0; + range.levelCount = 1; + range.baseArrayLayer = 0; + range.layerCount = 1; + + VkImageMemoryBarrier imageBarrierToTransfer = {}; + imageBarrierToTransfer.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + imageBarrierToTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imageBarrierToTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + imageBarrierToTransfer.image = LumaImage.Image.Image; + imageBarrierToTransfer.subresourceRange = range; + imageBarrierToTransfer.srcAccessMask = 0; + imageBarrierToTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 1, &imageBarrierToTransfer); + + VkBufferImageCopy copyRegion = {}; + copyRegion.bufferOffset = 0; + copyRegion.bufferRowLength = 0; + copyRegion.bufferImageHeight = 0; + copyRegion.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copyRegion.imageSubresource.mipLevel = 0; + copyRegion.imageSubresource.baseArrayLayer = 0; + copyRegion.imageSubresource.layerCount = 1; + copyRegion.imageExtent = lumaImageExtent; + vkCmdCopyBufferToImage(cmd, StagingBuffer.Buffer, LumaImage.Image.Image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, + ©Region); + + VkImageMemoryBarrier imageBarrierToReadable = imageBarrierToTransfer; + imageBarrierToReadable.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + imageBarrierToReadable.newLayout = VK_IMAGE_LAYOUT_GENERAL; + imageBarrierToReadable.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + imageBarrierToReadable.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, + 0, nullptr, 1, &imageBarrierToReadable); + + imageBarrierToTransfer.image = CbCrImage.Image.Image; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 1, &imageBarrierToTransfer); + + copyRegion.bufferOffset = (VkDeviceSize)(Width * Height); + copyRegion.imageExtent = cbCrImageExtent; + vkCmdCopyBufferToImage(cmd, StagingBuffer.Buffer, CbCrImage.Image.Image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, + ©Region); + + imageBarrierToReadable.image = CbCrImage.Image.Image; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, + 0, nullptr, 1, &imageBarrierToReadable); + }); +} + +void VkNV12Frame::Release() { + vmaUnmapMemory(Allocator, StagingBuffer.Allocation); +} + +} // namespace Vulkan +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/vulkan/nv12frame.h b/src/renderer/vulkan/nv12frame.h new file mode 100644 index 000000000..bc725f637 --- /dev/null +++ b/src/renderer/vulkan/nv12frame.h @@ -0,0 +1,31 @@ +#pragma once + +#include "../nv12frame.h" +#include "utils.h" + +#include + +namespace Impacto { +namespace Vulkan { + +class VkNV12Frame : public NV12Frame { + friend class Renderer; + + public: + void Init(float width, float height) override; + + void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) override; + void Release() override; + + protected: + VkTexture LumaImage{}; + VkTexture CbCrImage{}; + + private: + AllocatedBuffer StagingBuffer; + void* MappedStagingBuffer; +}; + +} // namespace Vulkan +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/vulkan/renderer.cpp b/src/renderer/vulkan/renderer.cpp index 6e0554d4b..bcdca8be3 100644 --- a/src/renderer/vulkan/renderer.cpp +++ b/src/renderer/vulkan/renderer.cpp @@ -8,6 +8,7 @@ #include "../../log.h" #include "3d/scene.h" #include "yuvframe.h" +#include "nv12frame.h" #ifndef IMPACTO_DISABLE_IMGUI #include @@ -737,6 +738,11 @@ void Renderer::Init() { PipelineYUVFrame->CreateWithShader( "YUVFrame", bindingDescription, attributeDescriptions.data(), attributeDescriptions.size(), TripleTextureSetLayout); + PipelineNV12Frame = new Pipeline(Device, RenderPass); + PipelineNV12Frame->SetPushConstants(&yuvFramePushConstants, 1); + PipelineNV12Frame->CreateWithShader( + "NV12Frame", bindingDescription, attributeDescriptions.data(), + attributeDescriptions.size(), TripleTextureSetLayout); VkPushConstantRange ccBoxPushConstant; ccBoxPushConstant.offset = 0; @@ -1128,9 +1134,15 @@ void Renderer::FreeTexture(uint32_t id) { } YUVFrame* Renderer::CreateYUVFrame(float width, float height) { - VideoFrameInternal = new VkYUVFrame(); - VideoFrameInternal->Init(width, height); - return (YUVFrame*)VideoFrameInternal; + VideoFrameInternalYUV = new VkYUVFrame(); + VideoFrameInternalYUV->Init(width, height); + return (YUVFrame*)VideoFrameInternalYUV; +} + +NV12Frame* Renderer::CreateNV12Frame(float width, float height) { + VideoFrameInternalNV12 = new VkNV12Frame(); + VideoFrameInternalNV12->Init(width, height); + return (NV12Frame*)VideoFrameInternalNV12; } void Renderer::DrawSprite(const Sprite& sprite, const CornersQuad& dest, @@ -1679,13 +1691,13 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, VkDescriptorImageInfo imageBufferInfo[3]; imageBufferInfo[0].sampler = Sampler; - imageBufferInfo[0].imageView = VideoFrameInternal->LumaImage.ImageView; + imageBufferInfo[0].imageView = VideoFrameInternalYUV->LumaImage.ImageView; imageBufferInfo[0].imageLayout = VK_IMAGE_LAYOUT_GENERAL; imageBufferInfo[1].sampler = Sampler; - imageBufferInfo[1].imageView = VideoFrameInternal->CbImage.ImageView; + imageBufferInfo[1].imageView = VideoFrameInternalYUV->CbImage.ImageView; imageBufferInfo[1].imageLayout = VK_IMAGE_LAYOUT_GENERAL; imageBufferInfo[2].sampler = Sampler; - imageBufferInfo[2].imageView = VideoFrameInternal->CrImage.ImageView; + imageBufferInfo[2].imageView = VideoFrameInternalYUV->CrImage.ImageView; imageBufferInfo[2].imageLayout = VK_IMAGE_LAYOUT_GENERAL; VkWriteDescriptorSet writeDescriptorSet{}; @@ -1723,6 +1735,59 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, for (int i = 0; i < 4; i++) vertices[i].Tint = tint; } +void Renderer::DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + const glm::vec4 tint, const bool alphaVideo) { + if (!Drawing) { + ImpLog(LogLevel::Error, LogChannel::Render, + "Renderer->DrawVideoTexture() called before BeginFrame()\n"); + return; + } + + EnsureMode(PipelineNV12Frame); + + VkDescriptorImageInfo imageBufferInfo[2]; + imageBufferInfo[0].sampler = Sampler; + imageBufferInfo[0].imageView = VideoFrameInternalNV12->LumaImage.ImageView; + imageBufferInfo[0].imageLayout = VK_IMAGE_LAYOUT_GENERAL; + imageBufferInfo[1].sampler = Sampler; + imageBufferInfo[1].imageView = VideoFrameInternalNV12->CbCrImage.ImageView; + imageBufferInfo[1].imageLayout = VK_IMAGE_LAYOUT_GENERAL; + + VkWriteDescriptorSet writeDescriptorSet{}; + writeDescriptorSet.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writeDescriptorSet.dstSet = 0; + writeDescriptorSet.dstBinding = 0; + writeDescriptorSet.descriptorCount = 2; + writeDescriptorSet.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + writeDescriptorSet.pImageInfo = imageBufferInfo; + + vkCmdPushDescriptorSetKHR( + CommandBuffers[CurrentFrameIndex], VK_PIPELINE_BIND_POINT_GRAPHICS, + PipelineNV12Frame->PipelineLayout, 0, 1, &writeDescriptorSet); + + YUVFramePushConstants constants = {}; + constants.IsAlpha = alphaVideo; + vkCmdPushConstants(CommandBuffers[CurrentFrameIndex], + CurrentPipeline->PipelineLayout, + VK_SHADER_STAGE_FRAGMENT_BIT, 0, + sizeof(YUVFramePushConstants), &constants); + + // OK, all good, make quad + MakeQuad(); + + VertexBufferSprites* vertices = + (VertexBufferSprites*)(VertexBuffer + VertexBufferOffset + + VertexBufferFill); + VertexBufferFill += 4 * sizeof(VertexBufferSprites); + + QuadSetUV(RectF(0.0f, 0.0f, frame.Width, frame.Height), + {frame.Width, frame.Height}, &vertices[0].UV, + sizeof(VertexBufferSprites)); + QuadSetPosition(dest, &vertices[0].Position, sizeof(VertexBufferSprites)); + + for (int i = 0; i < 4; i++) vertices[i].Tint = tint; +} + void Renderer::CaptureScreencap(Sprite& sprite) { if (Textures.count(sprite.Sheet.Texture) == 0) return; sprite.Sheet.IsScreenCap = true; diff --git a/src/renderer/vulkan/renderer.h b/src/renderer/vulkan/renderer.h index 65e8591a9..974989246 100644 --- a/src/renderer/vulkan/renderer.h +++ b/src/renderer/vulkan/renderer.h @@ -8,6 +8,7 @@ #include "utils.h" #include "window.h" #include "yuvframe.h" +#include "nv12frame.h" namespace Impacto { namespace Vulkan { @@ -61,6 +62,7 @@ class Renderer : public BaseRenderer { std::span outBuffer) override; void FreeTexture(uint32_t id) override; YUVFrame* CreateYUVFrame(float width, float height) override; + NV12Frame* CreateNV12Frame(float width, float height) override; void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, std::span tints, @@ -119,7 +121,8 @@ class Renderer : public BaseRenderer { void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo) override; - + void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo = false) override; void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, const glm::mat4 transformation, const glm::vec4 tint) override {}; // TODO: Implement @@ -220,6 +223,7 @@ class Renderer : public BaseRenderer { Pipeline* PipelineMaskedSprite; Pipeline* PipelineMaskedSpriteNoAlpha; Pipeline* PipelineYUVFrame; + Pipeline* PipelineNV12Frame; Pipeline* PipelineCCMessageBox; Pipeline* PipelineCHLCCMenuBackground; @@ -229,7 +233,8 @@ class Renderer : public BaseRenderer { uint32_t CurrentTexture = 0; uint32_t NextTextureId = 1; - VkYUVFrame* VideoFrameInternal; + VkYUVFrame* VideoFrameInternalYUV; + VkNV12Frame* VideoFrameInternalNV12; static int constexpr VertexBufferSize = 4096 * 4096; static int constexpr IndexBufferCount = diff --git a/src/shaders/dx9/NV12Frame_frag.hlsl b/src/shaders/dx9/NV12Frame_frag.hlsl new file mode 100644 index 000000000..62cf89d8a --- /dev/null +++ b/src/shaders/dx9/NV12Frame_frag.hlsl @@ -0,0 +1,37 @@ +bool IsAlpha : register(b0); + +struct PS_INPUT { + float4 position : POSITION; + float2 uv : TEXCOORD0; + float4 tint : COLOR; + float2 maskUV : TEXCOORD1; +}; + +sampler Luma : register(s0); +sampler CbCr : register(s1); + +static float4x4 yuv_to_rgb_rec601 = { + 1.16438, 0.00000, 1.59603, -0.87079, 1.16438, -0.39176, -0.81297, 0.52959, + 1.16438, 2.01723, 0.00000, -1.08139, 0, 0, 0, 1}; + +float4 getRGBA(float2 texUv) { + float4 yuvcolor = {1.0, 1.0, 1.0, 1.0}; + yuvcolor.r = tex2D(Luma, texUv).a; + yuvcolor.gb = tex2D(CbCr, texUv).ra; + + return mul(yuv_to_rgb_rec601, yuvcolor); +} + +float4 main(PS_INPUT input) : COLOR { + float4 color = {1.0, 1.0, 1.0, 1.0}; + + if (IsAlpha) { + color = getRGBA(float2(input.uv.x, input.uv.y / 2.0)); + color.a = getRGBA(float2(input.uv.x, input.uv.y / 2.0 + 0.5)).a; + } else { + color = getRGBA(input.uv); + } + color *= input.tint; + + return color; +} diff --git a/src/shaders/dx9/NV12Frame_vert.hlsl b/src/shaders/dx9/NV12Frame_vert.hlsl new file mode 100644 index 000000000..d1d297931 --- /dev/null +++ b/src/shaders/dx9/NV12Frame_vert.hlsl @@ -0,0 +1,24 @@ +struct VS_INPUT { + float2 Position : POSITION; + float2 UV : TEXCOORD0; + float4 Tint : COLOR; + float2 MaskUV : TEXCOORD1; +}; + +struct VS_OUTPUT { + float4 position : POSITION; + float2 uv : TEXCOORD0; + float4 tint : COLOR; + float2 maskUV : TEXCOORD1; +}; + +VS_OUTPUT main(VS_INPUT input) { + VS_OUTPUT output; + + output.position = float4(input.Position, 0.0, 1.0); + output.uv = input.UV; + output.tint = input.Tint; + output.maskUV = input.MaskUV; + + return output; +} diff --git a/src/shaders/opengl/NV12Frame_frag.glsl b/src/shaders/opengl/NV12Frame_frag.glsl new file mode 100644 index 000000000..84636d5d9 --- /dev/null +++ b/src/shaders/opengl/NV12Frame_frag.glsl @@ -0,0 +1,36 @@ +in vec2 uv; +in vec4 tint; + +out vec4 color; + +uniform sampler2D Luma; +uniform sampler2D CbCr; +uniform bool IsAlpha; + +const mat4 yuv_to_rgb_rec601 = mat4( + 1.16438, 0.00000, 1.59603, -0.87079, + 1.16438, -0.39176, -0.81297, 0.52959, + 1.16438, 2.01723, 0.00000, -1.08139, + 0, 0, 0, 1 +); + +vec4 getRGBA(vec2 texUv) { + vec4 yuvcolor = vec4(1.0); + + vec2 cbcr = texture(CbCr, texUv).rg; + yuvcolor.r = texture(Luma, texUv).r; + yuvcolor.g = cbcr.r; + yuvcolor.b = cbcr.g; + + return yuvcolor * yuv_to_rgb_rec601; +} + +void main() { + if (IsAlpha) { + color = getRGBA(vec2(uv.x, uv.y / 2.0)); + color.a = getRGBA(vec2(uv.x, uv.y / 2.0 + 0.5)).r; + } else { + color = getRGBA(uv); + } + color *= tint; +} \ No newline at end of file diff --git a/src/shaders/opengl/NV12Frame_vert.glsl b/src/shaders/opengl/NV12Frame_vert.glsl new file mode 100644 index 000000000..cdef1458d --- /dev/null +++ b/src/shaders/opengl/NV12Frame_vert.glsl @@ -0,0 +1,14 @@ +layout(location = 0) in vec2 Position; +layout(location = 1) in vec2 UV; +layout(location = 2) in vec4 Tint; + +out vec2 uv; +out vec4 tint; + +uniform mat4 Projection; + +void main() { + gl_Position = Projection * vec4(Position, 0.0, 1.0); + uv = UV; + tint = Tint; +} \ No newline at end of file diff --git a/src/shaders/vulkan/NV12Frame_frag.glsl b/src/shaders/vulkan/NV12Frame_frag.glsl new file mode 100644 index 000000000..66a12992a --- /dev/null +++ b/src/shaders/vulkan/NV12Frame_frag.glsl @@ -0,0 +1,38 @@ +#version 450 +#pragma shader_stage(fragment) + +layout(location = 0) in vec2 uv; +layout(location = 1) in vec4 tint; + +layout(location = 0) out vec4 color; + +layout(binding = 0) uniform sampler2D NV12Frame[2]; + +layout(push_constant) uniform constants +{ + bool IsAlpha; +} PushConstants; + +const mat4 yuv_to_rgb_rec601 = mat4( + 1.16438, 0.00000, 1.59603, -0.87079, + 1.16438, -0.39176, -0.81297, 0.52959, + 1.16438, 2.01723, 0.00000, -1.08139, + 0, 0, 0, 1 +); + +vec4 getRGBA(vec2 texUv) { + vec4 yuvcolor = vec4(1.0); + yuvcolor.r = texture(NV12Frame[0], texUv).r; + yuvcolor.gb = texture(NV12Frame[1], texUv).rg; + return yuvcolor * yuv_to_rgb_rec601; +} + +void main() { + if (PushConstants.IsAlpha) { + color = getRGBA(vec2(uv.x, uv.y / 2.0)); + color.a = getRGBA(vec2(uv.x, uv.y / 2.0 + 0.5)).r; + } else { + color = getRGBA(uv); + } + color *= tint; +} \ No newline at end of file diff --git a/src/shaders/vulkan/NV12Frame_frag.spv b/src/shaders/vulkan/NV12Frame_frag.spv new file mode 100644 index 000000000..6d162a11c Binary files /dev/null and b/src/shaders/vulkan/NV12Frame_frag.spv differ diff --git a/src/shaders/vulkan/NV12Frame_vert.glsl b/src/shaders/vulkan/NV12Frame_vert.glsl new file mode 100644 index 000000000..76e12b560 --- /dev/null +++ b/src/shaders/vulkan/NV12Frame_vert.glsl @@ -0,0 +1,16 @@ +#version 450 +#pragma shader_stage(vertex) + +layout(location = 0) in vec2 Position; +layout(location = 1) in vec2 UV; +layout(location = 2) in vec4 Tint; +layout(location = 3) in vec2 MaskUV; + +layout(location = 0) out vec2 uv; +layout(location = 1) out vec4 tint; + +void main() { + gl_Position = vec4(Position, 0.0, 1.0); + uv = UV; + tint = Tint; +} diff --git a/src/shaders/vulkan/NV12Frame_vert.spv b/src/shaders/vulkan/NV12Frame_vert.spv new file mode 100644 index 000000000..7f41fd1ba Binary files /dev/null and b/src/shaders/vulkan/NV12Frame_vert.spv differ diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index 94eed5db9..f784fed52 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -102,6 +102,63 @@ void FFmpegPlayer::Init() { IsInit = true; } +AVBufferRef* FFmpegPlayer::HwDecoderInit(const AVCodec* codec) { + for (int i = 0;; i++) { + const AVCodecHWConfig* cfg = avcodec_get_hw_config(codec, i); + if (!cfg) break; + if (!(cfg->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) continue; + + AVBufferRef* hw_device_ctx = NULL; + if (av_hwdevice_ctx_create(&hw_device_ctx, cfg->device_type, NULL, NULL, + 0) >= 0) { + HwVideoPixelFormat = static_cast(cfg->pix_fmt); + return hw_device_ctx; + } + } + return NULL; +} + +template +std::optional findDecoderCodec(av::Stream const& avStream) { + const AVCodecID codecId = avStream.codecParameters().codecId(); + std::optional result; + + auto checkDecode = [](av::Codec&& codec) -> std::optional { + if (!codec.canDecode()) { + const auto channel = [] { + switch (MediaType) { + case AVMEDIA_TYPE_VIDEO: + return LogChannel::Video; + case AVMEDIA_TYPE_AUDIO: + return LogChannel::Audio; + case AVMEDIA_TYPE_SUBTITLE: + return LogChannel::Subtitle; + default: + return LogChannel::General; + } + }(); + ImpLog(LogLevel::Error, channel, "Unsupported codec: {}!\n", + codec.name()); + return std::nullopt; + } + return std::optional{std::move(codec)}; + }; + + if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { +#ifdef __ANDROID__ + const AVCodecDescriptor* desc = avcodec_descriptor_get(codecId); + const std::string decoderName = fmt::format("{}_mediacodec", desc->name); + result = checkDecode(av::findDecodingCodec(decoderName)); +#endif + } + + if (!result) { + result = checkDecode(av::findDecodingCodec(codecId)); + } + + return result; +} + template void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, av::Stream&& avStream, int streamId) { @@ -110,22 +167,50 @@ void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, MediaType != AVMEDIA_TYPE_SUBTITLE) { static_assert(MediaType && false, "Unsupported MediaType"); } + std::optional codec = findDecoderCodec(avStream); + if (!codec) { + avStream.reset(); + } - const auto codec = - av::findDecodingCodec(avStream.codecParameters().codecId()); - if (!codec.canDecode()) { - ImpLog(LogLevel::Error, LogChannel::Video, "Unsupported codec!\n"); - streamOpt.reset(); + DecodingContext_t decoderContext{avStream, *codec}; + if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { + if (auto* ctx = HwDecoderInit(decoderContext.codec().raw()); ctx) { + decoderContext.raw()->hw_device_ctx = ctx; + decoderContext.raw()->opaque = this; + decoderContext.raw()->get_format = + [](AVCodecContext * ctx, + const enum AVPixelFormat* pix_fmts) -> enum AVPixelFormat { + auto* self = static_cast(ctx->opaque); + for (const auto* p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { + if (*p == self->HwVideoPixelFormat) return *p; + } + for (const auto* p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { + const AVPixFmtDescriptor* desc = av_pix_fmt_desc_get(*p); + if (!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) return *p; + } + return AV_PIX_FMT_NONE; + }; + } } std::error_code ec; - - DecodingContext_t decoderContext{avStream, codec}; - decoderContext.open({{{"threads", "auto"}}}, ec); + decoderContext.open({{{"threads", "auto"}}}, decoderContext.codec(), ec); decoderContext.setRefCountedFrames(true); if (ec) { - ImpLog(LogLevel::Error, LogChannel::Audio, - "Failed to open codec, error: {:s}", ec.message()); + const auto channel = [] { + switch (MediaType) { + case AVMEDIA_TYPE_VIDEO: + return LogChannel::Video; + case AVMEDIA_TYPE_AUDIO: + return LogChannel::Audio; + case AVMEDIA_TYPE_SUBTITLE: + return LogChannel::Subtitle; + default: + return LogChannel::General; + } + }(); + ImpLog(LogLevel::Error, channel, "Failed to open codec, error: {:s}", + ec.message()); } double rate = 1; @@ -214,14 +299,25 @@ void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) { videoStreamId); ScrWork[SW_MOVIEFRAME] = 0; ScrWork[SW_MOVIETOTALFRAME] = VideoStream->Duration; - if (!VideoTexture) - VideoTexture = - Renderer->CreateYUVFrame((float)VideoStream->CodecContext.width(), - (float)VideoStream->CodecContext.height()); - else { - VideoTexture->Width = (float)VideoStream->CodecContext.width(); - VideoTexture->Height = (float)VideoStream->CodecContext.height(); - } + std::visit( + [this](auto& videoText) { + if constexpr (std::is_same_v, + std::monostate>) { + if (HwVideoPixelFormat == AV_PIX_FMT_NONE) { + VideoTexture = Renderer->CreateYUVFrame( + (float)VideoStream->CodecContext.width(), + (float)VideoStream->CodecContext.height()); + } else { + VideoTexture = Renderer->CreateNV12Frame( + (float)VideoStream->CodecContext.width(), + (float)VideoStream->CodecContext.height()); + } + } else { + videoText->Width = (float)VideoStream->CodecContext.width(); + videoText->Height = (float)VideoStream->CodecContext.height(); + } + }, + VideoTexture); } if (audioStream.isAudio() && audioStream.isValid()) { OpenCodec(AudioStream, std::move(audioStream), @@ -261,12 +357,17 @@ void FFmpegPlayer::InitSubtitles( using Profile::Subtitle::SubtitleType; using namespace Subtitle; auto initSubPlayer = [this] { - if (!SubPlayer) { - if (VideoTexture) - SubPlayer.emplace(VideoTexture->Width, VideoTexture->Height); - else - SubPlayer.emplace(Profile::DesignWidth, Profile::DesignHeight); - } + if (SubPlayer) return; + std::visit( + [this](auto& videoText) { + if constexpr (std::is_same_v, + std::monostate>) { + SubPlayer.emplace(Profile::DesignWidth, Profile::DesignHeight); + } else { + SubPlayer.emplace(videoText->Width, videoText->Height); + } + }, + VideoTexture); }; auto mappingsItr = SubtitleMappings.find(StreamPtr->Meta.FileName); @@ -408,6 +509,23 @@ void FFmpegPlayer::Read() { } } +void FFmpegPlayer::ProcessVideoFrame(Frame_t& avFrame) { + auto rawFrame = avFrame.raw(); + if (rawFrame->format == AV_PIX_FMT_NONE || + !(av_pix_fmt_desc_get(avFrame.pixelFormat())->flags & + AV_PIX_FMT_FLAG_HWACCEL)) { + return; + } + AVFrame* swFrame = av_frame_alloc(); + if (av_hwframe_transfer_data(swFrame, rawFrame, 0) < 0) { + av_frame_free(&swFrame); + return; + } + av_frame_copy_props(swFrame, rawFrame); + av_frame_free(&rawFrame); + avFrame.reset(swFrame); +} + template void FFmpegPlayer::Decode(FFmpegStream& stream) { auto verifyPacket = @@ -460,21 +578,33 @@ void FFmpegPlayer::Decode(FFmpegStream& stream) { AVPacketItem packet = verifyPacket(peek); std::error_code ec; - if (packet.Serial != INT32_MIN) { - Frame_t frame = stream.CodecContext.decode(packet.Packet, ec); + auto processAndPush = [&](Frame_t&& f) { if (ec) { ImpLog(LogLevel::Error, LogChannel::Video, "Failed to decode {:s}", ec.message()); + return false; } - if (!frame) continue; // Skip Empty Padding Frames + if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { + ProcessVideoFrame(f); + } + if (!f) return false; if constexpr (MediaType == AVMEDIA_TYPE_SUBTITLE) { - auto* rawSubtitle = frame.raw(); + auto* rawSubtitle = f.raw(); if (rawSubtitle->start_display_time == 0 && rawSubtitle->end_display_time == 0) { rawSubtitle->end_display_time = packet.Packet.duration(); } } - pushFrame(std::move(frame)); + pushFrame(std::move(f)); + return true; + }; + + if (packet.Serial != INT32_MIN) { + Frame_t frame = stream.CodecContext.decode(packet.Packet, ec); + + while (processAndPush(std::move(frame))) { + frame = stream.CodecContext.decode(av::Packet{nullptr}, ec); + } } else { // Flush decoder since packets are finished bool decode = true; @@ -523,10 +653,16 @@ void FFmpegPlayer::Stop() { PreviousFrameTimestamp = {}; FormatContext.close(); StreamPtr.reset(); - if (VideoTexture) { - VideoTexture->Release(); - VideoTexture = 0; - } + std::visit( + [this](auto& videoText) { + if constexpr (!std::is_same_v, + std::monostate>) { + videoText->Release(); + VideoTexture = std::monostate{}; + } + }, + VideoTexture); + SetFlag(SF_MOVIEPLAY, false); } } @@ -598,9 +734,15 @@ void FFmpegPlayer::Update(float dt) { } PreviousFrameTimestamp = frame->Timestamp; - - VideoTexture->Submit(frame->Frame.data(0), frame->Frame.data(1), - frame->Frame.data(2)); + if (frame->Frame.pixelFormat() == AV_PIX_FMT_NV12) { + auto& nv12Frame = std::get(VideoTexture); + nv12Frame->Submit(frame->Frame.data(0), frame->Frame.raw()->linesize[0], + frame->Frame.data(1), frame->Frame.raw()->linesize[1]); + } else if (frame->Frame.pixelFormat() == AV_PIX_FMT_YUV420P) { + auto& yuvFrame = std::get(VideoTexture); + yuvFrame->Submit(frame->Frame.data(0), frame->Frame.data(1), + frame->Frame.data(2)); + } VideoClock.Set(frame->Timestamp.toDuration(), frame->Serial); MasterClock->SyncTo(&VideoClock); @@ -661,7 +803,14 @@ void FFmpegPlayer::Render(float videoAlpha) { const RectF dest = {0.0f, 0.0f, Profile::DesignWidth, Profile::DesignHeight}; const glm::vec4 tint = {1.0f, 1.0f, 1.0f, videoAlpha}; - Renderer->DrawVideoTexture(*VideoTexture, dest, tint, IsAlpha); + std::visit( + [this, &dest, &tint](auto& videoText) { + if constexpr (!std::is_same_v, + std::monostate>) { + Renderer->DrawVideoTexture(*videoText, dest, tint, IsAlpha); + } + }, + VideoTexture); if (SubPlayer) SubPlayer->Render(); } diff --git a/src/video/ffmpegplayer.h b/src/video/ffmpegplayer.h index 976339128..2ba67ccd8 100644 --- a/src/video/ffmpegplayer.h +++ b/src/video/ffmpegplayer.h @@ -73,6 +73,8 @@ class FFmpegPlayer : public VideoPlayer { av::Stream&& avStream, int streamId); void UpdateSubtitles(); + void ProcessVideoFrame(Frame_t& avFrame); + AVBufferRef* HwDecoderInit(const AVCodec* codec); static int constexpr FILESTREAMBUFFERSZ = 64 * 8192; std::condition_variable ReadCond; @@ -82,16 +84,17 @@ class FFmpegPlayer : public VideoPlayer { std::unique_ptr StreamPtr; av::FormatContext FormatContext; + AVPixelFormat HwVideoPixelFormat = AV_PIX_FMT_NONE; FFmpegFileIO IoContext; - Clock VideoClock; + Clock VideoClock; Clock* MasterClock{}; std::unique_ptr AudioPlayer; bool IsInit = false; - YUVFrame* VideoTexture; + std::variant VideoTexture; bool IsAlpha = false; bool Looping = false; diff --git a/vcpkg.json b/vcpkg.json index d7c4ee2cf..b154f3227 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -35,17 +35,9 @@ "name": "ffmpeg", "default-features": false, "features": [ - "amf" + "vaapi" ], - "platform": "!osx" - }, - { - "name": "ffmpeg", - "default-features": false, - "features": [ - "nvcodec" - ], - "platform": "linux | windows" + "platform": "linux" }, "libwebp", "fmt",