From a816a8ff6dddccdc61f18aa8031e4a4fceeedcf0 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sun, 29 Mar 2026 18:20:44 -0700 Subject: [PATCH 1/8] Hardware accelerated video decoding --- CMakeLists.txt | 8 +- src/renderer/dx9/nv12frame.cpp | 36 ++++++ src/renderer/dx9/nv12frame.h | 29 +++++ src/renderer/dx9/renderer.cpp | 57 +++++++++- src/renderer/dx9/renderer.h | 9 +- src/renderer/nv12frame.h | 20 ++++ src/renderer/opengl/nv12frame.cpp | 39 +++++++ src/renderer/opengl/nv12frame.h | 17 +++ src/renderer/opengl/renderer.cpp | 37 +++++++ src/renderer/opengl/renderer.h | 4 + src/renderer/opengl/shader.cpp | 19 ++++ src/renderer/opengl/shader.h | 24 ++++ src/renderer/renderer.h | 4 + src/renderer/vulkan/nv12frame.cpp | 146 +++++++++++++++++++++++++ src/renderer/vulkan/nv12frame.h | 30 +++++ src/renderer/vulkan/renderer.cpp | 77 ++++++++++++- src/renderer/vulkan/renderer.h | 9 +- src/shaders/dx9/NV12Frame_frag.hlsl | 37 +++++++ src/shaders/dx9/NV12Frame_vert.hlsl | 24 ++++ src/shaders/opengl/NV12Frame_frag.glsl | 36 ++++++ src/shaders/opengl/NV12Frame_vert.glsl | 14 +++ src/shaders/vulkan/NV12Frame_frag.glsl | 38 +++++++ src/shaders/vulkan/NV12Frame_frag.spv | Bin 0 -> 2572 bytes src/shaders/vulkan/NV12Frame_vert.glsl | 16 +++ src/shaders/vulkan/NV12Frame_vert.spv | Bin 0 -> 1164 bytes src/video/ffmpegplayer.cpp | 131 ++++++++++++++++++---- src/video/ffmpegplayer.h | 7 +- 27 files changed, 826 insertions(+), 42 deletions(-) create mode 100644 src/renderer/dx9/nv12frame.cpp create mode 100644 src/renderer/dx9/nv12frame.h create mode 100644 src/renderer/nv12frame.h create mode 100644 src/renderer/opengl/nv12frame.cpp create mode 100644 src/renderer/opengl/nv12frame.h create mode 100644 src/renderer/vulkan/nv12frame.cpp create mode 100644 src/renderer/vulkan/nv12frame.h create mode 100644 src/shaders/dx9/NV12Frame_frag.hlsl create mode 100644 src/shaders/dx9/NV12Frame_vert.hlsl create mode 100644 src/shaders/opengl/NV12Frame_frag.glsl create mode 100644 src/shaders/opengl/NV12Frame_vert.glsl create mode 100644 src/shaders/vulkan/NV12Frame_frag.glsl create mode 100644 src/shaders/vulkan/NV12Frame_frag.spv create mode 100644 src/shaders/vulkan/NV12Frame_vert.glsl create mode 100644 src/shaders/vulkan/NV12Frame_vert.spv diff --git a/CMakeLists.txt b/CMakeLists.txt index 518276aed..b9ed4ee0c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ set(Impacto_Header src/renderer/renderer.h src/renderer/window.h src/renderer/yuvframe.h + src/renderer/nv12frame.h src/data/savesystem.h src/data/tipssystem.h @@ -1124,6 +1125,7 @@ if (NOT DEFINED IMPACTO_DISABLE_OPENGL) src/renderer/opengl/shader.cpp src/renderer/opengl/glc.cpp src/renderer/opengl/yuvframe.cpp + src/renderer/opengl/nv12frame.cpp src/renderer/opengl/3d/renderable3d.cpp src/renderer/opengl/3d/scene.cpp @@ -1135,6 +1137,7 @@ if (NOT DEFINED IMPACTO_DISABLE_OPENGL) src/renderer/opengl/shader.h src/renderer/opengl/glc.h src/renderer/opengl/yuvframe.h + src/renderer/opengl/nv12frame.h src/renderer/opengl/3d/renderable3d.h src/renderer/opengl/3d/scene.h ) @@ -1219,6 +1222,7 @@ if (NOT DEFINED IMPACTO_DISABLE_VULKAN) src/renderer/vulkan/pipeline.cpp src/renderer/vulkan/utils.cpp src/renderer/vulkan/yuvframe.cpp + src/renderer/vulkan/nv12frame.cpp src/renderer/vulkan/3d/renderable3d.cpp src/renderer/vulkan/3d/scene.cpp ) @@ -1227,7 +1231,7 @@ if (NOT DEFINED IMPACTO_DISABLE_VULKAN) src/renderer/vulkan/renderer.h src/renderer/vulkan/pipeline.h src/renderer/vulkan/utils.h - src/renderer/vulkan/yuvframe.h + src/renderer/vulkan/nv12frame.h src/renderer/vulkan/3d/renderable3d.h src/renderer/vulkan/3d/scene.h ) @@ -1246,6 +1250,7 @@ if (NOT DEFINED IMPACTO_DISABLE_DX9) src/renderer/dx9/window.cpp src/renderer/dx9/shader.cpp src/renderer/dx9/yuvframe.cpp + src/renderer/dx9/nv12frame.cpp src/renderer/dx9/3d/scene.cpp src/renderer/dx9/3d/renderable3d.cpp ) @@ -1255,6 +1260,7 @@ if (NOT DEFINED IMPACTO_DISABLE_DX9) src/renderer/dx9/window.h src/renderer/dx9/shader.h src/renderer/dx9/yuvframe.h + src/renderer/dx9/nv12frame.h src/renderer/dx9/3d/scene.h src/renderer/dx9/3d/renderable3d.h ) diff --git a/src/renderer/dx9/nv12frame.cpp b/src/renderer/dx9/nv12frame.cpp new file mode 100644 index 000000000..9e63723a8 --- /dev/null +++ b/src/renderer/dx9/nv12frame.cpp @@ -0,0 +1,36 @@ +#include "nv12frame.h" + +#include "../../log.h" + +namespace Impacto { +namespace DirectX9 { + +DX9NV12Frame::DX9NV12Frame(IDirect3DDevice9* device) { Device = device; } + +void DX9NV12Frame::Init(float width, float height) { + Width = width; + Height = height; + auto err = Device->CreateTexture((UINT)Width, (UINT)Height, 1, 0, D3DFMT_A8, + D3DPOOL_MANAGED, &Luma, nullptr); + err = Device->CreateTexture((UINT)(Width / 2), (UINT)(Height / 2), 1, 0, + D3DFMT_A8L8, D3DPOOL_MANAGED, &CbCr, nullptr); +} + +void DX9NV12Frame::Submit(const void* luma, const void* cbcr) { + D3DLOCKED_RECT lockRect; + auto err = Luma->LockRect(0, &lockRect, NULL, 0); + memcpy(lockRect.pBits, luma, (size_t)(Width * Height)); + err = Luma->UnlockRect(0); + + err = CbCr->LockRect(0, &lockRect, NULL, 0); + memcpy(lockRect.pBits, cbcr, (size_t)((Width) * (Height / 2))); + err = CbCr->UnlockRect(0); +} + +void DX9NV12Frame::Release() { + Luma->Release(); + CbCr->Release(); +} + +} // namespace DirectX9 +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/dx9/nv12frame.h b/src/renderer/dx9/nv12frame.h new file mode 100644 index 000000000..59895370c --- /dev/null +++ b/src/renderer/dx9/nv12frame.h @@ -0,0 +1,29 @@ +#pragma once + +#include "../nv12frame.h" + +#include + +namespace Impacto { +namespace DirectX9 { + +class DX9NV12Frame : public NV12Frame { + friend class Renderer; + + public: + DX9NV12Frame(IDirect3DDevice9* device); + + void Init(float width, float height) override; + + void Submit(const void* luma, const void* cbcr) override; + void Release() override; + + protected: + IDirect3DDevice9* Device; + + IDirect3DTexture9* Luma; + IDirect3DTexture9* CbCr; +}; + +} // namespace DirectX9 +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/dx9/renderer.cpp b/src/renderer/dx9/renderer.cpp index cb90f393f..196406bb5 100644 --- a/src/renderer/dx9/renderer.cpp +++ b/src/renderer/dx9/renderer.cpp @@ -82,6 +82,8 @@ void Renderer::Init() { vertexDeclaration); ShaderYUVFrame = new Shader(); ShaderYUVFrame->Compile("YUVFrame", Device, vertexDeclaration); + ShaderNV12Frame = new Shader(); + ShaderNV12Frame->Compile("NV12Frame", Device, vertexDeclaration); ShaderCCMessageBox = new Shader(); ShaderCCMessageBox->Compile("CCMessageBoxSprite", Device, vertexDeclaration); ShaderCHLCCMenuBackground = new Shader(); @@ -258,9 +260,15 @@ void Renderer::FreeTexture(uint32_t id) { } YUVFrame* Renderer::CreateYUVFrame(float width, float height) { - VideoFrameInternal = new DX9YUVFrame(Device); - VideoFrameInternal->Init(width, height); - return (YUVFrame*)VideoFrameInternal; + VideoFrameInternalYUV = new DX9YUVFrame(Device); + VideoFrameInternalYUV->Init(width, height); + return (YUVFrame*)VideoFrameInternalYUV; +} + +NV12Frame* Renderer::CreateNV12Frame(float width, float height) { + VideoFrameInternalNV12 = new DX9NV12Frame(Device); + VideoFrameInternalNV12->Init(width, height); + return (NV12Frame*)VideoFrameInternalNV12; } void Renderer::DrawSprite(const Sprite& sprite, const CornersQuad& dest, @@ -687,9 +695,46 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, // Do we have space for one more sprite quad? EnsureSpaceAvailable(4, sizeof(VertexBufferSprites), 6); - auto err = Device->SetTexture(0, VideoFrameInternal->Luma); - err = Device->SetTexture(1, VideoFrameInternal->Cb); - err = Device->SetTexture(2, VideoFrameInternal->Cr); + auto err = Device->SetTexture(0, VideoFrameInternalYUV->Luma); + err = Device->SetTexture(1, VideoFrameInternalYUV->Cb); + err = Device->SetTexture(2, VideoFrameInternalYUV->Cr); + + // This is cursed man, idk + BOOL alphaVideoB = (BOOL)alphaVideo; + Device->SetPixelShaderConstantB(0, &alphaVideoB, 1); + + // OK, all good, make quad + + VertexBufferSprites* vertices = + (VertexBufferSprites*)(VertexBuffer + VertexBufferFill); + VertexBufferFill += 4 * sizeof(VertexBufferSprites); + + IndexBufferFill += 6; + + QuadSetUV(RectF(0.0f, 0.0f, frame.Width, frame.Height), + {frame.Width, frame.Height}, &vertices[0].UV, + sizeof(VertexBufferSprites)); + QuadSetPosition(dest, &vertices[0].Position, sizeof(VertexBufferSprites)); + + for (int i = 0; i < 4; i++) vertices[i].Tint = tint; +} + +void Renderer::DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + const glm::vec4 tint, const bool alphaVideo) { + if (!Drawing) { + ImpLog(LogLevel::Error, LogChannel::Render, + "Renderer->DrawVideoTexture() called before BeginFrame()\n"); + return; + } + + EnsureShader(ShaderNV12Frame); + CurrentTexture = std::numeric_limits::max(); + + // Do we have space for one more sprite quad? + EnsureSpaceAvailable(4, sizeof(VertexBufferSprites), 6); + + auto err = Device->SetTexture(0, VideoFrameInternalNV12->Luma); + err = Device->SetTexture(1, VideoFrameInternalNV12->CbCr); // This is cursed man, idk BOOL alphaVideoB = (BOOL)alphaVideo; diff --git a/src/renderer/dx9/renderer.h b/src/renderer/dx9/renderer.h index e9aa522ff..9e9be3039 100644 --- a/src/renderer/dx9/renderer.h +++ b/src/renderer/dx9/renderer.h @@ -4,6 +4,7 @@ #include "window.h" #include "shader.h" #include "yuvframe.h" +#include "nv12frame.h" #include @@ -33,6 +34,7 @@ class Renderer : public BaseRenderer { } void FreeTexture(uint32_t id) override; YUVFrame* CreateYUVFrame(float width, float height) override; + NV12Frame* CreateNV12Frame(float width, float height) override; void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, std::span tints, @@ -91,7 +93,8 @@ class Renderer : public BaseRenderer { void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo) override; - + void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo = false) override; void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, const glm::mat4 transformation, const glm::vec4 tint) override {}; // TODO: Implement @@ -134,13 +137,15 @@ class Renderer : public BaseRenderer { Shader* ShaderMaskedSprite; Shader* ShaderMaskedSpriteNoAlpha; Shader* ShaderYUVFrame; + Shader* ShaderNV12Frame; Shader* ShaderCCMessageBox; Shader* ShaderCHLCCMenuBackground; uint32_t CurrentTexture = 0; uint32_t NextTextureId = 1; - DX9YUVFrame* VideoFrameInternal; + DX9YUVFrame* VideoFrameInternalYUV; + DX9NV12Frame* VideoFrameInternalNV12; bool Drawing = false; diff --git a/src/renderer/nv12frame.h b/src/renderer/nv12frame.h new file mode 100644 index 000000000..b69b3d129 --- /dev/null +++ b/src/renderer/nv12frame.h @@ -0,0 +1,20 @@ +#pragma once + +#include "../impacto.h" + +namespace Impacto { + +class NV12Frame { + public: + float Width; + float Height; + uint32_t LumaId; + uint32_t CbCrId; + + virtual void Init(float width, float height) = 0; + + virtual void Submit(const void* luma, const void* cbcr) = 0; + virtual void Release() = 0; +}; + +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/opengl/nv12frame.cpp b/src/renderer/opengl/nv12frame.cpp new file mode 100644 index 000000000..40134b0ef --- /dev/null +++ b/src/renderer/opengl/nv12frame.cpp @@ -0,0 +1,39 @@ +#include "nv12frame.h" +#include "../../log.h" + +#include + +namespace Impacto { +namespace OpenGL { + +void GLNV12Frame::Init(float width, float height) { + Width = width; + Height = height; + GLuint yuv[2]; + + glGenTextures(2, yuv); + LumaId = yuv[0]; + CbCrId = yuv[1]; +} + +void GLNV12Frame::Submit(const void* luma, const void* cbcr) { + glBindTexture(GL_TEXTURE_2D, LumaId); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, (GLsizei)Width, (GLsizei)Height, 0, + GL_RED, GL_UNSIGNED_BYTE, luma); + glGenerateMipmap(GL_TEXTURE_2D); + + glBindTexture(GL_TEXTURE_2D, CbCrId); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RG, (GLsizei)(Width / 2), + (GLsizei)(Height / 2), 0, GL_RG, GL_UNSIGNED_BYTE, cbcr); + glGenerateMipmap(GL_TEXTURE_2D); +} + +void GLNV12Frame::Release() { + GLuint yuv[2]; + yuv[0] = LumaId; + yuv[1] = CbCrId; + glDeleteTextures(2, yuv); +} + +} // namespace OpenGL +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/opengl/nv12frame.h b/src/renderer/opengl/nv12frame.h new file mode 100644 index 000000000..c5526d42f --- /dev/null +++ b/src/renderer/opengl/nv12frame.h @@ -0,0 +1,17 @@ +#pragma once + +#include "../nv12frame.h" + +namespace Impacto { +namespace OpenGL { + +class GLNV12Frame : public NV12Frame { + public: + void Init(float width, float height) override; + + void Submit(const void* luma, const void* cbcr) override; + void Release() override; +}; + +} // namespace OpenGL +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/opengl/renderer.cpp b/src/renderer/opengl/renderer.cpp index 61b6570d4..674695f08 100644 --- a/src/renderer/opengl/renderer.cpp +++ b/src/renderer/opengl/renderer.cpp @@ -8,6 +8,7 @@ #include "../../log.h" #include "3d/scene.h" #include "yuvframe.h" +#include "nv12frame.h" #ifndef IMPACTO_DISABLE_IMGUI #include @@ -100,6 +101,7 @@ void Renderer::Init() { GaussianBlurShaderProgram.emplace(Shaders.Compile("GaussianBlur")); MosaicShaderProgram.emplace(Shaders.Compile("Mosaic")); SubtitleGlyphShaderProgram.emplace(Shaders.Compile("SubtitleGlyph")); + NV12FrameShaderProgram.emplace(Shaders.Compile("NV12Frame")); glGenSamplers((GLsizei)Samplers.size(), Samplers.data()); for (size_t i = 0; i < TextureUnitCount; i++) { @@ -261,6 +263,12 @@ YUVFrame* Renderer::CreateYUVFrame(float width, float height) { return (YUVFrame*)frame; } +NV12Frame* Renderer::CreateNV12Frame(float width, float height) { + auto frame = new GLNV12Frame(); + frame->Init(width, height); + return (NV12Frame*)frame; +} + void Renderer::InsertVertices( const std::span vertices, const std::span indices) { @@ -1014,6 +1022,35 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, InsertVerticesQuad(dest, RectF(0.0f, 0.0f, 1.0f, 1.0f), tint); } +void Renderer::DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + const glm::vec4 tint, const bool alphaVideo) { + if (!Drawing) { + ImpLog(LogLevel::Error, LogChannel::Render, + "Renderer->DrawVideoTexture() called before BeginFrame()\n"); + return; + } + + EnsureTopologyMode(TopologyMode::Triangles); + + UseTextures(std::array, 2>{ + std::pair{frame.LumaId, 0}, + std::pair{frame.CbCrId, 1}, + }); + + NV12FrameUniforms uniforms{ + .Projection = Projection, + .Luma = 0, + .CbCr = 1, + .IsAlpha = alphaVideo, + }; + + UseShader(*NV12FrameShaderProgram, uniforms); + + // OK, all good, make quad + + InsertVerticesQuad(dest, RectF(0.0f, 0.0f, 1.0f, 1.0f), tint); +} + void Renderer::CaptureScreencap(Sprite& sprite) { Flush(); sprite.Sheet.IsScreenCap = true; diff --git a/src/renderer/opengl/renderer.h b/src/renderer/opengl/renderer.h index 570b0e581..d0b0d7308 100644 --- a/src/renderer/opengl/renderer.h +++ b/src/renderer/opengl/renderer.h @@ -35,6 +35,7 @@ class Renderer : public BaseRenderer { std::span outBuffer) override; void FreeTexture(uint32_t id) override; YUVFrame* CreateYUVFrame(float width, float height) override; + NV12Frame* CreateNV12Frame(float width, float height) override; void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, std::span tints, @@ -92,6 +93,8 @@ class Renderer : public BaseRenderer { void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo) override; + void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo) override; void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, const glm::mat4 transformation, const glm::vec4 tint) override; @@ -144,6 +147,7 @@ class Renderer : public BaseRenderer { std::optional GaussianBlurShaderProgram; std::optional MosaicShaderProgram; std::optional SubtitleGlyphShaderProgram; + std::optional NV12FrameShaderProgram; const void* CurrentShaderProgram = nullptr; diff --git a/src/renderer/opengl/shader.cpp b/src/renderer/opengl/shader.cpp index 81a975655..6e0570dc9 100644 --- a/src/renderer/opengl/shader.cpp +++ b/src/renderer/opengl/shader.cpp @@ -768,5 +768,24 @@ void SubtitleGlyphShader::UploadUniforms(SubtitleGlyphUniforms newUniforms) { UpdateVar(newUniforms.CoverageMap, Uniforms.CoverageMap, CoverageMapLocation); } +NV12FrameShader::NV12FrameShader(GLint programId) + : Shader(programId), + ProjectionLocation(glGetUniformLocation(programId, "Projection")), + LumaLocation(glGetUniformLocation(programId, "Luma")), + CbCrLocation(glGetUniformLocation(programId, "CbCr")), + IsAlphaLocation(glGetUniformLocation(programId, "IsAlpha")) { + UploadVar(Uniforms.Projection, ProjectionLocation); + UploadVar(Uniforms.Luma, LumaLocation); + UploadVar(Uniforms.CbCr, CbCrLocation); + UploadVar(Uniforms.IsAlpha, IsAlphaLocation); +} + +void NV12FrameShader::UploadUniforms(NV12FrameUniforms newUniforms) { + UpdateVar(newUniforms.Projection, Uniforms.Projection, ProjectionLocation); + UpdateVar(newUniforms.Luma, Uniforms.Luma, LumaLocation); + UpdateVar(newUniforms.CbCr, Uniforms.CbCr, CbCrLocation); + UpdateVar(newUniforms.IsAlpha, Uniforms.IsAlpha, IsAlphaLocation); +} + } // namespace OpenGL } // namespace Impacto diff --git a/src/renderer/opengl/shader.h b/src/renderer/opengl/shader.h index 837b11ea3..be02380a6 100644 --- a/src/renderer/opengl/shader.h +++ b/src/renderer/opengl/shader.h @@ -644,5 +644,29 @@ class SubtitleGlyphShader : public Shader { const GLint CoverageMapLocation; }; +struct NV12FrameUniforms { + bool operator==(const NV12FrameUniforms& other) const = default; + + glm::mat4 Projection; + + GLint Luma = 0; + GLint CbCr = 0; + bool IsAlpha = false; +}; + +class NV12FrameShader : public Shader { + public: + NV12FrameShader(GLint programId); + + void UploadUniforms(NV12FrameUniforms uniforms) override; + + private: + const GLint ProjectionLocation; + + const GLint LumaLocation; + const GLint CbCrLocation; + const GLint IsAlphaLocation; +}; + } // namespace OpenGL } // namespace Impacto \ No newline at end of file diff --git a/src/renderer/renderer.h b/src/renderer/renderer.h index d077d7b0a..f48659364 100644 --- a/src/renderer/renderer.h +++ b/src/renderer/renderer.h @@ -6,6 +6,7 @@ #include "../spritesheet.h" #include "../text/text.h" #include "yuvframe.h" +#include "nv12frame.h" #include namespace Impacto { @@ -77,6 +78,7 @@ class BaseRenderer { std::span outBuffer) = 0; virtual void FreeTexture(uint32_t id) = 0; virtual YUVFrame* CreateYUVFrame(float width, float height) = 0; + virtual NV12Frame* CreateNV12Frame(float width, float height) = 0; virtual void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, @@ -372,6 +374,8 @@ class BaseRenderer { virtual void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo = false) = 0; + virtual void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo = false) = 0; virtual void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, glm::vec4 tint) = 0; diff --git a/src/renderer/vulkan/nv12frame.cpp b/src/renderer/vulkan/nv12frame.cpp new file mode 100644 index 000000000..eabbc4704 --- /dev/null +++ b/src/renderer/vulkan/nv12frame.cpp @@ -0,0 +1,146 @@ +#include "nv12frame.h" + +namespace Impacto { +namespace Vulkan { + +void VkNV12Frame::Init(float width, float height) { + Width = width; + Height = height; + + VkDeviceSize imageSize = (VkDeviceSize)(width * height); + VkDeviceSize bufferSize = + imageSize + 2 * (((VkDeviceSize)width / 2) * ((VkDeviceSize)height / 2)); + + StagingBuffer = CreateBuffer(bufferSize, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, + VMA_MEMORY_USAGE_CPU_ONLY); + vmaMapMemory(Allocator, StagingBuffer.Allocation, &MappedStagingBuffer); + + VkExtent3D lumaImageExtent; + lumaImageExtent.width = static_cast(width); + lumaImageExtent.height = static_cast(height); + lumaImageExtent.depth = 1; + + VkExtent3D cbCrImageExtent; + cbCrImageExtent.width = static_cast(width / 2); + cbCrImageExtent.height = static_cast(height / 2); + cbCrImageExtent.depth = 1; + + auto lumaImageInfo = + GetImageCreateInfo(VK_FORMAT_R8_UNORM, lumaImageExtent, + VkSampleCountFlagBits::VK_SAMPLE_COUNT_1_BIT); + VmaAllocationCreateInfo dimgAllocinfo = {}; + dimgAllocinfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + vmaCreateImage(Allocator, &lumaImageInfo, &dimgAllocinfo, + &LumaImage.Image.Image, &LumaImage.Image.Allocation, nullptr); + auto cbcrImageInfo = + GetImageCreateInfo(VK_FORMAT_R8G8_UNORM, cbCrImageExtent, + VkSampleCountFlagBits::VK_SAMPLE_COUNT_1_BIT); + vmaCreateImage(Allocator, &cbcrImageInfo, &dimgAllocinfo, + &CbCrImage.Image.Image, &CbCrImage.Image.Allocation, nullptr); + + VkImageViewCreateInfo imageInfo = {}; + imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + imageInfo.pNext = nullptr; + imageInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + imageInfo.image = LumaImage.Image.Image; + imageInfo.format = VK_FORMAT_R8_UNORM; + imageInfo.subresourceRange.baseMipLevel = 0; + imageInfo.subresourceRange.levelCount = 1; + imageInfo.subresourceRange.baseArrayLayer = 0; + imageInfo.subresourceRange.layerCount = 1; + imageInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + + vkCreateImageView(MainUploadContext.Device, &imageInfo, nullptr, + &LumaImage.ImageView); + + imageInfo.image = CbCrImage.Image.Image; + imageInfo.format = VK_FORMAT_R8G8_UNORM; + vkCreateImageView(MainUploadContext.Device, &imageInfo, nullptr, + &CbCrImage.ImageView); +} + +void VkNV12Frame::Submit(const void* luma, const void* cbcr) { + int cbcrOffset = (int)(Width * Height); + + uint8_t* mappedStagingBuffer = (uint8_t*)MappedStagingBuffer; + + memcpy(mappedStagingBuffer, luma, cbcrOffset); + memcpy(mappedStagingBuffer + cbcrOffset, cbcr, + (size_t)((Width) * (Height / 2))); + + ImmediateSubmit([&](VkCommandBuffer cmd) { + VkExtent3D lumaImageExtent; + lumaImageExtent.width = static_cast(Width); + lumaImageExtent.height = static_cast(Height); + lumaImageExtent.depth = 1; + VkExtent3D cbCrImageExtent; + cbCrImageExtent.width = static_cast(Width / 2); + cbCrImageExtent.height = static_cast(Height / 2); + cbCrImageExtent.depth = 1; + + VkImageSubresourceRange range; + range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + range.baseMipLevel = 0; + range.levelCount = 1; + range.baseArrayLayer = 0; + range.layerCount = 1; + + VkImageMemoryBarrier imageBarrierToTransfer = {}; + imageBarrierToTransfer.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + imageBarrierToTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imageBarrierToTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + imageBarrierToTransfer.image = LumaImage.Image.Image; + imageBarrierToTransfer.subresourceRange = range; + imageBarrierToTransfer.srcAccessMask = 0; + imageBarrierToTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 1, &imageBarrierToTransfer); + + VkBufferImageCopy copyRegion = {}; + copyRegion.bufferOffset = 0; + copyRegion.bufferRowLength = 0; + copyRegion.bufferImageHeight = 0; + copyRegion.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copyRegion.imageSubresource.mipLevel = 0; + copyRegion.imageSubresource.baseArrayLayer = 0; + copyRegion.imageSubresource.layerCount = 1; + copyRegion.imageExtent = lumaImageExtent; + vkCmdCopyBufferToImage(cmd, StagingBuffer.Buffer, LumaImage.Image.Image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, + ©Region); + + VkImageMemoryBarrier imageBarrierToReadable = imageBarrierToTransfer; + imageBarrierToReadable.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + imageBarrierToReadable.newLayout = VK_IMAGE_LAYOUT_GENERAL; + imageBarrierToReadable.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + imageBarrierToReadable.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, + 0, nullptr, 1, &imageBarrierToReadable); + + imageBarrierToTransfer.image = CbCrImage.Image.Image; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 1, &imageBarrierToTransfer); + + copyRegion.bufferOffset = (VkDeviceSize)(Width * Height); + copyRegion.imageExtent = cbCrImageExtent; + vkCmdCopyBufferToImage(cmd, StagingBuffer.Buffer, CbCrImage.Image.Image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, + ©Region); + + imageBarrierToReadable.image = CbCrImage.Image.Image; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, + 0, nullptr, 1, &imageBarrierToReadable); + }); +} + +void VkNV12Frame::Release() { + vmaUnmapMemory(Allocator, StagingBuffer.Allocation); +} + +} // namespace Vulkan +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/vulkan/nv12frame.h b/src/renderer/vulkan/nv12frame.h new file mode 100644 index 000000000..715fbe968 --- /dev/null +++ b/src/renderer/vulkan/nv12frame.h @@ -0,0 +1,30 @@ +#pragma once + +#include "../nv12frame.h" +#include "utils.h" + +#include + +namespace Impacto { +namespace Vulkan { + +class VkNV12Frame : public NV12Frame { + friend class Renderer; + + public: + void Init(float width, float height) override; + + void Submit(const void* luma, const void* cbcr) override; + void Release() override; + + protected: + VkTexture LumaImage{}; + VkTexture CbCrImage{}; + + private: + AllocatedBuffer StagingBuffer; + void* MappedStagingBuffer; +}; + +} // namespace Vulkan +} // namespace Impacto \ No newline at end of file diff --git a/src/renderer/vulkan/renderer.cpp b/src/renderer/vulkan/renderer.cpp index 26534d6a4..f3290d4ae 100644 --- a/src/renderer/vulkan/renderer.cpp +++ b/src/renderer/vulkan/renderer.cpp @@ -8,6 +8,7 @@ #include "../../log.h" #include "3d/scene.h" #include "yuvframe.h" +#include "nv12frame.h" #ifndef IMPACTO_DISABLE_IMGUI #include @@ -737,6 +738,11 @@ void Renderer::Init() { PipelineYUVFrame->CreateWithShader( "YUVFrame", bindingDescription, attributeDescriptions.data(), attributeDescriptions.size(), TripleTextureSetLayout); + PipelineNV12Frame = new Pipeline(Device, RenderPass); + PipelineNV12Frame->SetPushConstants(&yuvFramePushConstants, 1); + PipelineNV12Frame->CreateWithShader( + "NV12Frame", bindingDescription, attributeDescriptions.data(), + attributeDescriptions.size(), TripleTextureSetLayout); VkPushConstantRange ccBoxPushConstant; ccBoxPushConstant.offset = 0; @@ -1128,9 +1134,15 @@ void Renderer::FreeTexture(uint32_t id) { } YUVFrame* Renderer::CreateYUVFrame(float width, float height) { - VideoFrameInternal = new VkYUVFrame(); - VideoFrameInternal->Init(width, height); - return (YUVFrame*)VideoFrameInternal; + VideoFrameInternalYUV = new VkYUVFrame(); + VideoFrameInternalYUV->Init(width, height); + return (YUVFrame*)VideoFrameInternalYUV; +} + +NV12Frame* Renderer::CreateNV12Frame(float width, float height) { + VideoFrameInternalNV12 = new VkNV12Frame(); + VideoFrameInternalNV12->Init(width, height); + return (NV12Frame*)VideoFrameInternalNV12; } void Renderer::DrawSprite(const Sprite& sprite, const CornersQuad& dest, @@ -1679,13 +1691,13 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, VkDescriptorImageInfo imageBufferInfo[3]; imageBufferInfo[0].sampler = Sampler; - imageBufferInfo[0].imageView = VideoFrameInternal->LumaImage.ImageView; + imageBufferInfo[0].imageView = VideoFrameInternalYUV->LumaImage.ImageView; imageBufferInfo[0].imageLayout = VK_IMAGE_LAYOUT_GENERAL; imageBufferInfo[1].sampler = Sampler; - imageBufferInfo[1].imageView = VideoFrameInternal->CbImage.ImageView; + imageBufferInfo[1].imageView = VideoFrameInternalYUV->CbImage.ImageView; imageBufferInfo[1].imageLayout = VK_IMAGE_LAYOUT_GENERAL; imageBufferInfo[2].sampler = Sampler; - imageBufferInfo[2].imageView = VideoFrameInternal->CrImage.ImageView; + imageBufferInfo[2].imageView = VideoFrameInternalYUV->CrImage.ImageView; imageBufferInfo[2].imageLayout = VK_IMAGE_LAYOUT_GENERAL; VkWriteDescriptorSet writeDescriptorSet{}; @@ -1723,6 +1735,59 @@ void Renderer::DrawVideoTexture(const YUVFrame& frame, const RectF& dest, for (int i = 0; i < 4; i++) vertices[i].Tint = tint; } +void Renderer::DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + const glm::vec4 tint, const bool alphaVideo) { + if (!Drawing) { + ImpLog(LogLevel::Error, LogChannel::Render, + "Renderer->DrawVideoTexture() called before BeginFrame()\n"); + return; + } + + EnsureMode(PipelineNV12Frame); + + VkDescriptorImageInfo imageBufferInfo[2]; + imageBufferInfo[0].sampler = Sampler; + imageBufferInfo[0].imageView = VideoFrameInternalNV12->LumaImage.ImageView; + imageBufferInfo[0].imageLayout = VK_IMAGE_LAYOUT_GENERAL; + imageBufferInfo[1].sampler = Sampler; + imageBufferInfo[1].imageView = VideoFrameInternalNV12->CbCrImage.ImageView; + imageBufferInfo[1].imageLayout = VK_IMAGE_LAYOUT_GENERAL; + + VkWriteDescriptorSet writeDescriptorSet{}; + writeDescriptorSet.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writeDescriptorSet.dstSet = 0; + writeDescriptorSet.dstBinding = 0; + writeDescriptorSet.descriptorCount = 2; + writeDescriptorSet.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + writeDescriptorSet.pImageInfo = imageBufferInfo; + + vkCmdPushDescriptorSetKHR( + CommandBuffers[CurrentFrameIndex], VK_PIPELINE_BIND_POINT_GRAPHICS, + PipelineNV12Frame->PipelineLayout, 0, 1, &writeDescriptorSet); + + YUVFramePushConstants constants = {}; + constants.IsAlpha = alphaVideo; + vkCmdPushConstants(CommandBuffers[CurrentFrameIndex], + CurrentPipeline->PipelineLayout, + VK_SHADER_STAGE_FRAGMENT_BIT, 0, + sizeof(YUVFramePushConstants), &constants); + + // OK, all good, make quad + MakeQuad(); + + VertexBufferSprites* vertices = + (VertexBufferSprites*)(VertexBuffer + VertexBufferOffset + + VertexBufferFill); + VertexBufferFill += 4 * sizeof(VertexBufferSprites); + + QuadSetUV(RectF(0.0f, 0.0f, frame.Width, frame.Height), + {frame.Width, frame.Height}, &vertices[0].UV, + sizeof(VertexBufferSprites)); + QuadSetPosition(dest, &vertices[0].Position, sizeof(VertexBufferSprites)); + + for (int i = 0; i < 4; i++) vertices[i].Tint = tint; +} + void Renderer::CaptureScreencap(Sprite& sprite) { if (Textures.count(sprite.Sheet.Texture) == 0) return; sprite.Sheet.IsScreenCap = true; diff --git a/src/renderer/vulkan/renderer.h b/src/renderer/vulkan/renderer.h index 65e8591a9..974989246 100644 --- a/src/renderer/vulkan/renderer.h +++ b/src/renderer/vulkan/renderer.h @@ -8,6 +8,7 @@ #include "utils.h" #include "window.h" #include "yuvframe.h" +#include "nv12frame.h" namespace Impacto { namespace Vulkan { @@ -61,6 +62,7 @@ class Renderer : public BaseRenderer { std::span outBuffer) override; void FreeTexture(uint32_t id) override; YUVFrame* CreateYUVFrame(float width, float height) override; + NV12Frame* CreateNV12Frame(float width, float height) override; void DrawSprite(const Sprite& sprite, const CornersQuad& dest, glm::mat4 transformation, std::span tints, @@ -119,7 +121,8 @@ class Renderer : public BaseRenderer { void DrawVideoTexture(const YUVFrame& frame, const RectF& dest, glm::vec4 tint, bool alphaVideo) override; - + void DrawVideoTexture(const NV12Frame& frame, const RectF& dest, + glm::vec4 tint, bool alphaVideo = false) override; void DrawSubtitleGlyph(const Sprite& sprite, const CornersQuad& dest, const glm::mat4 transformation, const glm::vec4 tint) override {}; // TODO: Implement @@ -220,6 +223,7 @@ class Renderer : public BaseRenderer { Pipeline* PipelineMaskedSprite; Pipeline* PipelineMaskedSpriteNoAlpha; Pipeline* PipelineYUVFrame; + Pipeline* PipelineNV12Frame; Pipeline* PipelineCCMessageBox; Pipeline* PipelineCHLCCMenuBackground; @@ -229,7 +233,8 @@ class Renderer : public BaseRenderer { uint32_t CurrentTexture = 0; uint32_t NextTextureId = 1; - VkYUVFrame* VideoFrameInternal; + VkYUVFrame* VideoFrameInternalYUV; + VkNV12Frame* VideoFrameInternalNV12; static int constexpr VertexBufferSize = 4096 * 4096; static int constexpr IndexBufferCount = diff --git a/src/shaders/dx9/NV12Frame_frag.hlsl b/src/shaders/dx9/NV12Frame_frag.hlsl new file mode 100644 index 000000000..62cf89d8a --- /dev/null +++ b/src/shaders/dx9/NV12Frame_frag.hlsl @@ -0,0 +1,37 @@ +bool IsAlpha : register(b0); + +struct PS_INPUT { + float4 position : POSITION; + float2 uv : TEXCOORD0; + float4 tint : COLOR; + float2 maskUV : TEXCOORD1; +}; + +sampler Luma : register(s0); +sampler CbCr : register(s1); + +static float4x4 yuv_to_rgb_rec601 = { + 1.16438, 0.00000, 1.59603, -0.87079, 1.16438, -0.39176, -0.81297, 0.52959, + 1.16438, 2.01723, 0.00000, -1.08139, 0, 0, 0, 1}; + +float4 getRGBA(float2 texUv) { + float4 yuvcolor = {1.0, 1.0, 1.0, 1.0}; + yuvcolor.r = tex2D(Luma, texUv).a; + yuvcolor.gb = tex2D(CbCr, texUv).ra; + + return mul(yuv_to_rgb_rec601, yuvcolor); +} + +float4 main(PS_INPUT input) : COLOR { + float4 color = {1.0, 1.0, 1.0, 1.0}; + + if (IsAlpha) { + color = getRGBA(float2(input.uv.x, input.uv.y / 2.0)); + color.a = getRGBA(float2(input.uv.x, input.uv.y / 2.0 + 0.5)).a; + } else { + color = getRGBA(input.uv); + } + color *= input.tint; + + return color; +} diff --git a/src/shaders/dx9/NV12Frame_vert.hlsl b/src/shaders/dx9/NV12Frame_vert.hlsl new file mode 100644 index 000000000..d1d297931 --- /dev/null +++ b/src/shaders/dx9/NV12Frame_vert.hlsl @@ -0,0 +1,24 @@ +struct VS_INPUT { + float2 Position : POSITION; + float2 UV : TEXCOORD0; + float4 Tint : COLOR; + float2 MaskUV : TEXCOORD1; +}; + +struct VS_OUTPUT { + float4 position : POSITION; + float2 uv : TEXCOORD0; + float4 tint : COLOR; + float2 maskUV : TEXCOORD1; +}; + +VS_OUTPUT main(VS_INPUT input) { + VS_OUTPUT output; + + output.position = float4(input.Position, 0.0, 1.0); + output.uv = input.UV; + output.tint = input.Tint; + output.maskUV = input.MaskUV; + + return output; +} diff --git a/src/shaders/opengl/NV12Frame_frag.glsl b/src/shaders/opengl/NV12Frame_frag.glsl new file mode 100644 index 000000000..84636d5d9 --- /dev/null +++ b/src/shaders/opengl/NV12Frame_frag.glsl @@ -0,0 +1,36 @@ +in vec2 uv; +in vec4 tint; + +out vec4 color; + +uniform sampler2D Luma; +uniform sampler2D CbCr; +uniform bool IsAlpha; + +const mat4 yuv_to_rgb_rec601 = mat4( + 1.16438, 0.00000, 1.59603, -0.87079, + 1.16438, -0.39176, -0.81297, 0.52959, + 1.16438, 2.01723, 0.00000, -1.08139, + 0, 0, 0, 1 +); + +vec4 getRGBA(vec2 texUv) { + vec4 yuvcolor = vec4(1.0); + + vec2 cbcr = texture(CbCr, texUv).rg; + yuvcolor.r = texture(Luma, texUv).r; + yuvcolor.g = cbcr.r; + yuvcolor.b = cbcr.g; + + return yuvcolor * yuv_to_rgb_rec601; +} + +void main() { + if (IsAlpha) { + color = getRGBA(vec2(uv.x, uv.y / 2.0)); + color.a = getRGBA(vec2(uv.x, uv.y / 2.0 + 0.5)).r; + } else { + color = getRGBA(uv); + } + color *= tint; +} \ No newline at end of file diff --git a/src/shaders/opengl/NV12Frame_vert.glsl b/src/shaders/opengl/NV12Frame_vert.glsl new file mode 100644 index 000000000..cdef1458d --- /dev/null +++ b/src/shaders/opengl/NV12Frame_vert.glsl @@ -0,0 +1,14 @@ +layout(location = 0) in vec2 Position; +layout(location = 1) in vec2 UV; +layout(location = 2) in vec4 Tint; + +out vec2 uv; +out vec4 tint; + +uniform mat4 Projection; + +void main() { + gl_Position = Projection * vec4(Position, 0.0, 1.0); + uv = UV; + tint = Tint; +} \ No newline at end of file diff --git a/src/shaders/vulkan/NV12Frame_frag.glsl b/src/shaders/vulkan/NV12Frame_frag.glsl new file mode 100644 index 000000000..66a12992a --- /dev/null +++ b/src/shaders/vulkan/NV12Frame_frag.glsl @@ -0,0 +1,38 @@ +#version 450 +#pragma shader_stage(fragment) + +layout(location = 0) in vec2 uv; +layout(location = 1) in vec4 tint; + +layout(location = 0) out vec4 color; + +layout(binding = 0) uniform sampler2D NV12Frame[2]; + +layout(push_constant) uniform constants +{ + bool IsAlpha; +} PushConstants; + +const mat4 yuv_to_rgb_rec601 = mat4( + 1.16438, 0.00000, 1.59603, -0.87079, + 1.16438, -0.39176, -0.81297, 0.52959, + 1.16438, 2.01723, 0.00000, -1.08139, + 0, 0, 0, 1 +); + +vec4 getRGBA(vec2 texUv) { + vec4 yuvcolor = vec4(1.0); + yuvcolor.r = texture(NV12Frame[0], texUv).r; + yuvcolor.gb = texture(NV12Frame[1], texUv).rg; + return yuvcolor * yuv_to_rgb_rec601; +} + +void main() { + if (PushConstants.IsAlpha) { + color = getRGBA(vec2(uv.x, uv.y / 2.0)); + color.a = getRGBA(vec2(uv.x, uv.y / 2.0 + 0.5)).r; + } else { + color = getRGBA(uv); + } + color *= tint; +} \ No newline at end of file diff --git a/src/shaders/vulkan/NV12Frame_frag.spv b/src/shaders/vulkan/NV12Frame_frag.spv new file mode 100644 index 0000000000000000000000000000000000000000..6d162a11c7a4fb6bf3b44f2d8f905555f212220b GIT binary patch literal 2572 zcmZ9NT~pOn6o)s^K?6exQ52&CSXmk+DQYGJmZ*`U=1Xtp5qUEN@*J4DtBYouu9~qv zMfY9mjMGII^-a3(WyYrGIqY@jKkS*sTEE{~d#$zC-v2{O^Wc`0no@h(n!ZRIt1~sj zB)HbPo}9WeHB_zL89H+Gbt5`bOFht-?d*FBt=y|DF6($7xhO_<5UY`T&-1>YgO!4;^bWCs^d$m3l*5w^bUA>t-5f!5Nm&)BZ`#h-7~Gx zguWZts}+i&#kV)%#ZB0_+TwBzl_rU|-+lor@W2KHx77P=2A?hTJlODW^ww9v_EyB! z6MIKpOx;}l^&3cadlz;qTp8I-Ki-enxH5PvT29gbG+MuR*3aKIw48Cqcc6`xI|w#* z2a?j)V~6m_cOr5{*@NdYl?MA!g zPiE}=UFO?^E+f`6pLj3ga)0lJST; zBlB%PjRhVCy8^#~{mc2^8~!8UoW{r6*uVbx+rz-ez<=~jKMH&t zJn~>m?~HRkZy2CmRO%jK|sKOwJ(A$v1t99QHSf$p1`?b2^u~GWsv7zK3@GOT51G$O2+7 z+U9mHmk?)FM$GLQFC+31a|PQNzk!JP09*b$^0?<3BIjJE;XL1U#9aDD>B)1;oxtzx zK18-5^7_3~xv2XQ_U7I`hLew&PjXDLw@=~Zqqm#ba`yI3?)Mh9y!RgcRn7#Mn9e5Yc(mCBlJg2eVy*O%qj%`i( zn2}sDmpRSlO#KJ$TMWE{Z7<*EnXh8Y85jBFp745)BfekrSku*p9HTGaX**)?&!BzF z9q8?dXE2v`C*mA>(fg1dB;IWo_H&t!v->~Jyw`Kx=W~pF%(x$0-ev9q^dMrrGGcA9 zaZ&pPY-{gFtYz&N5&5Y75_YlnfgB@$kyClDml1iz)r3S32eIwJw->!0!gi0#zKoHJ oJ+ETBryGeqL)h*yw|k6{i#vP`+dV(z8I57fjUxYXA7bzBfA+7ZF9Y7Ee5fDV&b-9rc4?;|g8iL7kfCPhZH5;9niR(hxff!HzHh+~D z6TYu^dflGXRKHi%RrPu%YjmbVxEJQaOxO;iwG9FZ>v)mizgFfYaIMeML&UVSKsRd4X67{^p$?55uz$8zR# zepk+;j&Z}tWl`77t??}CJHHrdQD?;rIAYDH`O^ee|4DE7-TW?R)#`jlWV?4IpE$M2 z=@aJ}Ip1kd6nm@6>po|3)3&vCA5NVRHOiSZ7$q)bkNmEvq$$8*^(lXOB}l x+xrP-t}*qi`G6YEyMcM<-s?NMip@_w{ZG_rEd6)Jtos5}k1=(N|Domethods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) continue; + + AVBufferRef* hw_device_ctx = NULL; + if (av_hwdevice_ctx_create(&hw_device_ctx, cfg->device_type, NULL, NULL, + 0) >= 0) { + HwVideoPixelFormat = static_cast(cfg->pix_fmt); + return hw_device_ctx; + } + } + return NULL; +} + template void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, av::Stream&& avStream, int streamId) { @@ -119,8 +135,23 @@ void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, } std::error_code ec; - DecodingContext_t decoderContext{avStream, codec}; + if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { + if (auto* ctx = HwDecoderInit(codec.raw()); ctx) { + decoderContext.raw()->hw_device_ctx = ctx; + decoderContext.raw()->opaque = this; + decoderContext.raw()->get_format = + [](AVCodecContext * ctx, const enum AVPixelFormat* pix_fmts) + ->enum AVPixelFormat { + auto* self = static_cast(ctx->opaque); + for (const auto* p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { + if (*p == self->HwVideoPixelFormat) return *p; + } + return AV_PIX_FMT_NONE; + }; + } + } + decoderContext.open({{{"threads", "auto"}}}, ec); decoderContext.setRefCountedFrames(true); if (ec) { @@ -214,14 +245,25 @@ void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) { videoStreamId); ScrWork[SW_MOVIEFRAME] = 0; ScrWork[SW_MOVIETOTALFRAME] = VideoStream->Duration; - if (!VideoTexture) - VideoTexture = - Renderer->CreateYUVFrame((float)VideoStream->CodecContext.width(), - (float)VideoStream->CodecContext.height()); - else { - VideoTexture->Width = (float)VideoStream->CodecContext.width(); - VideoTexture->Height = (float)VideoStream->CodecContext.height(); - } + std::visit( + [this](auto& videoText) { + if constexpr (std::is_same_v, + std::monostate>) { + if (HwVideoPixelFormat == AV_PIX_FMT_NONE) { + VideoTexture = Renderer->CreateYUVFrame( + (float)VideoStream->CodecContext.width(), + (float)VideoStream->CodecContext.height()); + } else { + VideoTexture = Renderer->CreateNV12Frame( + (float)VideoStream->CodecContext.width(), + (float)VideoStream->CodecContext.height()); + } + } else { + videoText->Width = (float)VideoStream->CodecContext.width(); + videoText->Height = (float)VideoStream->CodecContext.height(); + } + }, + VideoTexture); } if (audioStream.isAudio() && audioStream.isValid()) { OpenCodec(AudioStream, std::move(audioStream), @@ -261,12 +303,17 @@ void FFmpegPlayer::InitSubtitles( using Profile::Subtitle::SubtitleType; using namespace Subtitle; auto initSubPlayer = [this] { - if (!SubPlayer) { - if (VideoTexture) - SubPlayer.emplace(VideoTexture->Width, VideoTexture->Height); - else - SubPlayer.emplace(Profile::DesignWidth, Profile::DesignHeight); - } + if (SubPlayer) return; + std::visit( + [this](auto& videoText) { + if constexpr (std::is_same_v, + std::monostate>) { + SubPlayer.emplace(Profile::DesignWidth, Profile::DesignHeight); + } else { + SubPlayer.emplace(videoText->Width, videoText->Height); + } + }, + VideoTexture); }; auto mappingsItr = SubtitleMappings.find(StreamPtr->Meta.FileName); @@ -408,6 +455,23 @@ void FFmpegPlayer::Read() { } } +void FFmpegPlayer::ProcessVideoFrame(Frame_t& avFrame) { + auto rawFrame = avFrame.raw(); + if (rawFrame->format == AV_PIX_FMT_NONE || + !(av_pix_fmt_desc_get(avFrame.pixelFormat())->flags & + AV_PIX_FMT_FLAG_HWACCEL)) { + return; + } + AVFrame* swFrame = av_frame_alloc(); + if (av_hwframe_transfer_data(swFrame, rawFrame, 0) < 0) { + av_frame_free(&swFrame); + return; + } + av_frame_copy_props(swFrame, rawFrame); + av_frame_free(&rawFrame); + avFrame.reset(swFrame); +} + template void FFmpegPlayer::Decode(FFmpegStream& stream) { auto verifyPacket = @@ -466,6 +530,9 @@ void FFmpegPlayer::Decode(FFmpegStream& stream) { ImpLog(LogLevel::Error, LogChannel::Video, "Failed to decode {:s}", ec.message()); } + if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { + ProcessVideoFrame(frame); + } if (!frame) continue; // Skip Empty Padding Frames if constexpr (MediaType == AVMEDIA_TYPE_SUBTITLE) { auto* rawSubtitle = frame.raw(); @@ -523,10 +590,16 @@ void FFmpegPlayer::Stop() { PreviousFrameTimestamp = {}; FormatContext.close(); StreamPtr.reset(); - if (VideoTexture) { - VideoTexture->Release(); - VideoTexture = 0; - } + std::visit( + [this](auto& videoText) { + if constexpr (!std::is_same_v, + std::monostate>) { + videoText->Release(); + VideoTexture = std::monostate{}; + } + }, + VideoTexture); + SetFlag(SF_MOVIEPLAY, false); } } @@ -598,9 +671,14 @@ void FFmpegPlayer::Update(float dt) { } PreviousFrameTimestamp = frame->Timestamp; - - VideoTexture->Submit(frame->Frame.data(0), frame->Frame.data(1), - frame->Frame.data(2)); + if (frame->Frame.pixelFormat() == AV_PIX_FMT_NV12) { + auto& nv12Frame = std::get(VideoTexture); + nv12Frame->Submit(frame->Frame.data(0), frame->Frame.data(1)); + } else if(frame->Frame.pixelFormat() == AV_PIX_FMT_YUV420P) { + auto& yuvFrame = std::get(VideoTexture); + yuvFrame->Submit(frame->Frame.data(0), frame->Frame.data(1), + frame->Frame.data(2)); + } VideoClock.Set(frame->Timestamp.toDuration(), frame->Serial); MasterClock->SyncTo(&VideoClock); @@ -661,7 +739,14 @@ void FFmpegPlayer::Render(float videoAlpha) { const RectF dest = {0.0f, 0.0f, Profile::DesignWidth, Profile::DesignHeight}; const glm::vec4 tint = {1.0f, 1.0f, 1.0f, videoAlpha}; - Renderer->DrawVideoTexture(*VideoTexture, dest, tint, IsAlpha); + std::visit( + [this, &dest, &tint](auto& videoText) { + if constexpr (!std::is_same_v, + std::monostate>) { + Renderer->DrawVideoTexture(*videoText, dest, tint, IsAlpha); + } + }, + VideoTexture); if (SubPlayer) SubPlayer->Render(); } diff --git a/src/video/ffmpegplayer.h b/src/video/ffmpegplayer.h index 976339128..2ba67ccd8 100644 --- a/src/video/ffmpegplayer.h +++ b/src/video/ffmpegplayer.h @@ -73,6 +73,8 @@ class FFmpegPlayer : public VideoPlayer { av::Stream&& avStream, int streamId); void UpdateSubtitles(); + void ProcessVideoFrame(Frame_t& avFrame); + AVBufferRef* HwDecoderInit(const AVCodec* codec); static int constexpr FILESTREAMBUFFERSZ = 64 * 8192; std::condition_variable ReadCond; @@ -82,16 +84,17 @@ class FFmpegPlayer : public VideoPlayer { std::unique_ptr StreamPtr; av::FormatContext FormatContext; + AVPixelFormat HwVideoPixelFormat = AV_PIX_FMT_NONE; FFmpegFileIO IoContext; - Clock VideoClock; + Clock VideoClock; Clock* MasterClock{}; std::unique_ptr AudioPlayer; bool IsInit = false; - YUVFrame* VideoTexture; + std::variant VideoTexture; bool IsAlpha = false; bool Looping = false; From 0d54d7b67c208d7e58ce7552abcf9a8c2ddd78c9 Mon Sep 17 00:00:00 2001 From: Lint Action Date: Mon, 30 Mar 2026 01:33:21 +0000 Subject: [PATCH 2/8] Fix code style issues with clang_format --- src/video/ffmpegplayer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index 500c6058c..b284cedcf 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -141,8 +141,8 @@ void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, decoderContext.raw()->hw_device_ctx = ctx; decoderContext.raw()->opaque = this; decoderContext.raw()->get_format = - [](AVCodecContext * ctx, const enum AVPixelFormat* pix_fmts) - ->enum AVPixelFormat { + [](AVCodecContext * ctx, + const enum AVPixelFormat* pix_fmts) -> enum AVPixelFormat { auto* self = static_cast(ctx->opaque); for (const auto* p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { if (*p == self->HwVideoPixelFormat) return *p; From 2167bcb4262f6481d9766d64a8e6345d358e8953 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sat, 4 Apr 2026 01:04:24 -0700 Subject: [PATCH 3/8] Orientation fixes and make app not die on rotation --- android/app/src/main/AndroidManifest.xml | 3 ++- src/renderer/opengl/window.cpp | 3 +++ src/renderer/vulkan/window.cpp | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 33d2c38b3..2964d8bed 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -35,6 +35,7 @@ diff --git a/src/renderer/opengl/window.cpp b/src/renderer/opengl/window.cpp index 916bc525c..d349c79c7 100644 --- a/src/renderer/opengl/window.cpp +++ b/src/renderer/opengl/window.cpp @@ -166,6 +166,9 @@ void GLWindow::Init() { ImpLog(LogLevel::Info, LogChannel::General, "Creating window\n"); IsInit = true; +#ifdef __ANDROID__ + SDL_SetHint(SDL_HINT_ORIENTATIONS, "LandscapeLeft LandscapeRight"); +#endif if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER) != 0) { ImpLog(LogLevel::Fatal, LogChannel::General, "SDL initialisation failed: {:s}\n", SDL_GetError()); diff --git a/src/renderer/vulkan/window.cpp b/src/renderer/vulkan/window.cpp index 5f786cc53..f525cccf3 100644 --- a/src/renderer/vulkan/window.cpp +++ b/src/renderer/vulkan/window.cpp @@ -61,6 +61,9 @@ void VulkanWindow::Init() { ImpLog(LogLevel::Info, LogChannel::General, "Creating window\n"); IsInit = true; +#ifdef __ANDROID__ + SDL_SetHint(SDL_HINT_ORIENTATIONS, "LandscapeLeft LandscapeRight"); +#endif if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER) != 0) { ImpLog(LogLevel::Fatal, LogChannel::General, "SDL initialisation failed: {:s}\n", SDL_GetError()); From 4259d45e89e3c745acb43d4d919d88e5141b7334 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sat, 4 Apr 2026 00:38:59 -0700 Subject: [PATCH 4/8] Hack to fix libass linking libc++_shared.so on android --- CMakeLists.txt | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9ed4ee0c..3c1a56a4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1059,13 +1059,24 @@ if (NOT DEFINED IMPACTO_DISABLE_LIBASS) list(APPEND Impacto_Src src/subtitle/ass/subtitlerenderer.cpp) list(APPEND Impacto_Header src/subtitle/ass/subtitlerenderer.h) - pkg_check_modules(LIBASS REQUIRED IMPORTED_TARGET GLOBAL libass) - list(APPEND Impacto_Include_Dirs ${LIBASS_INCLUDE_DIRS}) + pkg_check_modules(ass REQUIRED IMPORTED_TARGET GLOBAL libass) + # Libass links libc++ which overrides the static stllib linkage + if(ANDROID) + get_target_property(ASS_LINK_LIBS PkgConfig::ass INTERFACE_LINK_LIBRARIES) + set(CLEANED_ASS_LINK_LIBS "") + foreach(LIB_PATH IN LISTS ASS_LINK_LIBS) + get_filename_component(LIB_NAME "${LIB_PATH}" NAME) + + # Check if the filename starts with "libc++." or "libc++_" + if(NOT ("${LIB_NAME}" MATCHES "^libc\\+\\+\\." OR "${LIB_NAME}" MATCHES "^libc\\+\\+_")) + list(APPEND CLEANED_ASS_LINK_LIBS "${LIB_PATH}") + endif() + endforeach() + + set_target_properties(PkgConfig::ass PROPERTIES INTERFACE_LINK_LIBRARIES "${CLEANED_ASS_LINK_LIBS}") + endif() list(APPEND Impacto_Libs - ${LIBASS_LINK_LIBRARIES} - ) - list(APPEND Impacto_Compile_Options - ${LIBASS_CFLAGS_OTHER} + PkgConfig::ass ) endif() From a96b22b25a7c632fdf9c2b917210a2893e489aac Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sun, 5 Apr 2026 05:15:07 -0700 Subject: [PATCH 5/8] Add support for hwaccelerated android and linux --- .github/workflows/pipeline.yml | 2 +- CMakeLists.txt | 4 +- CMakePresets.json | 1 + portfiles/avcpp/0001-codeccontext.patch | 157 ++++++++++++++++++++++++ portfiles/avcpp/portfile.cmake | 1 + src/renderer/dx9/nv12frame.cpp | 22 +++- src/renderer/dx9/nv12frame.h | 3 +- src/renderer/nv12frame.h | 3 +- src/renderer/opengl/nv12frame.cpp | 8 +- src/renderer/opengl/nv12frame.h | 3 +- src/renderer/opengl/renderer.cpp | 11 +- src/renderer/vulkan/nv12frame.cpp | 27 ++-- src/renderer/vulkan/nv12frame.h | 3 +- src/video/ffmpegplayer.cpp | 114 +++++++++++++---- vcpkg.json | 12 +- 15 files changed, 316 insertions(+), 55 deletions(-) create mode 100644 portfiles/avcpp/0001-codeccontext.patch diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index fef6e8303..812fd9cf6 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -144,7 +144,7 @@ jobs: - name: Install Dependencies Linux run: | sudo apt-get update - sudo apt-get install nasm libx11-dev libxft-dev libxext-dev libwayland-dev libxkbcommon-dev libegl1-mesa-dev libibus-1.0-dev libxrandr-dev libltdl-dev mono-complete autoconf autoconf-archive automake libtool + sudo apt-get install nasm libx11-dev libxft-dev libxext-dev libwayland-dev libxkbcommon-dev libegl1-mesa-dev libibus-1.0-dev libxrandr-dev libltdl-dev mono-complete autoconf autoconf-archive automake libtool libdrm-dev if: matrix.os_name == 'linux' || matrix.os_name == 'android' - name: Install Dependencies Mac run: brew install nasm mono diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c1a56a4c..d082f88e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1168,19 +1168,21 @@ if (NOT DEFINED IMPACTO_DISABLE_FFMPEG) ) if (NOT VCPKG_TOOLCHAIN) - set(AVCPP_PATCH git apply ${CMAKE_CURRENT_SOURCE_DIR}/vendor/patches/avcpp.patch) if(NOT BUILD_SHARED_LIBS) set(AV_ENABLE_SHARED OFF CACHE BOOL "Enable shared library build (Off)" FORCE) endif() set(AV_DISABLE_AVDEVICE ON CACHE INTERNAL "Disable FFMPEG AVDevice") set(AV_DISABLE_AVFILTER ON CACHE INTERNAL "Disable FFMPEG AVFilter") set(AV_BUILD_EXAMPLES OFF CACHE INTERNAL "Build AVCPP Examples") + set(AVCPP_PATCH git apply ${CMAKE_CURRENT_SOURCE_DIR}/portfiles/avcpp/0001-codeccontext.patch) FetchContent_Declare( avcpp GIT_REPOSITORY "https://github.com/h4tr3d/avcpp" GIT_TAG "00e29ea0c0f7036423b80553aaccacd70f90aace" SYSTEM GIT_SUBMODULES "" + PATCH_COMMAND ${AVCPP_PATCH} + UPDATE_DISCONNECTED 1 ) FetchContent_MakeAvailable(avcpp) diff --git a/CMakePresets.json b/CMakePresets.json index 259fc3ded..3671b1fe2 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -54,6 +54,7 @@ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/triplets", + "VCPKG_OVERLAY_PORTS": "${sourceDir}/portfiles", "IMPACTO_WARNINGS": "ON" } }, diff --git a/portfiles/avcpp/0001-codeccontext.patch b/portfiles/avcpp/0001-codeccontext.patch new file mode 100644 index 000000000..88b33cc53 --- /dev/null +++ b/portfiles/avcpp/0001-codeccontext.patch @@ -0,0 +1,157 @@ +diff --git a/src/avcpp/codeccontext.cpp b/src/avcpp/codeccontext.cpp +index f67807e..6f3783f 100644 +--- a/src/avcpp/codeccontext.cpp ++++ b/src/avcpp/codeccontext.cpp +@@ -90,7 +90,7 @@ int decode(AVCodecContext *avctx, + } + + ret = avcodec_receive_frame(avctx, picture); +- if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) ++ if (ret < 0 && ret != AVERROR_EOF) + return ret; + if (ret >= 0 && got_picture_ptr) + *got_picture_ptr = 1; +@@ -203,17 +203,17 @@ VideoDecoderContext &VideoDecoderContext::operator=(VideoDecoderContext&& other) + return moveOperator(std::move(other)); + } + +-VideoFrame VideoDecoderContext::decode(const Packet &packet, OptionalErrorCode ec, bool autoAllocateFrame) ++VideoFrame VideoDecoderContext::decode(const Packet &packet, OptionalErrorCode ec, bool autoAllocateFrame, bool send_packet) + { +- return decodeVideo(ec, packet, 0, nullptr, autoAllocateFrame); ++ return decodeVideo(ec, packet, 0, nullptr, autoAllocateFrame, send_packet); + } + + VideoFrame VideoDecoderContext::decode(const Packet &packet, size_t offset, size_t &decodedBytes, OptionalErrorCode ec, bool autoAllocateFrame) + { +- return decodeVideo(ec, packet, offset, &decodedBytes, autoAllocateFrame); ++ return decodeVideo(ec, packet, offset, &decodedBytes, autoAllocateFrame, true); + } + +-VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet &packet, size_t offset, size_t *decodedBytes, bool autoAllocateFrame) ++VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet &packet, size_t offset, size_t *decodedBytes, bool autoAllocateFrame, bool send_packet) + { + clear_if(ec); + +@@ -230,7 +230,7 @@ VideoFrame VideoDecoderContext::decodeVideo(OptionalErrorCode ec, const Packet & + } + + int gotFrame = 0; +- auto st = decodeCommon(outFrame, packet, offset, gotFrame, avcodec_decode_video_legacy); ++ auto st = decodeCommon(outFrame, packet, offset, send_packet, gotFrame, avcodec_decode_video_legacy); + + if (get<1>(st)) { + throws_if(ec, get<0>(st), *get<1>(st)); +@@ -882,7 +882,7 @@ void CodecContext2::open(const Codec &codec, AVDictionary **options, OptionalErr + throws_if(ec, stat, ffmpeg_category()); + } + +-std::pair CodecContext2::decodeCommon(AVFrame *outFrame, const Packet &inPacket, size_t offset, int &frameFinished, int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) noexcept ++std::pair CodecContext2::decodeCommon(AVFrame *outFrame, const Packet &inPacket, size_t offset, bool send_packet, int &frameFinished, int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) noexcept + { + if (!isValid()) + return make_error_pair(Errors::CodecInvalid); +@@ -902,7 +902,7 @@ std::pair CodecContext2::decodeCommon(AVFrame *outF + pkt.data += offset; + pkt.size -= offset; + +- int decoded = decodeProc(m_raw, outFrame, &frameFinished, &pkt); ++ int decoded = decodeProc(m_raw, outFrame, &frameFinished, send_packet? &pkt: nullptr); + return make_error_pair(decoded); + } + +@@ -940,19 +940,19 @@ AudioDecoderContext &AudioDecoderContext::operator=(AudioDecoderContext &&other) + return moveOperator(std::move(other)); + } + +-AudioSamples AudioDecoderContext::decode(const Packet &inPacket, OptionalErrorCode ec) ++AudioSamples AudioDecoderContext::decode(const Packet &inPacket, OptionalErrorCode ec, bool send_packet) + { +- return decode(inPacket, 0u, ec); ++ return decode(inPacket, 0u, ec, send_packet); + } + +-AudioSamples AudioDecoderContext::decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec) ++AudioSamples AudioDecoderContext::decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec, bool send_packet) + { + clear_if(ec); + + AudioSamples outSamples; + + int gotFrame = 0; +- auto st = decodeCommon(outSamples, inPacket, offset, gotFrame, avcodec_decode_audio_legacy); ++ auto st = decodeCommon(outSamples, inPacket, offset, send_packet, gotFrame, avcodec_decode_audio_legacy); + if (get<1>(st)) + { + throws_if(ec, get<0>(st), *get<1>(st)); +@@ -1016,10 +1016,11 @@ std::pair + CodecContext2::decodeCommon(T &outFrame, + const Packet &inPacket, + size_t offset, ++ bool send_packet, + int &frameFinished, + int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)) + { +- auto st = decodeCommon(outFrame.raw(), inPacket, offset, frameFinished, decodeProc); ++ auto st = decodeCommon(outFrame.raw(), inPacket, offset, send_packet, frameFinished, decodeProc); + if (std::get<1>(st)) + return st; + +diff --git a/src/avcpp/codeccontext.h b/src/avcpp/codeccontext.h +index 2e32904..3e8f7d0 100644 +--- a/src/avcpp/codeccontext.h ++++ b/src/avcpp/codeccontext.h +@@ -150,7 +150,7 @@ protected: + + + std::pair +- decodeCommon(AVFrame *outFrame, const class Packet &inPacket, size_t offset, int &frameFinished, ++ decodeCommon(AVFrame *outFrame, const class Packet &inPacket, size_t offset, bool send_packet, int &frameFinished, + int (*decodeProc)(AVCodecContext*, AVFrame*,int *, const AVPacket *)) noexcept; + + std::pair +@@ -163,6 +163,7 @@ public: + decodeCommon(T &outFrame, + const class Packet &inPacket, + size_t offset, ++ bool send_packet, + int &frameFinished, + int (*decodeProc)(AVCodecContext *, AVFrame *, int *, const AVPacket *)); + +@@ -417,12 +418,14 @@ public: + * av#throws the function will throw on error instead + * @param autoAllocateFrame it true - output will be allocated at the ffmpeg internal, otherwise + * it will be allocated before decode proc call. ++ * @param send_packets whether to send the packet or not (for successive decodes) + * @return encoded video frame, if error: exception thrown or error code returns, in both cases + * output undefined. + */ + VideoFrame decode(const Packet &packet, + OptionalErrorCode ec = throws(), +- bool autoAllocateFrame = true); ++ bool autoAllocateFrame = true, ++ bool send_packet = true); + + /** + * @brief decodeVideo - decode video packet with additional parameters +@@ -449,7 +452,8 @@ private: + const Packet &packet, + size_t offset, + size_t *decodedBytes, +- bool autoAllocateFrame); ++ bool autoAllocateFrame, ++ bool send_packet); + + }; + +@@ -598,8 +602,8 @@ public: + + AudioDecoderContext& operator=(AudioDecoderContext&& other); + +- AudioSamples decode(const Packet &inPacket, OptionalErrorCode ec = throws()); +- AudioSamples decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec = throws()); ++ AudioSamples decode(const Packet &inPacket, OptionalErrorCode ec = throws(), bool send_packet = true); ++ AudioSamples decode(const Packet &inPacket, size_t offset, OptionalErrorCode ec = throws(), bool send_packet = true); + + }; + diff --git a/portfiles/avcpp/portfile.cmake b/portfiles/avcpp/portfile.cmake index d24755bd4..f8b813802 100644 --- a/portfiles/avcpp/portfile.cmake +++ b/portfiles/avcpp/portfile.cmake @@ -5,6 +5,7 @@ vcpkg_from_github( SHA512 7c36a90ed878f5addc1e3ebed21db66088f39f57605f73c6b7f260a3c0537d4a908f60fb17be0ec801e2e7eaeee6b3b701a9c7f99f2e0ba0a00eb97b62567cc4 PATCHES 0002-av_init_packet_deprecation.patch + 0001-codeccontext.patch ) string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" AVCPP_ENABLE_STATIC) diff --git a/src/renderer/dx9/nv12frame.cpp b/src/renderer/dx9/nv12frame.cpp index 9e63723a8..58b75f471 100644 --- a/src/renderer/dx9/nv12frame.cpp +++ b/src/renderer/dx9/nv12frame.cpp @@ -16,14 +16,30 @@ void DX9NV12Frame::Init(float width, float height) { D3DFMT_A8L8, D3DPOOL_MANAGED, &CbCr, nullptr); } -void DX9NV12Frame::Submit(const void* luma, const void* cbcr) { +void Impacto::DirectX9::DX9NV12Frame::Submit(const void* luma, int lumaStride, + const void* cbcr, int cbcrStride) { D3DLOCKED_RECT lockRect; auto err = Luma->LockRect(0, &lockRect, NULL, 0); - memcpy(lockRect.pBits, luma, (size_t)(Width * Height)); + const uint8_t* src = (const uint8_t*)luma; + uint8_t* dst = (uint8_t*)lockRect.pBits; + for (int y = 0; y < (int)Height; y++) { + memcpy(dst, src, (int)Width); + src += lumaStride; + dst += lockRect.Pitch; + } + err = Luma->UnlockRect(0); err = CbCr->LockRect(0, &lockRect, NULL, 0); - memcpy(lockRect.pBits, cbcr, (size_t)((Width) * (Height / 2))); + + src = (const uint8_t*)cbcr; + dst = (uint8_t*)lockRect.pBits; + for (int y = 0; y < (int)Height / 2; y++) { + memcpy(dst, src, (int)Width); + src += cbcrStride; + dst += lockRect.Pitch; + } + err = CbCr->UnlockRect(0); } diff --git a/src/renderer/dx9/nv12frame.h b/src/renderer/dx9/nv12frame.h index 59895370c..da09e0897 100644 --- a/src/renderer/dx9/nv12frame.h +++ b/src/renderer/dx9/nv12frame.h @@ -15,7 +15,8 @@ class DX9NV12Frame : public NV12Frame { void Init(float width, float height) override; - void Submit(const void* luma, const void* cbcr) override; + void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) override; void Release() override; protected: diff --git a/src/renderer/nv12frame.h b/src/renderer/nv12frame.h index b69b3d129..116e3b437 100644 --- a/src/renderer/nv12frame.h +++ b/src/renderer/nv12frame.h @@ -13,7 +13,8 @@ class NV12Frame { virtual void Init(float width, float height) = 0; - virtual void Submit(const void* luma, const void* cbcr) = 0; + virtual void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) = 0; virtual void Release() = 0; }; diff --git a/src/renderer/opengl/nv12frame.cpp b/src/renderer/opengl/nv12frame.cpp index 40134b0ef..a0cabb7c2 100644 --- a/src/renderer/opengl/nv12frame.cpp +++ b/src/renderer/opengl/nv12frame.cpp @@ -16,16 +16,20 @@ void GLNV12Frame::Init(float width, float height) { CbCrId = yuv[1]; } -void GLNV12Frame::Submit(const void* luma, const void* cbcr) { +void Impacto::OpenGL::GLNV12Frame::Submit(const void* luma, int lumaStride, + const void* cbcr, int cbcrStride) { glBindTexture(GL_TEXTURE_2D, LumaId); + glPixelStorei(GL_UNPACK_ROW_LENGTH, lumaStride); glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, (GLsizei)Width, (GLsizei)Height, 0, GL_RED, GL_UNSIGNED_BYTE, luma); glGenerateMipmap(GL_TEXTURE_2D); glBindTexture(GL_TEXTURE_2D, CbCrId); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RG, (GLsizei)(Width / 2), + glPixelStorei(GL_UNPACK_ROW_LENGTH, cbcrStride / 2); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, (GLsizei)(Width / 2), (GLsizei)(Height / 2), 0, GL_RG, GL_UNSIGNED_BYTE, cbcr); glGenerateMipmap(GL_TEXTURE_2D); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); } void GLNV12Frame::Release() { diff --git a/src/renderer/opengl/nv12frame.h b/src/renderer/opengl/nv12frame.h index c5526d42f..da4179bb1 100644 --- a/src/renderer/opengl/nv12frame.h +++ b/src/renderer/opengl/nv12frame.h @@ -9,7 +9,8 @@ class GLNV12Frame : public NV12Frame { public: void Init(float width, float height) override; - void Submit(const void* luma, const void* cbcr) override; + void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) override; void Release() override; }; diff --git a/src/renderer/opengl/renderer.cpp b/src/renderer/opengl/renderer.cpp index 674695f08..5626f6249 100644 --- a/src/renderer/opengl/renderer.cpp +++ b/src/renderer/opengl/renderer.cpp @@ -193,26 +193,27 @@ uint32_t Renderer::SubmitTexture(TexFmt format, uint8_t* buffer, int width, glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 16); // Load in data - const GLuint texFormat = [format]() { + const auto [internalFormat, + texFormat] = [format]() -> std::pair { switch (format) { case TexFmt_RGBA: glPixelStorei(GL_UNPACK_ALIGNMENT, 4); - return GL_RGBA; + return {GL_RGBA, GL_RGBA}; case TexFmt_RGB: glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - return GL_RGB; + return {GL_RGB, GL_RGB}; case TexFmt_U8: glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - return GL_RED; + return {GL_R8, GL_RED}; default: throw std::invalid_argument( fmt::format("Unimplemented texture format {}", (int)format)); } }(); - glTexImage2D(GL_TEXTURE_2D, 0, texFormat, width, height, 0, texFormat, + glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, width, height, 0, texFormat, GL_UNSIGNED_BYTE, buffer); // Build mip chain diff --git a/src/renderer/vulkan/nv12frame.cpp b/src/renderer/vulkan/nv12frame.cpp index eabbc4704..ca18ad347 100644 --- a/src/renderer/vulkan/nv12frame.cpp +++ b/src/renderer/vulkan/nv12frame.cpp @@ -59,14 +59,25 @@ void VkNV12Frame::Init(float width, float height) { &CbCrImage.ImageView); } -void VkNV12Frame::Submit(const void* luma, const void* cbcr) { - int cbcrOffset = (int)(Width * Height); - - uint8_t* mappedStagingBuffer = (uint8_t*)MappedStagingBuffer; - - memcpy(mappedStagingBuffer, luma, cbcrOffset); - memcpy(mappedStagingBuffer + cbcrOffset, cbcr, - (size_t)((Width) * (Height / 2))); +void Impacto::Vulkan::VkNV12Frame::Submit(const void* luma, int lumaStride, + const void* cbcr, int cbcrStride) { + const int cbcrOffset = (int)(Width * Height); + + const uint8_t* src = (const uint8_t*)luma; + uint8_t* dst = (uint8_t*)MappedStagingBuffer; + for (int y = 0; y < Height; y++) { + memcpy(dst, src, (int)Width); + src += lumaStride; + dst += (int)Width; + } + + src = (const uint8_t*)cbcr; + dst = (uint8_t*)MappedStagingBuffer + cbcrOffset; + for (int y = 0; y < (int)Height / 2; y++) { + memcpy(dst, src, (int)Width); + src += cbcrStride; + dst += (int)Width; + } ImmediateSubmit([&](VkCommandBuffer cmd) { VkExtent3D lumaImageExtent; diff --git a/src/renderer/vulkan/nv12frame.h b/src/renderer/vulkan/nv12frame.h index 715fbe968..bc725f637 100644 --- a/src/renderer/vulkan/nv12frame.h +++ b/src/renderer/vulkan/nv12frame.h @@ -14,7 +14,8 @@ class VkNV12Frame : public NV12Frame { public: void Init(float width, float height) override; - void Submit(const void* luma, const void* cbcr) override; + void Submit(const void* luma, int lumaStride, const void* cbcr, + int cbcrStride) override; void Release() override; protected: diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index b284cedcf..d23cc465f 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -118,6 +118,47 @@ AVBufferRef* FFmpegPlayer::HwDecoderInit(const AVCodec* codec) { return NULL; } +template +std::optional findDecoderCodec(av::Stream const& avStream) { + const AVCodecID codecId = avStream.codecParameters().codecId(); + std::optional result; + + auto checkDecode = [](av::Codec&& codec) -> std::optional { + if (!codec.canDecode()) { + const auto channel = [] { + switch (MediaType) { + case AVMEDIA_TYPE_VIDEO: + return LogChannel::Video; + case AVMEDIA_TYPE_AUDIO: + return LogChannel::Audio; + case AVMEDIA_TYPE_SUBTITLE: + return LogChannel::Subtitle; + default: + return LogChannel::General; + } + }(); + ImpLog(LogLevel::Error, channel, "Unsupported codec: {}!\n", + codec.name()); + return std::nullopt; + } + return std::optional{std::move(codec)}; + }; + + if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { +#ifdef __ANDROID__ + const AVCodecDescriptor* desc = avcodec_descriptor_get(codecId); + const std::string decoderName = fmt::format("{}_mediacodec", desc->name); + result = checkDecode(av::findDecodingCodec(decoderName)); +#endif + } + + if (!result) { + result = checkDecode(av::findDecodingCodec(codecId)); + } + + return result; +} + template void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, av::Stream&& avStream, int streamId) { @@ -126,18 +167,14 @@ void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, MediaType != AVMEDIA_TYPE_SUBTITLE) { static_assert(MediaType && false, "Unsupported MediaType"); } - - const auto codec = - av::findDecodingCodec(avStream.codecParameters().codecId()); - if (!codec.canDecode()) { - ImpLog(LogLevel::Error, LogChannel::Video, "Unsupported codec!\n"); - streamOpt.reset(); + std::optional codec = findDecoderCodec(avStream); + if (!codec) { + avStream.reset(); } - std::error_code ec; - DecodingContext_t decoderContext{avStream, codec}; + DecodingContext_t decoderContext{avStream, *codec}; if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { - if (auto* ctx = HwDecoderInit(codec.raw()); ctx) { + if (auto* ctx = HwDecoderInit(decoderContext.codec().raw()); ctx) { decoderContext.raw()->hw_device_ctx = ctx; decoderContext.raw()->opaque = this; decoderContext.raw()->get_format = @@ -147,16 +184,33 @@ void FFmpegPlayer::OpenCodec(std::optional>& streamOpt, for (const auto* p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { if (*p == self->HwVideoPixelFormat) return *p; } + for (const auto* p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { + const AVPixFmtDescriptor* desc = av_pix_fmt_desc_get(*p); + if (!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) return *p; + } return AV_PIX_FMT_NONE; }; } } - decoderContext.open({{{"threads", "auto"}}}, ec); + std::error_code ec; + decoderContext.open({{{"threads", "auto"}}}, decoderContext.codec(), ec); decoderContext.setRefCountedFrames(true); if (ec) { - ImpLog(LogLevel::Error, LogChannel::Audio, - "Failed to open codec, error: {:s}", ec.message()); + const auto channel = [] { + switch (MediaType) { + case AVMEDIA_TYPE_VIDEO: + return LogChannel::Video; + case AVMEDIA_TYPE_AUDIO: + return LogChannel::Audio; + case AVMEDIA_TYPE_SUBTITLE: + return LogChannel::Subtitle; + default: + return LogChannel::General; + } + }(); + ImpLog(LogLevel::Error, channel, "Failed to open codec, error: {:s}", + ec.message()); } double rate = 1; @@ -524,24 +578,41 @@ void FFmpegPlayer::Decode(FFmpegStream& stream) { AVPacketItem packet = verifyPacket(peek); std::error_code ec; - if (packet.Serial != INT32_MIN) { - Frame_t frame = stream.CodecContext.decode(packet.Packet, ec); - if (ec) { + auto processAndPush = [&](Frame_t&& f) { + if (ec == std::error_code{AVERROR(EAGAIN), av::ffmpeg_category()}) + return false; + if (ec) ImpLog(LogLevel::Error, LogChannel::Video, "Failed to decode {:s}", ec.message()); - } if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) { - ProcessVideoFrame(frame); + ProcessVideoFrame(f); } - if (!frame) continue; // Skip Empty Padding Frames + if (!f) return true; if constexpr (MediaType == AVMEDIA_TYPE_SUBTITLE) { - auto* rawSubtitle = frame.raw(); + auto* rawSubtitle = f.raw(); if (rawSubtitle->start_display_time == 0 && rawSubtitle->end_display_time == 0) { rawSubtitle->end_display_time = packet.Packet.duration(); } } - pushFrame(std::move(frame)); + pushFrame(std::move(f)); + return true; + }; + + if (packet.Serial != INT32_MIN) { + Frame_t frame = stream.CodecContext.decode(packet.Packet, ec); + + while (processAndPush(std::move(frame))) { + frame = [&] { + if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) + return stream.CodecContext.decode(packet.Packet, ec, true, false); + else if constexpr (MediaType == AVMEDIA_TYPE_AUDIO) + return stream.CodecContext.decode(packet.Packet, ec, false); + else { + return stream.CodecContext.decode(packet.Packet, ec); + } + }(); + } } else { // Flush decoder since packets are finished bool decode = true; @@ -673,7 +744,8 @@ void FFmpegPlayer::Update(float dt) { PreviousFrameTimestamp = frame->Timestamp; if (frame->Frame.pixelFormat() == AV_PIX_FMT_NV12) { auto& nv12Frame = std::get(VideoTexture); - nv12Frame->Submit(frame->Frame.data(0), frame->Frame.data(1)); + nv12Frame->Submit(frame->Frame.data(0), frame->Frame.raw()->linesize[0], + frame->Frame.data(1), frame->Frame.raw()->linesize[1]); } else if(frame->Frame.pixelFormat() == AV_PIX_FMT_YUV420P) { auto& yuvFrame = std::get(VideoTexture); yuvFrame->Submit(frame->Frame.data(0), frame->Frame.data(1), diff --git a/vcpkg.json b/vcpkg.json index d7c4ee2cf..b154f3227 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -35,17 +35,9 @@ "name": "ffmpeg", "default-features": false, "features": [ - "amf" + "vaapi" ], - "platform": "!osx" - }, - { - "name": "ffmpeg", - "default-features": false, - "features": [ - "nvcodec" - ], - "platform": "linux | windows" + "platform": "linux" }, "libwebp", "fmt", From 6bdf7c1724470fa8fc76290ee835359dfa316f19 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sun, 5 Apr 2026 15:30:15 -0700 Subject: [PATCH 6/8] Static only windows avcpp for now until symbols merge --- portfiles/avcpp/portfile.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/portfiles/avcpp/portfile.cmake b/portfiles/avcpp/portfile.cmake index f8b813802..ad38cb40d 100644 --- a/portfiles/avcpp/portfile.cmake +++ b/portfiles/avcpp/portfile.cmake @@ -1,3 +1,8 @@ +if(VCPKG_TARGET_IS_WINDOWS) + # avcpp doesn't export any symbols + vcpkg_check_linkage(ONLY_STATIC_LIBRARY) +endif() + vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO h4tr3d/avcpp From 7e66d2ec0308fbba5d36205728f0adddef9f3e77 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Thu, 9 Apr 2026 00:18:56 -0700 Subject: [PATCH 7/8] Fix finding ffmpeg deps --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d082f88e1..ff54278fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1199,6 +1199,10 @@ if (NOT DEFINED IMPACTO_DISABLE_FFMPEG) PkgConfig::FFmpeg ) else () + find_package(FFMPEG REQUIRED) + list(APPEND Impacto_Include_Dirs ${FFMPEG_INCLUDE_DIRS}) + list(APPEND Impacto_Libs ${FFMPEG_LIBRARIES}) + find_package(avcpp CONFIG REQUIRED) list(APPEND Impacto_Libs avcpp::avcpp From c65b3446b86764a8f067203fd27af9120b5129dd Mon Sep 17 00:00:00 2001 From: Lint Action Date: Thu, 9 Apr 2026 07:26:50 +0000 Subject: [PATCH 8/8] Fix code style issues with clang_format --- src/video/ffmpegplayer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index d23cc465f..4d8e1da7e 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -746,7 +746,7 @@ void FFmpegPlayer::Update(float dt) { auto& nv12Frame = std::get(VideoTexture); nv12Frame->Submit(frame->Frame.data(0), frame->Frame.raw()->linesize[0], frame->Frame.data(1), frame->Frame.raw()->linesize[1]); - } else if(frame->Frame.pixelFormat() == AV_PIX_FMT_YUV420P) { + } else if (frame->Frame.pixelFormat() == AV_PIX_FMT_YUV420P) { auto& yuvFrame = std::get(VideoTexture); yuvFrame->Submit(frame->Frame.data(0), frame->Frame.data(1), frame->Frame.data(2));