From 8f8438f1e4deae19628e9bef95a9453394b9195d Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Tue, 5 May 2026 23:43:53 +0200 Subject: [PATCH 01/28] Add video encode/decode round-trip sample Adds the VideoEncodeDecode sample with an animated compute-generated NV12 source, GPU encode/decode round-trip display, and CLI codec selection for H.264, H.265, and AV1. Uses the backend-neutral NRIVideo flow across Vulkan and D3D12: video sessions and pictures, encode feedback readback, decode picture state queries, Annex-B header generation, AV1 metadata preparation, and aligned NV12 upload/readback layouts. Integrates the sample into CMake, shader configuration, README, and NRI submodule expectations, gating the target on the required NRIVideo version so older NRI trees skip cleanly. --- .gitignore | 2 + CMakeLists.txt | 19 + External/NRIFramework | 2 +- README.md | 3 +- Shaders/Shaders.cfg | 1 + Shaders/VideoEncodePattern.cs.hlsl | 163 +++ Source/VideoEncodeDecode.cpp | 1928 ++++++++++++++++++++++++++++ 7 files changed, 2116 insertions(+), 2 deletions(-) create mode 100644 Shaders/VideoEncodePattern.cs.hlsl create mode 100644 Source/VideoEncodeDecode.cpp diff --git a/.gitignore b/.gitignore index e251e72..c10db7b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ pso_cache.bin # can be a symbolic link _Data +build +build-video-sample diff --git a/CMakeLists.txt b/CMakeLists.txt index d3557cd..b62c798 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,6 +190,25 @@ add_sample(Resources c) add_sample(SceneViewer cpp) add_sample(Triangle cpp) +set(NRI_VIDEO_HEADER "${NRI_SOURCE_DIR}/Include/Extensions/NRIVideo.h") +set(NRI_VIDEO_VERSION 0) +if(EXISTS "${NRI_VIDEO_HEADER}") + file(STRINGS "${NRI_VIDEO_HEADER}" NRI_VIDEO_VERSION_LINE REGEX "^#define NRI_VIDEO_VERSION ") + if(NRI_VIDEO_VERSION_LINE MATCHES "^#define NRI_VIDEO_VERSION +([0-9]+)") + set(NRI_VIDEO_VERSION "${CMAKE_MATCH_1}") + endif() +endif() + +if(NRI_VIDEO_VERSION GREATER_EQUAL 1) + add_sample(VideoEncodeDecode cpp) + + if(WIN32) + target_link_libraries(VideoEncodeDecode PRIVATE d3d12) + endif() +else() + message(STATUS "Skipping VideoEncodeDecode: selected NRI source does not provide NRIVideo version 1") +endif() + # Wrapper depends on Vulkan SDK availability if(DEFINED ENV{VULKAN_SDK}) add_sample(Wrapper cpp) diff --git a/External/NRIFramework b/External/NRIFramework index 520a19e..3f3ffdb 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit 520a19e30485d27884d532c203e91adc8f4a8d2f +Subproject commit 3f3ffdb3a3a9ea087582dff6a0981d1979958d2a diff --git a/README.md b/README.md index 75f2bfb..070b8e8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Or by running scripts only: ### CMake options - `DISABLE_SHADER_COMPILATION` - disable compilation of shaders (shaders can be built on other platform) +- `NRI_SOURCE_OVERRIDE_DIR` - use an alternate NRI source tree, for example an in-review NRI branch - `NRIF_USE_WAYLAND` - use Wayland instead of X11 on Linux ## How to run @@ -57,5 +58,5 @@ The executables from `_Bin` directory load resources from `_Data`, therefore the - Resources - various resources allocation related stuff - SceneViewer - loading & rendering of meshes with materials (also tests programmable sample locations, shading rate and pipeline statistics) - Triangle - simple textured triangle rendering (also multiview demonstration in _FLEXIBLE_ mode) +- VideoEncodeDecode - H.264/H.265/AV1 NV12 video encode/decode round trip; built only when the selected NRI source provides the NRIVideo extension - Wrapper - shows how to wrap native D3D11/D3D12/VK objects into *NRI* entities - diff --git a/Shaders/Shaders.cfg b/Shaders/Shaders.cfg index 54c53b1..3e7e9a7 100644 --- a/Shaders/Shaders.cfg +++ b/Shaders/Shaders.cfg @@ -8,6 +8,7 @@ Box5.fs.hlsl -T ps Box6.fs.hlsl -T ps Box7.fs.hlsl -T ps Compute.cs.hlsl -T cs +VideoEncodePattern.cs.hlsl -T cs DescriptorHeapIndexing.cs.hlsl -T cs -m 6_6 GenerateSceneDrawCalls.cs.hlsl -T cs Forward.fs.hlsl -T ps diff --git a/Shaders/VideoEncodePattern.cs.hlsl b/Shaders/VideoEncodePattern.cs.hlsl new file mode 100644 index 0000000..bf34cb1 --- /dev/null +++ b/Shaders/VideoEncodePattern.cs.hlsl @@ -0,0 +1,163 @@ +// © 2021 NVIDIA Corporation + +#include "NRI.hlsl" + +NRI_RESOURCE(RWBuffer, g_Nv12Buffer, u, 0, 0); +NRI_FORMAT("rgba8") NRI_RESOURCE(RWTexture2D, g_SourcePreview, u, 1, 0); +NRI_FORMAT("rgba8") NRI_RESOURCE(RWTexture2D, g_DecodePreview, u, 2, 0); + +static const uint OP_GENERATE_PATTERN = 0u; +static const uint OP_NV12_TO_PREVIEW = 1u; + +struct PatternRootConstants { + uint width; + uint height; + uint yOffsetBytes; + uint yRowPitchBytes; + uint uvRowPitchBytes; + uint uvOffsetBytes; + uint operation; + float time; + uint padding; + uint padding1; +}; + +NRI_ROOT_CONSTANTS(PatternRootConstants, g_Pattern, 0, 0); + +float3 MakePatternColor(float2 normalizedPixelPos, float time) { + const float fx = normalizedPixelPos.x; + const float fy = normalizedPixelPos.y; + const float cx = fx - 0.5f; + const float cy = fy - 0.5f; + const float radius = sqrt(cx * cx + cy * cy); + const float angle = atan2(cy, cx); + + const float sweep = sin(angle * 3.0f + time * 1.7f) * 0.5f + 0.5f; + const float rings = sin((radius * 16.0f - time * 1.25f) * 6.283185307179586f) * 0.5f + 0.5f; + const float diagonal = sin((fx * 5.0f + fy * 3.0f + time * 0.45f) * 6.283185307179586f) * 0.5f + 0.5f; + const float grid = (((uint)(fx * 16.0f) ^ (uint)(fy * 10.0f)) & 1) ? 0.08f : 0.0f; + + const float r = min(1.0f, 0.10f + 0.78f * sweep + 0.18f * diagonal + grid); + const float g = min(1.0f, 0.14f + 0.72f * rings + 0.20f * fy + grid); + const float b = min(1.0f, 0.18f + 0.52f * diagonal + 0.34f * (1.0f - radius) + grid); + + return float3(r, g, b); +} + +uint ClampToByte(float v) { + return (v <= 0.0f) ? 0u : (v >= 255.0f ? 255u : uint(v)); +} + +uint RGBToY(uint3 rgb) { + return ClampToByte(16.0f + 0.257f * float(rgb.x) + 0.504f * float(rgb.y) + 0.098f * float(rgb.z)); +} + +uint RGBToU(uint3 rgb) { + return ClampToByte(128.0f - 0.148f * float(rgb.x) - 0.291f * float(rgb.y) + 0.439f * float(rgb.z)); +} + +uint RGBToV(uint3 rgb) { + return ClampToByte(128.0f + 0.439f * float(rgb.x) - 0.368f * float(rgb.y) - 0.071f * float(rgb.z)); +} + +float3 YuvToRgb(uint y, uint u, uint v) { + const float yy = float(y); + const float uu = float(u) - 128.0f; + const float vv = float(v) - 128.0f; + + const float r = clamp((298.082f * (yy - 16.0f) + 408.583f * vv + 128.0f) / 256.0f, 0.0f, 255.0f); + const float g = clamp((298.082f * (yy - 16.0f) - 100.291f * uu - 208.120f * vv + 128.0f) / 256.0f, 0.0f, 255.0f); + const float b = clamp((298.082f * (yy - 16.0f) + 516.412f * uu + 128.0f) / 256.0f, 0.0f, 255.0f); + + return float3(r / 255.0f, g / 255.0f, b / 255.0f); +} + +float4 LoadPatternColor(uint px, uint py) { + return float4(MakePatternColor(float2(px, py) / float2(g_Pattern.width - 1u, g_Pattern.height - 1u), g_Pattern.time), 1.0f); +} + +void StorePreview(uint2 pixel, float4 color) { + if (g_Pattern.operation == OP_NV12_TO_PREVIEW) + g_DecodePreview[pixel] = color; + else + g_SourcePreview[pixel] = color; +} + +[numthreads(1, 1, 1)] +void main(uint3 dispatchThreadID : SV_DispatchThreadID) +{ + const uint blockX = dispatchThreadID.x * 4u; + const uint y = dispatchThreadID.y; + + if (blockX + 3u >= g_Pattern.width || y >= g_Pattern.height) + return; + + const float4 c0 = LoadPatternColor(blockX + 0u, y); + const float4 c1 = LoadPatternColor(blockX + 1u, y); + const float4 c2 = LoadPatternColor(blockX + 2u, y); + const float4 c3 = LoadPatternColor(blockX + 3u, y); + + if (g_Pattern.operation == OP_GENERATE_PATTERN) { + StorePreview(uint2(blockX + 0u, y), c0); + StorePreview(uint2(blockX + 1u, y), c1); + StorePreview(uint2(blockX + 2u, y), c2); + StorePreview(uint2(blockX + 3u, y), c3); + + const uint3 rgb0 = uint3(c0.rgb * 255.0f); + const uint3 rgb1 = uint3(c1.rgb * 255.0f); + const uint3 rgb2 = uint3(c2.rgb * 255.0f); + const uint3 rgb3 = uint3(c3.rgb * 255.0f); + + const uint y0 = RGBToY(rgb0); + const uint y1 = RGBToY(rgb1); + const uint y2 = RGBToY(rgb2); + const uint y3 = RGBToY(rgb3); + + const uint packedY = y0 | (y1 << 8u) | (y2 << 16u) | (y3 << 24u); + const uint yWordIndex = (g_Pattern.yOffsetBytes + y * g_Pattern.yRowPitchBytes + blockX) / 4u; + g_Nv12Buffer[yWordIndex] = packedY; + + if ((y & 1u) != 0u) + return; + if (y + 1u >= g_Pattern.height) + return; + + const float4 c4 = LoadPatternColor(blockX + 0u, y + 1u); + const float4 c5 = LoadPatternColor(blockX + 1u, y + 1u); + const float4 c6 = LoadPatternColor(blockX + 2u, y + 1u); + const float4 c7 = LoadPatternColor(blockX + 3u, y + 1u); + + const uint3 rgb4 = uint3(c4.rgb * 255.0f); + const uint3 rgb5 = uint3(c5.rgb * 255.0f); + const uint3 rgb6 = uint3(c6.rgb * 255.0f); + const uint3 rgb7 = uint3(c7.rgb * 255.0f); + + const uint u0 = (RGBToU(rgb0) + RGBToU(rgb1) + RGBToU(rgb4) + RGBToU(rgb5) + 2u) >> 2u; + const uint v0 = (RGBToV(rgb0) + RGBToV(rgb1) + RGBToV(rgb4) + RGBToV(rgb5) + 2u) >> 2u; + const uint u1 = (RGBToU(rgb2) + RGBToU(rgb3) + RGBToU(rgb6) + RGBToU(rgb7) + 2u) >> 2u; + const uint v1 = (RGBToV(rgb2) + RGBToV(rgb3) + RGBToV(rgb6) + RGBToV(rgb7) + 2u) >> 2u; + + const uint packedUV = u0 | (v0 << 8u) | (u1 << 16u) | (v1 << 24u); + const uint uvWordIndex = (g_Pattern.uvOffsetBytes + ((y >> 1u) * g_Pattern.uvRowPitchBytes + blockX)) / 4u; + g_Nv12Buffer[uvWordIndex] = packedUV; + return; + } + + if (g_Pattern.operation == OP_NV12_TO_PREVIEW) { + const uint uvBase = (g_Pattern.uvOffsetBytes + (y >> 1u) * g_Pattern.uvRowPitchBytes + blockX) + ((y & 1u) * 0u); + for (uint i = 0; i < 4; i++) { + const uint px = blockX + i; + const uint yIndex = y * g_Pattern.yRowPitchBytes + px + g_Pattern.yOffsetBytes; + const uint yWord = g_Nv12Buffer[yIndex / 4u]; + const uint yValue = (yWord >> ((yIndex & 3u) * 8u)) & 255u; + + const uint uvOffset = uvBase + (i & 2u); + const uint uvWord = g_Nv12Buffer[(uvOffset) / 4u]; + const uint uvShift = (uvOffset & 3u) * 8u; + const uint uValue = (uvWord >> uvShift) & 255u; + const uint vValue = (uvWord >> (uvShift + 8u)) & 255u; + + StorePreview(uint2(px, y), float4(YuvToRgb(yValue, uValue, vValue), 1.0f)); + } + } +} diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp new file mode 100644 index 0000000..a2bc461 --- /dev/null +++ b/Source/VideoEncodeDecode.cpp @@ -0,0 +1,1928 @@ +// © 2021 NVIDIA Corporation + +#if defined(_WIN32) +# include +#endif + +#include "NRIFramework.h" + +#include "Extensions/NRIVideo.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t VIDEO_WIDTH = 1920; +constexpr uint32_t VIDEO_HEIGHT = 1088; +constexpr double ROUND_TRIP_INTERVAL_SEC = 1.0 / 60.0; +constexpr uint64_t BITSTREAM_SIZE = 2 * 1024 * 1024; +constexpr uint64_t ENCODED_SLICE_OFFSET = 4096; +constexpr uint64_t AV1_HEADER_READBACK_SIZE = 4096; +constexpr uint64_t METADATA_SIZE = 4 * 1024 * 1024; +constexpr uint64_t RESOLVED_METADATA_SIZE = 4096; + +static_assert(VIDEO_WIDTH % 4 == 0, "Compute-backed NV12 writer expects width divisible by 4"); +static_assert(VIDEO_WIDTH % 16 == 0, "H.264 macroblock width should stay aligned"); +static_assert(VIDEO_HEIGHT % 16 == 0, "H.264 macroblock height should stay aligned"); + +enum PatternOperation : uint32_t { + OP_GENERATE_PATTERN = 0, + OP_NV12_TO_PREVIEW = 1, +}; + +enum class SampleCodec : uint8_t { + H264, + H265, + AV1, +}; + +static const char* GetCodecName(SampleCodec codec) { + switch (codec) { + case SampleCodec::H265: + return "H.265"; + case SampleCodec::AV1: + return "AV1"; + case SampleCodec::H264: + default: + return "H.264"; + } +} + +static nri::VideoCodec GetNriCodec(SampleCodec codec) { + switch (codec) { + case SampleCodec::H265: + return nri::VideoCodec::H265; + case SampleCodec::AV1: + return nri::VideoCodec::AV1; + case SampleCodec::H264: + default: + return nri::VideoCodec::H264; + } +} + +static uint64_t GetEncodedPayloadHeaderSkip(SampleCodec codec, uint64_t encodedBitstreamBytes) { + const uint64_t headerSize = codec == SampleCodec::H264 ? 1 : 0; + return std::min(headerSize, encodedBitstreamBytes); +} + +static nri::VideoAV1SequenceDesc MakeAV1SequenceDesc() { + nri::VideoAV1SequenceDesc desc = {}; + desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | + nri::VideoAV1SequenceBits::ENABLE_CDEF | + nri::VideoAV1SequenceBits::ENABLE_RESTORATION | + nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; + desc.bitDepth = 8; + desc.subsamplingX = 1; + desc.subsamplingY = 1; + desc.maxFrameWidthMinus1 = VIDEO_WIDTH - 1; + desc.maxFrameHeightMinus1 = VIDEO_HEIGHT - 1; + desc.frameWidthBitsMinus1 = 15; + desc.frameHeightBitsMinus1 = 15; + desc.orderHintBitsMinus1 = 7; + desc.seqForceIntegerMv = 2; + desc.seqForceScreenContentTools = 2; + desc.colorPrimaries = 1; + desc.transferCharacteristics = 1; + desc.matrixCoefficients = 1; + desc.chromaSamplePosition = 1; + return desc; +} + +struct QueuedFrame { + nri::CommandAllocator* commandAllocator = nullptr; + nri::CommandBuffer* commandBuffer = nullptr; +}; + +struct PatternConstants { + uint32_t width = VIDEO_WIDTH; + uint32_t height = VIDEO_HEIGHT; + uint32_t yOffsetBytes = 0; + uint32_t yRowPitchBytes = 0; + uint32_t uvRowPitchBytes = 0; + uint32_t uvOffsetBytes = 0; + uint32_t operation = OP_GENERATE_PATTERN; + float time = 0.0f; + uint32_t _padding = 0; + uint32_t _padding1 = 0; +}; + +struct Nv12BufferLayout { + uint32_t yRowPitchBytes = VIDEO_WIDTH; + uint32_t ySlicePitchBytes = VIDEO_WIDTH * VIDEO_HEIGHT; + uint64_t uvOffsetBytes = uint64_t(VIDEO_WIDTH) * VIDEO_HEIGHT; + uint32_t uvRowPitchBytes = VIDEO_WIDTH; + uint32_t uvSlicePitchBytes = VIDEO_WIDTH * VIDEO_HEIGHT / 2; + uint64_t totalSizeBytes = uint64_t(VIDEO_WIDTH) * VIDEO_HEIGHT * 3 / 2; +}; + +static uint64_t AlignUp(uint64_t value, uint64_t alignment) { + return alignment == 0 ? value : ((value + alignment - 1) / alignment) * alignment; +} + +static Nv12BufferLayout MakeNv12BufferLayout(const nri::DeviceDesc& deviceDesc) { + const uint32_t rowAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureRow, 1u); + const uint32_t sliceAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureSlice, 1u); + + Nv12BufferLayout layout = {}; + layout.yRowPitchBytes = (uint32_t)AlignUp(VIDEO_WIDTH, rowAlignment); + layout.ySlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.yRowPitchBytes) * VIDEO_HEIGHT, sliceAlignment); + layout.uvOffsetBytes = layout.ySlicePitchBytes; + layout.uvRowPitchBytes = (uint32_t)AlignUp(VIDEO_WIDTH, rowAlignment); + layout.uvSlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.uvRowPitchBytes) * (VIDEO_HEIGHT / 2), sliceAlignment); + layout.totalSizeBytes = layout.uvOffsetBytes + layout.uvSlicePitchBytes; + return layout; +} + +template +static bool SubmitOneTime(nri::CoreInterface& core, nri::Queue& queue, Record&& record) { + nri::CommandAllocator* allocator = nullptr; + nri::CommandBuffer* commandBuffer = nullptr; + bool ok = core.CreateCommandAllocator(queue, allocator) == nri::Result::SUCCESS && allocator && core.CreateCommandBuffer(*allocator, commandBuffer) == nri::Result::SUCCESS && commandBuffer && core.BeginCommandBuffer(*commandBuffer, nullptr) == nri::Result::SUCCESS; + if (ok) { + std::forward(record)(*commandBuffer); + ok = core.EndCommandBuffer(*commandBuffer) == nri::Result::SUCCESS; + } + if (ok) { + const nri::CommandBuffer* commandBuffers[] = {commandBuffer}; + nri::QueueSubmitDesc submit = {}; + submit.commandBuffers = commandBuffers; + submit.commandBufferNum = 1; + ok = core.QueueSubmit(queue, submit) == nri::Result::SUCCESS && core.QueueWaitIdle(&queue) == nri::Result::SUCCESS; + } + if (commandBuffer) + core.DestroyCommandBuffer(commandBuffer); + if (allocator) + core.DestroyCommandAllocator(allocator); + return ok; +} + +template +static bool SubmitOneTime( + nri::CoreInterface& core, nri::Queue& queue, nri::DescriptorPool* descriptorPool, Record&& record) { + nri::CommandAllocator* allocator = nullptr; + nri::CommandBuffer* commandBuffer = nullptr; + bool ok = core.CreateCommandAllocator(queue, allocator) == nri::Result::SUCCESS && allocator && core.CreateCommandBuffer(*allocator, commandBuffer) == nri::Result::SUCCESS && commandBuffer && + core.BeginCommandBuffer(*commandBuffer, descriptorPool) == nri::Result::SUCCESS; + if (ok) { + std::forward(record)(*commandBuffer); + ok = core.EndCommandBuffer(*commandBuffer) == nri::Result::SUCCESS; + } + if (ok) { + const nri::CommandBuffer* commandBuffers[] = {commandBuffer}; + nri::QueueSubmitDesc submit = {}; + submit.commandBuffers = commandBuffers; + submit.commandBufferNum = 1; + ok = core.QueueSubmit(queue, submit) == nri::Result::SUCCESS && core.QueueWaitIdle(&queue) == nri::Result::SUCCESS; + } + if (commandBuffer) + core.DestroyCommandBuffer(commandBuffer); + if (allocator) + core.DestroyCommandAllocator(allocator); + return ok; +} + +static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, const Nv12BufferLayout& layout, nri::Buffer& src, nri::Texture& dst) { + return SubmitOneTime(core, queue, [&](nri::CommandBuffer& commandBuffer) { + nri::BufferBarrierDesc bufferBarrier = {}; + bufferBarrier.buffer = &src; + bufferBarrier.before = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + bufferBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + + nri::TextureBarrierDesc textureBarrier = {}; + textureBarrier.texture = &dst; + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarrier.after = {nri::AccessBits::COPY_DESTINATION, nri::Layout::COPY_DESTINATION, nri::StageBits::COPY}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = &bufferBarrier; + barrierDesc.bufferNum = 1; + barrierDesc.textures = &textureBarrier; + barrierDesc.textureNum = 1; + core.CmdBarrier(commandBuffer, barrierDesc); + + nri::TextureRegionDesc lumaRegion = {}; + lumaRegion.width = VIDEO_WIDTH; + lumaRegion.height = VIDEO_HEIGHT; + lumaRegion.depth = 1; + lumaRegion.planes = nri::PlaneBits::PLANE_0; + + nri::TextureDataLayoutDesc lumaLayout = {}; + lumaLayout.rowPitch = layout.yRowPitchBytes; + lumaLayout.slicePitch = layout.ySlicePitchBytes; + core.CmdUploadBufferToTexture(commandBuffer, dst, lumaRegion, src, lumaLayout); + + nri::TextureRegionDesc chromaRegion = {}; + chromaRegion.width = VIDEO_WIDTH; + chromaRegion.height = VIDEO_HEIGHT; + chromaRegion.depth = 1; + chromaRegion.planes = nri::PlaneBits::PLANE_1; + + nri::TextureDataLayoutDesc chromaLayout = {}; + chromaLayout.offset = layout.uvOffsetBytes; + chromaLayout.rowPitch = layout.uvRowPitchBytes; + chromaLayout.slicePitch = layout.uvSlicePitchBytes; + core.CmdUploadBufferToTexture(commandBuffer, dst, chromaRegion, src, chromaLayout); + + textureBarrier.before = textureBarrier.after; + textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + bufferBarrier.before = bufferBarrier.after; + bufferBarrier.after = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + core.CmdBarrier(commandBuffer, barrierDesc); + }); +} + +} // namespace + +class Sample : public SampleBase { +public: + Sample() = default; + ~Sample(); + + bool Initialize(nri::GraphicsAPI graphicsAPI, bool) override; + void InitCmdLine(cmdline::parser& cmdLine) override; + void ReadCmdLine(cmdline::parser& cmdLine) override; + void LatencySleep(uint32_t frameIndex) override; + void PrepareFrame(uint32_t frameIndex) override; + void RenderFrame(uint32_t frameIndex) override; + +private: + bool InitializeGraphics(nri::GraphicsAPI graphicsAPI); + bool TryInitializePreviewTextures(nri::GraphicsAPI graphicsAPI); + void InitializeGeneratedFrames(float timeSec); + bool CanRunRoundTrip() const; + void TryInitializeVideo(nri::GraphicsAPI graphicsAPI); + PatternConstants MakePatternConstants(PatternOperation operation, float timeSec) const; + bool GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage = false); + bool WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders); + bool TrySubmitEncodeAndMetadataReadback(float timeSec); + bool TryDecodePendingMetadata(float timeSec); + bool DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo, float timeSec); + bool TryRunRoundTrip(float timeSec); + void DrawTexturePanel(const char* label, nri::Descriptor* texture, const ImVec2& size); + +private: + NRIInterface NRI = {}; + nri::VideoInterface Video = {}; + nri::GraphicsAPI m_GraphicsAPI = nri::GraphicsAPI::NONE; + + nri::Device* m_Device = nullptr; + nri::Streamer* m_Streamer = nullptr; + nri::SwapChain* m_SwapChain = nullptr; + nri::Queue* m_GraphicsQueue = nullptr; + nri::Queue* m_VideoEncodeQueue = nullptr; + nri::Queue* m_VideoDecodeQueue = nullptr; + nri::Fence* m_FrameFence = nullptr; + + nri::VideoSession* m_EncodeSession = nullptr; + nri::VideoSession* m_DecodeSession = nullptr; + nri::VideoSessionParameters* m_EncodeParameters = nullptr; + nri::VideoSessionParameters* m_DecodeParameters = nullptr; + nri::Texture* m_EncodeTexture = nullptr; + nri::Texture* m_ReconstructedTexture = nullptr; + nri::Texture* m_DecodeTexture = nullptr; + nri::Texture* m_SourcePreviewTexture = nullptr; + nri::Texture* m_DecodePreviewTexture = nullptr; + nri::Buffer* m_UploadBuffer = nullptr; + nri::Descriptor* m_UploadBufferView = nullptr; + nri::Descriptor* m_SourcePreviewStorage = nullptr; + nri::Descriptor* m_DecodePreviewStorage = nullptr; + nri::Descriptor* m_SourcePreviewTextureView = nullptr; + nri::Descriptor* m_DecodePreviewTextureView = nullptr; + nri::PipelineLayout* m_GeneratePipelineLayout = nullptr; + nri::Pipeline* m_GenerateComputePipeline = nullptr; + nri::DescriptorPool* m_GenerateDescriptorPool = nullptr; + nri::DescriptorSet* m_GenerateDescriptorSet = nullptr; + nri::Buffer* m_BitstreamHeaderUploadBuffer = nullptr; + nri::Buffer* m_BitstreamHeaderReadbackBuffer = nullptr; + nri::Buffer* m_BitstreamBuffer = nullptr; + nri::Buffer* m_DecodeBitstreamBuffer = nullptr; + nri::Buffer* m_MetadataBuffer = nullptr; + nri::Buffer* m_ResolvedMetadataBuffer = nullptr; + nri::Buffer* m_ResolvedMetadataReadbackBuffer = nullptr; + nri::VideoPicture* m_EncodePicture = nullptr; + nri::VideoPicture* m_ReconstructedPicture = nullptr; + nri::VideoPicture* m_DecodePicture = nullptr; + nri::CommandAllocator* m_MetadataReadbackCommandAllocator = nullptr; + nri::CommandBuffer* m_MetadataReadbackCommandBuffer = nullptr; + nri::Fence* m_MetadataReadbackFence = nullptr; + + std::vector m_QueuedFrames; + std::vector m_SwapChainTextures; + + Nv12BufferLayout m_Nv12Layout = {}; + nri::Format m_SwapChainFormat = nri::Format::UNKNOWN; + std::string m_VideoStatus = "Initializing video"; + std::string m_PreviewStatus = "Initializing preview"; + std::string m_CodecArg = "H264"; + SampleCodec m_Codec = SampleCodec::H264; + nri::VideoH264SequenceParameterSetDesc m_H264Sps = {}; + nri::VideoH264PictureParameterSetDesc m_H264Pps = {}; + nri::VideoH265VideoParameterSetDesc m_H265Vps = {}; + nri::VideoH265SequenceParameterSetDesc m_H265Sps = {}; + nri::VideoH265PictureParameterSetDesc m_H265Pps = {}; + nri::VideoAV1SequenceDesc m_AV1Sequence = {}; + double m_StartTimeSec = 0.0; + double m_LastRoundTripTimeSec = -1.0; + bool m_VideoReady = false; + bool m_DecodePreviewReady = false; + bool m_PreviewTexturesShaderReadable = false; + bool m_MetadataReadbackPending = false; + uint64_t m_MetadataReadbackFenceValue = 0; +}; + +Sample::~Sample() { + if (NRI.HasCore()) { + NRI.DeviceWaitIdle(m_Device); + + if (Video.DestroyVideoPicture) { + if (m_DecodePicture) + Video.DestroyVideoPicture(*m_DecodePicture); + if (m_ReconstructedPicture) + Video.DestroyVideoPicture(*m_ReconstructedPicture); + if (m_EncodePicture) + Video.DestroyVideoPicture(*m_EncodePicture); + if (m_DecodeParameters) + Video.DestroyVideoSessionParameters(*m_DecodeParameters); + if (m_EncodeParameters) + Video.DestroyVideoSessionParameters(*m_EncodeParameters); + if (m_DecodeSession) + Video.DestroyVideoSession(*m_DecodeSession); + if (m_EncodeSession) + Video.DestroyVideoSession(*m_EncodeSession); + } + + if (m_MetadataReadbackCommandBuffer) + NRI.DestroyCommandBuffer(m_MetadataReadbackCommandBuffer); + if (m_MetadataReadbackCommandAllocator) + NRI.DestroyCommandAllocator(m_MetadataReadbackCommandAllocator); + if (m_MetadataReadbackFence) + NRI.DestroyFence(m_MetadataReadbackFence); + if (m_ResolvedMetadataReadbackBuffer) + NRI.DestroyBuffer(m_ResolvedMetadataReadbackBuffer); + if (m_ResolvedMetadataBuffer) + NRI.DestroyBuffer(m_ResolvedMetadataBuffer); + if (m_GenerateDescriptorPool) + NRI.DestroyDescriptorPool(m_GenerateDescriptorPool); + if (m_GenerateComputePipeline) + NRI.DestroyPipeline(m_GenerateComputePipeline); + if (m_GeneratePipelineLayout) + NRI.DestroyPipelineLayout(m_GeneratePipelineLayout); + if (m_UploadBufferView) + NRI.DestroyDescriptor(m_UploadBufferView); + if (m_MetadataBuffer) + NRI.DestroyBuffer(m_MetadataBuffer); + if (m_DecodeBitstreamBuffer) + NRI.DestroyBuffer(m_DecodeBitstreamBuffer); + if (m_BitstreamBuffer) + NRI.DestroyBuffer(m_BitstreamBuffer); + if (m_BitstreamHeaderReadbackBuffer) + NRI.DestroyBuffer(m_BitstreamHeaderReadbackBuffer); + if (m_BitstreamHeaderUploadBuffer) + NRI.DestroyBuffer(m_BitstreamHeaderUploadBuffer); + if (m_UploadBuffer) + NRI.DestroyBuffer(m_UploadBuffer); + if (m_SourcePreviewStorage) + NRI.DestroyDescriptor(m_SourcePreviewStorage); + if (m_DecodePreviewStorage) + NRI.DestroyDescriptor(m_DecodePreviewStorage); + if (m_SourcePreviewTextureView) + NRI.DestroyDescriptor(m_SourcePreviewTextureView); + if (m_DecodePreviewTextureView) + NRI.DestroyDescriptor(m_DecodePreviewTextureView); + if (m_SourcePreviewTexture) + NRI.DestroyTexture(m_SourcePreviewTexture); + if (m_DecodePreviewTexture) + NRI.DestroyTexture(m_DecodePreviewTexture); + if (m_DecodeTexture) + NRI.DestroyTexture(m_DecodeTexture); + if (m_ReconstructedTexture) + NRI.DestroyTexture(m_ReconstructedTexture); + if (m_EncodeTexture) + NRI.DestroyTexture(m_EncodeTexture); + + for (QueuedFrame& queuedFrame : m_QueuedFrames) { + NRI.DestroyCommandBuffer(queuedFrame.commandBuffer); + NRI.DestroyCommandAllocator(queuedFrame.commandAllocator); + } + + for (SwapChainTexture& swapChainTexture : m_SwapChainTextures) { + NRI.DestroyFence(swapChainTexture.acquireSemaphore); + NRI.DestroyFence(swapChainTexture.releaseSemaphore); + NRI.DestroyDescriptor(swapChainTexture.colorAttachment); + } + + NRI.DestroyFence(m_FrameFence); + } + + if (NRI.HasSwapChain()) + NRI.DestroySwapChain(m_SwapChain); + + if (NRI.HasStreamer()) + NRI.DestroyStreamer(m_Streamer); + + DestroyImgui(); + + nri::nriDestroyDevice(m_Device); +} + +void Sample::InitCmdLine(cmdline::parser& cmdLine) { + cmdLine.add("codec", 0, "video codec: H264, H265, or AV1", false, m_CodecArg, cmdline::oneof("H264", "H265", "AV1")); +} + +void Sample::ReadCmdLine(cmdline::parser& cmdLine) { + m_CodecArg = cmdLine.get("codec"); + m_Codec = m_CodecArg == "H265" ? SampleCodec::H265 : (m_CodecArg == "AV1" ? SampleCodec::AV1 : SampleCodec::H264); +} + +bool Sample::Initialize(nri::GraphicsAPI graphicsAPI, bool) { + m_GraphicsAPI = graphicsAPI; + if (!InitializeGraphics(graphicsAPI)) + return false; + + m_StartTimeSec = m_Timer.GetTimeStamp() * 0.001; + if (!TryInitializePreviewTextures(graphicsAPI)) + return false; + TryInitializeVideo(graphicsAPI); + InitializeGeneratedFrames(0.0f); + + return InitImgui(*m_Device); +} + +bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { + nri::AdapterDesc adapterDesc[2] = {}; + uint32_t adapterDescsNum = helper::GetCountOf(adapterDesc); + NRI_ABORT_ON_FAILURE(nri::nriEnumerateAdapters(adapterDesc, adapterDescsNum)); + + nri::DeviceCreationDesc deviceCreationDesc = {}; + nri::QueueFamilyDesc queueFamilies[] = { + {nullptr, 1, nri::QueueType::GRAPHICS}, + {nullptr, 1, nri::QueueType::VIDEO_ENCODE}, + {nullptr, 1, nri::QueueType::VIDEO_DECODE}, + }; + + deviceCreationDesc.graphicsAPI = graphicsAPI; + deviceCreationDesc.enableGraphicsAPIValidation = m_DebugAPI; + deviceCreationDesc.enableNRIValidation = m_DebugNRI; + deviceCreationDesc.enableD3D11CommandBufferEmulation = D3D11_ENABLE_COMMAND_BUFFER_EMULATION; + deviceCreationDesc.disableD3D12EnhancedBarriers = D3D12_DISABLE_ENHANCED_BARRIERS; + deviceCreationDesc.vkBindingOffsets = VK_BINDING_OFFSETS; + deviceCreationDesc.adapterDesc = &adapterDesc[std::min(m_AdapterIndex, adapterDescsNum - 1)]; + deviceCreationDesc.allocationCallbacks = m_AllocationCallbacks; + deviceCreationDesc.queueFamilies = queueFamilies; + deviceCreationDesc.queueFamilyNum = helper::GetCountOf(queueFamilies); + NRI_ABORT_ON_FAILURE(nri::nriCreateDevice(deviceCreationDesc, m_Device)); + + NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::CoreInterface), (nri::CoreInterface*)&NRI)); + NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::HelperInterface), (nri::HelperInterface*)&NRI)); + NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::StreamerInterface), (nri::StreamerInterface*)&NRI)); + NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::SwapChainInterface), (nri::SwapChainInterface*)&NRI)); + + m_Nv12Layout = MakeNv12BufferLayout(NRI.GetDeviceDesc(*m_Device)); + + nri::StreamerDesc streamerDesc = {}; + streamerDesc.dynamicBufferMemoryLocation = nri::MemoryLocation::HOST_UPLOAD; + streamerDesc.dynamicBufferDesc = {0, 0, nri::BufferUsageBits::VERTEX_BUFFER | nri::BufferUsageBits::INDEX_BUFFER}; + streamerDesc.constantBufferMemoryLocation = nri::MemoryLocation::HOST_UPLOAD; + streamerDesc.queuedFrameNum = GetQueuedFrameNum(); + NRI_ABORT_ON_FAILURE(NRI.CreateStreamer(*m_Device, streamerDesc, m_Streamer)); + + NRI_ABORT_ON_FAILURE(NRI.GetQueue(*m_Device, nri::QueueType::GRAPHICS, 0, m_GraphicsQueue)); + NRI_ABORT_ON_FAILURE(NRI.CreateFence(*m_Device, 0, m_FrameFence)); + + nri::SwapChainDesc swapChainDesc = {}; + swapChainDesc.window = GetWindow(); + swapChainDesc.queue = m_GraphicsQueue; + swapChainDesc.format = nri::SwapChainFormat::BT709_G22_8BIT; + swapChainDesc.flags = (m_Vsync ? nri::SwapChainBits::VSYNC : nri::SwapChainBits::NONE) | nri::SwapChainBits::ALLOW_TEARING; + swapChainDesc.width = (uint16_t)GetOutputResolution().x; + swapChainDesc.height = (uint16_t)GetOutputResolution().y; + swapChainDesc.textureNum = GetOptimalSwapChainTextureNum(); + swapChainDesc.queuedFrameNum = GetQueuedFrameNum(); + NRI_ABORT_ON_FAILURE(NRI.CreateSwapChain(*m_Device, swapChainDesc, m_SwapChain)); + + uint32_t swapChainTextureNum; + nri::Texture* const* swapChainTextures = NRI.GetSwapChainTextures(*m_SwapChain, swapChainTextureNum); + m_SwapChainFormat = NRI.GetTextureDesc(*swapChainTextures[0]).format; + + for (uint32_t i = 0; i < swapChainTextureNum; i++) { + nri::TextureViewDesc textureViewDesc = {swapChainTextures[i], nri::TextureView::COLOR_ATTACHMENT, m_SwapChainFormat}; + + nri::Descriptor* colorAttachment = nullptr; + NRI_ABORT_ON_FAILURE(NRI.CreateTextureView(textureViewDesc, colorAttachment)); + + nri::Fence* acquireSemaphore = nullptr; + NRI_ABORT_ON_FAILURE(NRI.CreateFence(*m_Device, nri::SWAPCHAIN_SEMAPHORE, acquireSemaphore)); + + nri::Fence* releaseSemaphore = nullptr; + NRI_ABORT_ON_FAILURE(NRI.CreateFence(*m_Device, nri::SWAPCHAIN_SEMAPHORE, releaseSemaphore)); + + SwapChainTexture& swapChainTexture = m_SwapChainTextures.emplace_back(); + swapChainTexture = {}; + swapChainTexture.acquireSemaphore = acquireSemaphore; + swapChainTexture.releaseSemaphore = releaseSemaphore; + swapChainTexture.texture = swapChainTextures[i]; + swapChainTexture.colorAttachment = colorAttachment; + swapChainTexture.attachmentFormat = m_SwapChainFormat; + } + + m_QueuedFrames.resize(GetQueuedFrameNum()); + for (QueuedFrame& queuedFrame : m_QueuedFrames) { + NRI_ABORT_ON_FAILURE(NRI.CreateCommandAllocator(*m_GraphicsQueue, queuedFrame.commandAllocator)); + NRI_ABORT_ON_FAILURE(NRI.CreateCommandBuffer(*queuedFrame.commandAllocator, queuedFrame.commandBuffer)); + } + + return true; +} + +PatternConstants Sample::MakePatternConstants(PatternOperation operation, float timeSec) const { + PatternConstants patternConstants = {}; + patternConstants.width = VIDEO_WIDTH; + patternConstants.height = VIDEO_HEIGHT; + patternConstants.yRowPitchBytes = m_Nv12Layout.yRowPitchBytes; + patternConstants.uvRowPitchBytes = m_Nv12Layout.uvRowPitchBytes; + patternConstants.uvOffsetBytes = (uint32_t)m_Nv12Layout.uvOffsetBytes; + patternConstants.operation = operation; + patternConstants.time = timeSec; + return patternConstants; +} + +void Sample::InitializeGeneratedFrames(float timeSec) { + if (!m_SourcePreviewTexture || !m_SourcePreviewStorage || !m_UploadBuffer || !m_UploadBufferView) + return; + + PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); + + if (m_EncodeTexture) { + if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { + m_PreviewStatus = "Failed to generate source pattern via compute"; + return; + } + + if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture)) { + m_PreviewStatus = "Failed to upload NV12 source to video texture"; + return; + } + m_PreviewStatus = "Source preview is generated by compute"; + return; + } + + if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { + m_PreviewStatus = "Failed to generate source pattern via compute"; + return; + } + + m_PreviewStatus = "Source preview is generated by compute"; +} + +bool Sample::GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage) { + if (!m_GeneratePipelineLayout || !m_GenerateComputePipeline || !m_GenerateDescriptorSet || !m_UploadBufferView || !previewTexture) + return false; + if ((constants.width % 4) != 0 || (constants.height % 2) != 0) + return false; + + const uint32_t dispatchX = (constants.width + 3) / 4; + const uint32_t dispatchY = constants.height; + + const bool previewTexturesShaderReadable = m_PreviewTexturesShaderReadable; + const bool submitted = SubmitOneTime( + NRI, + *m_GraphicsQueue, + m_GenerateDescriptorPool, + [this, &constants, dispatchX, dispatchY, previewTexturesShaderReadable, returnSourceBufferToShaderStorage](nri::CommandBuffer& commandBuffer) { + nri::SetDescriptorSetDesc descriptorSet = {0, m_GenerateDescriptorSet}; + + nri::BufferBarrierDesc bufferBarrier = {}; + bufferBarrier.buffer = m_UploadBuffer; + bufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + bufferBarrier.after = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + + nri::TextureBarrierDesc textureBarriers[2] = {}; + textureBarriers[0].texture = m_SourcePreviewTexture; + textureBarriers[1].texture = m_DecodePreviewTexture; + for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { + if (previewTexturesShaderReadable) + textureBarrier.before = {nri::AccessBits::SHADER_RESOURCE, nri::Layout::SHADER_RESOURCE, nri::StageBits::ALL}; + else + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::UNDEFINED, nri::StageBits::ALL}; + textureBarrier.after = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::Layout::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + } + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = &bufferBarrier; + barrierDesc.bufferNum = 1; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + + nri::SetRootConstantsDesc rootConstants = {0, &constants, sizeof(PatternConstants)}; + NRI.CmdBarrier(commandBuffer, barrierDesc); + NRI.CmdSetPipelineLayout(commandBuffer, nri::BindPoint::COMPUTE, *m_GeneratePipelineLayout); + NRI.CmdSetDescriptorSet(commandBuffer, descriptorSet); + NRI.CmdSetPipeline(commandBuffer, *m_GenerateComputePipeline); + NRI.CmdSetRootConstants(commandBuffer, rootConstants); + NRI.CmdDispatch(commandBuffer, {dispatchX, dispatchY, 1}); + + bufferBarrier.before = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + if (returnSourceBufferToShaderStorage) + bufferBarrier.after = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + else + bufferBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { + textureBarrier.before = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::Layout::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + textureBarrier.after = {nri::AccessBits::SHADER_RESOURCE, nri::Layout::SHADER_RESOURCE, nri::StageBits::ALL}; + } + NRI.CmdBarrier(commandBuffer, barrierDesc); + }); + + if (submitted) + m_PreviewTexturesShaderReadable = true; + + return submitted; +} + +bool Sample::TryInitializePreviewTextures(nri::GraphicsAPI) { + if (m_SourcePreviewTexture && m_DecodePreviewTexture && m_SourcePreviewTextureView && m_DecodePreviewTextureView) + return true; + + nri::TextureDesc previewTextureDesc = {}; + previewTextureDesc.type = nri::TextureType::TEXTURE_2D; + previewTextureDesc.format = nri::Format::RGBA8_UNORM; + previewTextureDesc.width = VIDEO_WIDTH; + previewTextureDesc.height = VIDEO_HEIGHT; + previewTextureDesc.mipNum = 1; + previewTextureDesc.layerNum = 1; + previewTextureDesc.usage = nri::TextureUsageBits::SHADER_RESOURCE | nri::TextureUsageBits::SHADER_RESOURCE_STORAGE; + + if (!m_SourcePreviewTexture) { + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, previewTextureDesc, m_SourcePreviewTexture) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create source preview texture"; + return false; + } + NRI.SetDebugName(m_SourcePreviewTexture, "VideoSourcePreviewTexture"); + } + + if (!m_DecodePreviewTexture) { + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, previewTextureDesc, m_DecodePreviewTexture) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create decode preview texture"; + return false; + } + NRI.SetDebugName(m_DecodePreviewTexture, "VideoDecodePreviewTexture"); + } + + if (!m_PreviewTexturesShaderReadable) { + const bool initialized = SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::TextureBarrierDesc textureBarriers[2] = {}; + textureBarriers[0].texture = m_SourcePreviewTexture; + textureBarriers[1].texture = m_DecodePreviewTexture; + + for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::UNDEFINED, nri::StageBits::ALL}; + textureBarrier.after = {nri::AccessBits::SHADER_RESOURCE, nri::Layout::SHADER_RESOURCE, nri::StageBits::ALL}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + } + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + NRI.CmdBarrier(commandBuffer, barrierDesc); + }); + + if (!initialized) { + m_PreviewStatus = "Failed to initialize preview texture layouts"; + return false; + } + + m_PreviewTexturesShaderReadable = true; + } + + nri::TextureViewDesc sourceTextureViewDesc = {m_SourcePreviewTexture, nri::TextureView::TEXTURE, previewTextureDesc.format}; + nri::TextureViewDesc decodeTextureViewDesc = {m_DecodePreviewTexture, nri::TextureView::TEXTURE, previewTextureDesc.format}; + + if (!m_SourcePreviewTextureView) { + if (NRI.CreateTextureView(sourceTextureViewDesc, m_SourcePreviewTextureView) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create source preview ImGui texture view"; + return false; + } + } + + if (!m_DecodePreviewTextureView) { + if (NRI.CreateTextureView(decodeTextureViewDesc, m_DecodePreviewTextureView) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create decode preview ImGui texture view"; + return false; + } + } + + nri::TextureViewDesc sourceStorageTextureViewDesc = {m_SourcePreviewTexture, nri::TextureView::STORAGE_TEXTURE, nri::Format::RGBA8_UNORM}; + nri::TextureViewDesc decodeStorageTextureViewDesc = {m_DecodePreviewTexture, nri::TextureView::STORAGE_TEXTURE, nri::Format::RGBA8_UNORM}; + + if (!m_SourcePreviewStorage) { + if (NRI.CreateTextureView(sourceStorageTextureViewDesc, m_SourcePreviewStorage) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create source preview storage texture view"; + return false; + } + } + + if (!m_DecodePreviewStorage) { + if (NRI.CreateTextureView(decodeStorageTextureViewDesc, m_DecodePreviewStorage) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create decode preview storage texture view"; + return false; + } + } + + if (!m_UploadBuffer) { + nri::BufferDesc uploadBufferDesc = {}; + uploadBufferDesc.size = m_Nv12Layout.totalSizeBytes; + uploadBufferDesc.usage = nri::BufferUsageBits::SHADER_RESOURCE_STORAGE; + + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, uploadBufferDesc, m_UploadBuffer) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create compute NV12 buffer"; + return false; + } + } + + if (!m_UploadBufferView) { + nri::BufferViewDesc uploadBufferViewDesc = {}; + uploadBufferViewDesc.buffer = m_UploadBuffer; + uploadBufferViewDesc.format = nri::Format::R32_UINT; + uploadBufferViewDesc.type = nri::BufferView::STORAGE_BUFFER; + uploadBufferViewDesc.size = NRI.GetBufferDesc(*m_UploadBuffer).size; + + if (NRI.CreateBufferView(uploadBufferViewDesc, m_UploadBufferView) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create compute NV12 buffer view"; + return false; + } + } + + if (!m_GeneratePipelineLayout) { + nri::DescriptorRangeDesc descriptorRanges[] = { + {0, 1, nri::DescriptorType::STORAGE_BUFFER, nri::StageBits::COMPUTE_SHADER}, + {1, 1, nri::DescriptorType::STORAGE_TEXTURE, nri::StageBits::COMPUTE_SHADER}, + {2, 1, nri::DescriptorType::STORAGE_TEXTURE, nri::StageBits::COMPUTE_SHADER}, + }; + nri::DescriptorSetDesc descriptorSetDescs[] = {{0, descriptorRanges, helper::GetCountOf(descriptorRanges)}}; + + nri::RootConstantDesc rootConstantDesc = {}; + rootConstantDesc.registerIndex = 0; + rootConstantDesc.size = sizeof(PatternConstants); + rootConstantDesc.shaderStages = nri::StageBits::COMPUTE_SHADER; + + nri::PipelineLayoutDesc pipelineLayoutDesc = {}; + pipelineLayoutDesc.rootConstantNum = 1; + pipelineLayoutDesc.rootConstants = &rootConstantDesc; + pipelineLayoutDesc.descriptorSetNum = helper::GetCountOf(descriptorSetDescs); + pipelineLayoutDesc.descriptorSets = descriptorSetDescs; + pipelineLayoutDesc.shaderStages = nri::StageBits::COMPUTE_SHADER; + if (NRI.CreatePipelineLayout(*m_Device, pipelineLayoutDesc, m_GeneratePipelineLayout) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create compute pipeline layout for pattern generation"; + return false; + } + } + + if (!m_GenerateComputePipeline) { + const nri::DeviceDesc& deviceDesc = NRI.GetDeviceDesc(*m_Device); + utils::ShaderCodeStorage shaderCodeStorage; + nri::ComputePipelineDesc computePipelineDesc = {}; + computePipelineDesc.pipelineLayout = m_GeneratePipelineLayout; + computePipelineDesc.shader = utils::LoadShader(deviceDesc.graphicsAPI, "VideoEncodePattern.cs", shaderCodeStorage); + if (NRI.CreateComputePipeline(*m_Device, computePipelineDesc, m_GenerateComputePipeline) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create pattern generation compute pipeline"; + return false; + } + } + + if (!m_GenerateDescriptorPool) { + nri::DescriptorPoolDesc descriptorPoolDesc = {}; + descriptorPoolDesc.descriptorSetMaxNum = 1; + descriptorPoolDesc.storageBufferMaxNum = 1; + descriptorPoolDesc.storageTextureMaxNum = 2; + if (NRI.CreateDescriptorPool(*m_Device, descriptorPoolDesc, m_GenerateDescriptorPool) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to create compute descriptor pool for pattern generation"; + return false; + } + } + + if (!m_GenerateDescriptorSet) { + if (NRI.AllocateDescriptorSets(*m_GenerateDescriptorPool, *m_GeneratePipelineLayout, 0, &m_GenerateDescriptorSet, 1, 0) != nri::Result::SUCCESS) { + m_PreviewStatus = "Failed to allocate compute descriptor set for pattern generation"; + return false; + } + + nri::UpdateDescriptorRangeDesc updateDescriptorRangeDescs[] = { + {m_GenerateDescriptorSet, 0, 0, &m_UploadBufferView, 1}, + {m_GenerateDescriptorSet, 1, 0, &m_SourcePreviewStorage, 1}, + {m_GenerateDescriptorSet, 2, 0, &m_DecodePreviewStorage, 1}, + }; + NRI.UpdateDescriptorRanges(updateDescriptorRangeDescs, helper::GetCountOf(updateDescriptorRangeDescs)); + } + + return true; +} + +void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { + const nri::DeviceDesc& deviceDesc = NRI.GetDeviceDesc(*m_Device); + + if (graphicsAPI == nri::GraphicsAPI::D3D11) { + m_VideoStatus = "D3D11 does not expose NRI video queues"; + return; + } + + if (!deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_ENCODE] || !deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_DECODE]) { + m_VideoStatus = "Adapter has no NRI video encode/decode queues"; + return; + } + + if (nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::VideoInterface), &Video) != nri::Result::SUCCESS || !Video.CreateVideoSession) { + m_VideoStatus = "NRI video interface is unavailable"; + return; + } + + nri::VideoSessionDesc encodeSessionDesc = {}; + encodeSessionDesc.usage = nri::VideoUsage::ENCODE; + encodeSessionDesc.codec = GetNriCodec(m_Codec); + encodeSessionDesc.format = nri::Format::NV12_UNORM; + encodeSessionDesc.width = VIDEO_WIDTH; + encodeSessionDesc.height = VIDEO_HEIGHT; + encodeSessionDesc.maxReferenceNum = 1; + + nri::VideoSessionDesc decodeSessionDesc = encodeSessionDesc; + decodeSessionDesc.usage = nri::VideoUsage::DECODE; + decodeSessionDesc.maxReferenceNum = 16; + + if (Video.GetVideoQueue(*m_Device, encodeSessionDesc, m_VideoEncodeQueue) != nri::Result::SUCCESS || Video.GetVideoQueue(*m_Device, decodeSessionDesc, m_VideoDecodeQueue) != nri::Result::SUCCESS) { + m_VideoStatus = std::string("Failed to get ") + GetCodecName(m_Codec) + "-capable video queues"; + return; + } + + if (Video.CreateVideoSession(*m_Device, encodeSessionDesc, m_EncodeSession) != nri::Result::SUCCESS) { + m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " encode session"; + return; + } + + if (Video.CreateVideoSession(*m_Device, decodeSessionDesc, m_DecodeSession) != nri::Result::SUCCESS) { + m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " decode session"; + return; + } + + nri::VideoH264SequenceParameterSetDesc sps = {}; + sps.flags = nri::VideoH264SequenceParameterSetBits::DIRECT_8X8_INFERENCE | nri::VideoH264SequenceParameterSetBits::FRAME_MBS_ONLY; + sps.profileIdc = 100; + sps.levelIdc = 42; + sps.chromaFormatIdc = 1; + sps.sequenceParameterSetId = 0; + sps.log2MaxFrameNumMinus4 = 0; + sps.pictureOrderCountType = 0; + sps.log2MaxPictureOrderCountLsbMinus4 = 0; + sps.referenceFrameNum = 1; + sps.pictureWidthInMbsMinus1 = VIDEO_WIDTH / 16 - 1; + sps.pictureHeightInMapUnitsMinus1 = VIDEO_HEIGHT / 16 - 1; + + nri::VideoH264PictureParameterSetDesc pps = {}; + pps.flags = nri::VideoH264PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; + pps.sequenceParameterSetId = 0; + pps.pictureParameterSetId = 0; + pps.refIndexL0DefaultActiveMinus1 = 0; + pps.refIndexL1DefaultActiveMinus1 = 0; + m_H264Sps = sps; + m_H264Pps = pps; + + nri::VideoH264SessionParametersDesc h264Parameters = {}; + h264Parameters.sequenceParameterSets = &sps; + h264Parameters.sequenceParameterSetNum = 1; + h264Parameters.pictureParameterSets = &pps; + h264Parameters.pictureParameterSetNum = 1; + h264Parameters.maxSequenceParameterSetNum = 1; + h264Parameters.maxPictureParameterSetNum = 1; + + nri::VideoH265VideoParameterSetDesc vps = {}; + vps.flags = nri::VideoH265VideoParameterSetBits::TEMPORAL_ID_NESTING; + vps.videoParameterSetId = 0; + vps.maxSubLayersMinus1 = 0; + vps.profileTierLevel.flags = nri::VideoH265ProfileTierLevelBits::FRAME_ONLY_CONSTRAINT; + vps.profileTierLevel.generalProfileIdc = 1; + vps.profileTierLevel.generalLevelIdc = 90; + vps.decPicBufMgr.maxDecPicBufferingMinus1[0] = 2; + vps.decPicBufMgr.maxNumReorderPics[0] = 1; + + nri::VideoH265SequenceParameterSetDesc h265Sps = {}; + h265Sps.flags = nri::VideoH265SequenceParameterSetBits::TEMPORAL_ID_NESTING | nri::VideoH265SequenceParameterSetBits::AMP_ENABLED | + nri::VideoH265SequenceParameterSetBits::SAMPLE_ADAPTIVE_OFFSET_ENABLED; + h265Sps.videoParameterSetId = vps.videoParameterSetId; + h265Sps.maxSubLayersMinus1 = vps.maxSubLayersMinus1; + h265Sps.sequenceParameterSetId = 0; + h265Sps.chromaFormatIdc = 1; + h265Sps.pictureWidthInLumaSamples = VIDEO_WIDTH; + h265Sps.pictureHeightInLumaSamples = VIDEO_HEIGHT; + h265Sps.log2MaxPictureOrderCountLsbMinus4 = 3; + h265Sps.log2MinLumaCodingBlockSizeMinus3 = 0; + h265Sps.log2DiffMaxMinLumaCodingBlockSize = 2; + h265Sps.log2MinLumaTransformBlockSizeMinus2 = 0; + h265Sps.log2DiffMaxMinLumaTransformBlockSize = 3; + h265Sps.maxTransformHierarchyDepthInter = 3; + h265Sps.maxTransformHierarchyDepthIntra = 3; + h265Sps.profileTierLevel = vps.profileTierLevel; + h265Sps.decPicBufMgr = vps.decPicBufMgr; + + nri::VideoH265PictureParameterSetDesc h265Pps = {}; + h265Pps.flags = nri::VideoH265PictureParameterSetBits::CABAC_INIT_PRESENT | nri::VideoH265PictureParameterSetBits::TRANSFORM_SKIP_ENABLED | + nri::VideoH265PictureParameterSetBits::CU_QP_DELTA_ENABLED | nri::VideoH265PictureParameterSetBits::SLICE_CHROMA_QP_OFFSETS_PRESENT | + nri::VideoH265PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; + h265Pps.pictureParameterSetId = 0; + h265Pps.sequenceParameterSetId = h265Sps.sequenceParameterSetId; + h265Pps.videoParameterSetId = vps.videoParameterSetId; + m_H265Vps = vps; + m_H265Sps = h265Sps; + m_H265Pps = h265Pps; + + nri::VideoH265SessionParametersDesc h265Parameters = {}; + h265Parameters.videoParameterSets = &vps; + h265Parameters.videoParameterSetNum = 1; + h265Parameters.sequenceParameterSets = &h265Sps; + h265Parameters.sequenceParameterSetNum = 1; + h265Parameters.pictureParameterSets = &h265Pps; + h265Parameters.pictureParameterSetNum = 1; + h265Parameters.maxVideoParameterSetNum = 1; + h265Parameters.maxSequenceParameterSetNum = 1; + h265Parameters.maxPictureParameterSetNum = 1; + + m_AV1Sequence = MakeAV1SequenceDesc(); + nri::VideoAV1SessionParametersDesc av1Parameters = {}; + av1Parameters.sequence = m_AV1Sequence; + + nri::VideoSessionParametersDesc encodeParametersDesc = {}; + encodeParametersDesc.session = m_EncodeSession; + encodeParametersDesc.h264Parameters = m_Codec == SampleCodec::H264 ? &h264Parameters : nullptr; + encodeParametersDesc.h265Parameters = m_Codec == SampleCodec::H265 ? &h265Parameters : nullptr; + encodeParametersDesc.av1Parameters = m_Codec == SampleCodec::AV1 ? &av1Parameters : nullptr; + + nri::VideoSessionParametersDesc decodeParametersDesc = {}; + decodeParametersDesc.session = m_DecodeSession; + decodeParametersDesc.h264Parameters = m_Codec == SampleCodec::H264 ? &h264Parameters : nullptr; + decodeParametersDesc.h265Parameters = m_Codec == SampleCodec::H265 ? &h265Parameters : nullptr; + decodeParametersDesc.av1Parameters = m_Codec == SampleCodec::AV1 ? &av1Parameters : nullptr; + + if (Video.CreateVideoSessionParameters(*m_Device, encodeParametersDesc, m_EncodeParameters) != nri::Result::SUCCESS) { + m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " encode parameters"; + return; + } + + if (Video.CreateVideoSessionParameters(*m_Device, decodeParametersDesc, m_DecodeParameters) != nri::Result::SUCCESS) { + m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " decode parameters"; + return; + } + + nri::TextureDesc encodeTextureDesc = {}; + encodeTextureDesc.type = nri::TextureType::TEXTURE_2D; + encodeTextureDesc.usage = nri::TextureUsageBits::VIDEO_ENCODE; + encodeTextureDesc.format = nri::Format::NV12_UNORM; + encodeTextureDesc.width = VIDEO_WIDTH; + encodeTextureDesc.height = VIDEO_HEIGHT; + encodeTextureDesc.mipNum = 1; + encodeTextureDesc.layerNum = 1; + + nri::TextureDesc decodeTextureDesc = encodeTextureDesc; + decodeTextureDesc.usage = nri::TextureUsageBits::VIDEO_DECODE; + + nri::VideoTextureDesc encodeVideoTextureDesc = {}; + encodeVideoTextureDesc.textureDesc = encodeTextureDesc; + encodeVideoTextureDesc.codec = GetNriCodec(m_Codec); + + nri::VideoTextureDesc decodeVideoTextureDesc = {}; + decodeVideoTextureDesc.textureDesc = decodeTextureDesc; + decodeVideoTextureDesc.codec = GetNriCodec(m_Codec); + + if (Video.CreateCommittedVideoTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeVideoTextureDesc, m_EncodeTexture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create NV12 encode texture"; + return; + } + NRI.SetDebugName(m_EncodeTexture, "VideoEncodeTexture"); + + if (Video.CreateCommittedVideoTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeVideoTextureDesc, m_ReconstructedTexture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create NV12 reconstructed texture"; + return; + } + NRI.SetDebugName(m_ReconstructedTexture, "VideoReconstructedTexture"); + + if (Video.CreateCommittedVideoTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, decodeVideoTextureDesc, m_DecodeTexture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create NV12 decode texture"; + return; + } + NRI.SetDebugName(m_DecodeTexture, "VideoDecodeTexture"); + + if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::TextureBarrierDesc textureBarriers[3] = {}; + textureBarriers[0].texture = m_EncodeTexture; + textureBarriers[1].texture = m_ReconstructedTexture; + textureBarriers[2].texture = m_DecodeTexture; + + for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::UNDEFINED, nri::StageBits::ALL}; + textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + } + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + NRI.CmdBarrier(commandBuffer, barrierDesc); + })) { + m_VideoStatus = "Failed to initialize video texture layouts"; + return; + } + + if (!m_GenerateComputePipeline) { + nri::TextureViewDesc sourceStorageTextureViewDesc = {m_SourcePreviewTexture, nri::TextureView::STORAGE_TEXTURE, nri::Format::RGBA8_UNORM}; + nri::TextureViewDesc decodeStorageTextureViewDesc = {m_DecodePreviewTexture, nri::TextureView::STORAGE_TEXTURE, nri::Format::RGBA8_UNORM}; + + if (!m_SourcePreviewStorage) { + if (NRI.CreateTextureView(sourceStorageTextureViewDesc, m_SourcePreviewStorage) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create source preview storage texture view"; + return; + } + } + + if (!m_DecodePreviewStorage) { + if (NRI.CreateTextureView(decodeStorageTextureViewDesc, m_DecodePreviewStorage) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create decode preview storage texture view"; + return; + } + } + + nri::BufferDesc uploadBufferDesc = {}; + uploadBufferDesc.size = m_Nv12Layout.totalSizeBytes; + uploadBufferDesc.usage = nri::BufferUsageBits::SHADER_RESOURCE_STORAGE; + + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, uploadBufferDesc, m_UploadBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create NV12 upload buffer"; + return; + } + + nri::BufferViewDesc uploadBufferViewDesc = {}; + uploadBufferViewDesc.buffer = m_UploadBuffer; + uploadBufferViewDesc.format = nri::Format::R32_UINT; + uploadBufferViewDesc.type = nri::BufferView::STORAGE_BUFFER; + uploadBufferViewDesc.size = uploadBufferDesc.size; + + if (NRI.CreateBufferView(uploadBufferViewDesc, m_UploadBufferView) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create NV12 compute output buffer view"; + return; + } + + utils::ShaderCodeStorage shaderCodeStorage; + { + nri::DescriptorRangeDesc descriptorRanges[] = { + {0, 1, nri::DescriptorType::STORAGE_BUFFER, nri::StageBits::COMPUTE_SHADER}, + {1, 1, nri::DescriptorType::STORAGE_TEXTURE, nri::StageBits::COMPUTE_SHADER}, + {2, 1, nri::DescriptorType::STORAGE_TEXTURE, nri::StageBits::COMPUTE_SHADER}, + }; + nri::DescriptorSetDesc descriptorSetDescs[] = {{0, descriptorRanges, helper::GetCountOf(descriptorRanges)}}; + + nri::RootConstantDesc rootConstantDesc = {}; + rootConstantDesc.registerIndex = 0; + rootConstantDesc.size = sizeof(PatternConstants); + rootConstantDesc.shaderStages = nri::StageBits::COMPUTE_SHADER; + + nri::PipelineLayoutDesc pipelineLayoutDesc = {}; + pipelineLayoutDesc.rootConstantNum = 1; + pipelineLayoutDesc.rootConstants = &rootConstantDesc; + pipelineLayoutDesc.descriptorSetNum = helper::GetCountOf(descriptorSetDescs); + pipelineLayoutDesc.descriptorSets = descriptorSetDescs; + pipelineLayoutDesc.shaderStages = nri::StageBits::COMPUTE_SHADER; + if (NRI.CreatePipelineLayout(*m_Device, pipelineLayoutDesc, m_GeneratePipelineLayout) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create compute pipeline layout for pattern generation"; + return; + } + + nri::ComputePipelineDesc computePipelineDesc = {}; + computePipelineDesc.pipelineLayout = m_GeneratePipelineLayout; + computePipelineDesc.shader = utils::LoadShader(deviceDesc.graphicsAPI, "VideoEncodePattern.cs", shaderCodeStorage); + if (NRI.CreateComputePipeline(*m_Device, computePipelineDesc, m_GenerateComputePipeline) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create pattern generation compute pipeline"; + return; + } + + nri::DescriptorPoolDesc descriptorPoolDesc = {}; + descriptorPoolDesc.descriptorSetMaxNum = 1; + descriptorPoolDesc.storageBufferMaxNum = 1; + descriptorPoolDesc.storageTextureMaxNum = 2; + descriptorPoolDesc.textureMaxNum = 2; + if (NRI.CreateDescriptorPool(*m_Device, descriptorPoolDesc, m_GenerateDescriptorPool) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create compute descriptor pool for pattern generation"; + return; + } + + if (NRI.AllocateDescriptorSets(*m_GenerateDescriptorPool, *m_GeneratePipelineLayout, 0, &m_GenerateDescriptorSet, 1, 0) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to allocate compute descriptor set for pattern generation"; + return; + } + + nri::UpdateDescriptorRangeDesc updateDescriptorRangeDescs[] = { + {m_GenerateDescriptorSet, 0, 0, &m_UploadBufferView, 1}, + {m_GenerateDescriptorSet, 1, 0, &m_SourcePreviewStorage, 1}, + {m_GenerateDescriptorSet, 2, 0, &m_DecodePreviewStorage, 1}, + }; + NRI.UpdateDescriptorRanges(updateDescriptorRangeDescs, helper::GetCountOf(updateDescriptorRangeDescs)); + } + } + + nri::BufferDesc bitstreamHeaderUploadBufferDesc = {}; + bitstreamHeaderUploadBufferDesc.size = ENCODED_SLICE_OFFSET; + + nri::BufferDesc bitstreamBufferDesc = {}; + bitstreamBufferDesc.size = BITSTREAM_SIZE; + bitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE | nri::BufferUsageBits::VIDEO_DECODE; + + nri::BufferDesc decodeBitstreamBufferDesc = {}; + decodeBitstreamBufferDesc.size = BITSTREAM_SIZE; + decodeBitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_DECODE; + + nri::BufferDesc metadataBufferDesc = {}; + metadataBufferDesc.size = METADATA_SIZE; + metadataBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; + + nri::BufferDesc resolvedMetadataBufferDesc = {}; + resolvedMetadataBufferDesc.size = RESOLVED_METADATA_SIZE; + resolvedMetadataBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; + + nri::BufferDesc resolvedMetadataReadbackBufferDesc = {}; + resolvedMetadataReadbackBufferDesc.size = RESOLVED_METADATA_SIZE; + resolvedMetadataReadbackBufferDesc.usage = nri::BufferUsageBits::NONE; + + nri::BufferDesc bitstreamHeaderReadbackBufferDesc = {}; + bitstreamHeaderReadbackBufferDesc.size = AV1_HEADER_READBACK_SIZE; + bitstreamHeaderReadbackBufferDesc.usage = nri::BufferUsageBits::NONE; + + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_UPLOAD, 0.0f, bitstreamHeaderUploadBufferDesc, m_BitstreamHeaderUploadBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create bitstream header upload buffer"; + return; + } + + if (Video.CreateCommittedVideoBitstreamBuffer(*m_Device, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create encode bitstream buffer"; + return; + } + + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_READBACK, 0.0f, bitstreamHeaderReadbackBufferDesc, m_BitstreamHeaderReadbackBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create bitstream header readback buffer"; + return; + } + + if (Video.CreateCommittedVideoBitstreamBuffer(*m_Device, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create decode bitstream buffer"; + return; + } + + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, metadataBufferDesc, m_MetadataBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create encode metadata buffer"; + return; + } + + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, resolvedMetadataBufferDesc, m_ResolvedMetadataBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create resolved encode metadata buffer"; + return; + } + + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_READBACK, 0.0f, resolvedMetadataReadbackBufferDesc, m_ResolvedMetadataReadbackBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create resolved encode metadata readback buffer"; + return; + } + + if (NRI.CreateFence(*m_Device, 0, m_MetadataReadbackFence) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create metadata readback fence"; + return; + } + + if (NRI.CreateCommandAllocator(*m_GraphicsQueue, m_MetadataReadbackCommandAllocator) != nri::Result::SUCCESS || + NRI.CreateCommandBuffer(*m_MetadataReadbackCommandAllocator, m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create metadata readback command buffer"; + return; + } + + nri::VideoPictureDesc encodePictureDesc = {}; + encodePictureDesc.texture = m_EncodeTexture; + encodePictureDesc.usage = nri::VideoPictureUsage::ENCODE_INPUT; + encodePictureDesc.format = nri::Format::NV12_UNORM; + encodePictureDesc.width = VIDEO_WIDTH; + encodePictureDesc.height = VIDEO_HEIGHT; + + nri::VideoPictureDesc decodePictureDesc = {}; + decodePictureDesc.texture = m_DecodeTexture; + decodePictureDesc.usage = nri::VideoPictureUsage::DECODE_OUTPUT; + decodePictureDesc.format = nri::Format::NV12_UNORM; + decodePictureDesc.width = VIDEO_WIDTH; + decodePictureDesc.height = VIDEO_HEIGHT; + + nri::VideoPictureDesc reconstructedPictureDesc = encodePictureDesc; + reconstructedPictureDesc.texture = m_ReconstructedTexture; + reconstructedPictureDesc.usage = nri::VideoPictureUsage::ENCODE_REFERENCE; + + if (Video.CreateVideoPicture(*m_Device, encodePictureDesc, m_EncodePicture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create encode picture"; + return; + } + + if (Video.CreateVideoPicture(*m_Device, reconstructedPictureDesc, m_ReconstructedPicture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create reconstructed picture"; + return; + } + + if (Video.CreateVideoPicture(*m_Device, decodePictureDesc, m_DecodePicture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create decode picture"; + return; + } + + m_VideoReady = true; + m_VideoStatus = std::string("NRI video queues and ") + GetCodecName(m_Codec) + " encode/decode objects initialized"; +} + +bool Sample::WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders) { + if (m_Codec == SampleCodec::AV1) { + void* headerPtr = NRI.MapBuffer(*m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); + if (!headerPtr) { + m_VideoStatus = "Failed to map bitstream header upload buffer"; + return false; + } + std::memset(headerPtr, 0, (size_t)ENCODED_SLICE_OFFSET); + NRI.UnmapBuffer(*m_BitstreamHeaderUploadBuffer); + annexBHeaders.clear(); + return true; + } + + nri::VideoAnnexBParameterSetsDesc annexBDesc = {}; + annexBDesc.codec = GetNriCodec(m_Codec); + annexBDesc.h264Sps = &m_H264Sps; + annexBDesc.h264Pps = &m_H264Pps; + annexBDesc.h265Vps = &m_H265Vps; + annexBDesc.h265Sps = &m_H265Sps; + annexBDesc.h265Pps = &m_H265Pps; + + if (Video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS || annexBDesc.writtenSize == 0 || annexBDesc.writtenSize >= ENCODED_SLICE_OFFSET) { + m_VideoStatus = std::string("Failed to query ") + GetCodecName(m_Codec) + " Annex-B parameter-set size"; + return false; + } + + annexBHeaders.resize((size_t)annexBDesc.writtenSize); + annexBDesc.dst = annexBHeaders.data(); + annexBDesc.dstSize = annexBHeaders.size(); + if (Video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS) { + m_VideoStatus = std::string("Failed to build ") + GetCodecName(m_Codec) + " Annex-B parameter sets"; + return false; + } + + void* headerPtr = NRI.MapBuffer(*m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); + if (!headerPtr) { + m_VideoStatus = "Failed to map bitstream header upload buffer"; + return false; + } + std::memset(headerPtr, 0, (size_t)ENCODED_SLICE_OFFSET); + std::memcpy(headerPtr, annexBHeaders.data(), annexBHeaders.size()); + NRI.UnmapBuffer(*m_BitstreamHeaderUploadBuffer); + return true; +} + +bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { + if (!CanRunRoundTrip()) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " round trip is not currently supported in this configuration"; + return false; + } + + PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); + if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { + m_VideoStatus = "Failed to generate NV12 source pattern via compute"; + return false; + } + + if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture)) { + m_VideoStatus = "Failed to upload NV12 source to video texture"; + return false; + } + + std::vector annexBHeaders; + if (!WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) + return false; + + if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + NRI.CmdZeroBuffer(commandBuffer, *m_BitstreamBuffer, 0, BITSTREAM_SIZE); + NRI.CmdCopyBuffer(commandBuffer, *m_BitstreamBuffer, 0, *m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); + })) { + m_VideoStatus = std::string("Failed to upload ") + GetCodecName(m_Codec) + " Annex-B parameter sets"; + return false; + } + + nri::VideoEncodePictureDesc pictureDesc = {}; + pictureDesc.frameType = nri::VideoEncodeFrameType::IDR; + pictureDesc.idrPictureId = 1; + + uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((VIDEO_WIDTH + 7) >> 3))}; + uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((VIDEO_HEIGHT + 7) >> 3))}; + uint16_t av1WidthInSuperblocksMinus1[] = {(uint16_t)(((VIDEO_WIDTH + 63) / 64) - 1)}; + uint16_t av1HeightInSuperblocksMinus1[] = {(uint16_t)(((VIDEO_HEIGHT + 63) / 64) - 1)}; + nri::VideoAV1TileLayoutDesc av1TileLayout = {}; + av1TileLayout.columnNum = 1; + av1TileLayout.rowNum = 1; + av1TileLayout.tileSizeBytesMinus1 = 3; + av1TileLayout.uniformSpacing = 1; + av1TileLayout.miColumnStarts = av1MiColumnStarts; + av1TileLayout.miRowStarts = av1MiRowStarts; + av1TileLayout.widthInSuperblocksMinus1 = av1WidthInSuperblocksMinus1; + av1TileLayout.heightInSuperblocksMinus1 = av1HeightInSuperblocksMinus1; + nri::VideoAV1LoopFilterDesc av1LoopFilter = {}; + av1LoopFilter.refDeltas[0] = 1; + av1LoopFilter.refDeltas[4] = -1; + av1LoopFilter.refDeltas[6] = -1; + av1LoopFilter.refDeltas[7] = -1; + nri::VideoAV1CdefDesc av1Cdef = {}; + nri::VideoAV1LoopRestorationDesc av1LoopRestoration = {}; + nri::VideoAV1GlobalMotionDesc av1GlobalMotion = {}; + for (auto& params : av1GlobalMotion.params) { + params[2] = 1 << 16; + params[5] = 1 << 16; + } + nri::VideoAV1PictureDesc av1PictureDesc = {}; + av1PictureDesc.currentFrameId = 0; + av1PictureDesc.refreshFrameFlags = 0xFF; + av1PictureDesc.primaryReferenceName = nri::VideoAV1ReferenceName::NONE; + av1PictureDesc.flags = nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE | + nri::VideoAV1PictureBits::DISABLE_CDF_UPDATE | + nri::VideoAV1PictureBits::ALLOW_SCREEN_CONTENT_TOOLS | + nri::VideoAV1PictureBits::FORCE_INTEGER_MV | + nri::VideoAV1PictureBits::SHOW_FRAME | + nri::VideoAV1PictureBits::SHOWABLE_FRAME; + av1PictureDesc.renderWidthMinus1 = VIDEO_WIDTH - 1; + av1PictureDesc.renderHeightMinus1 = VIDEO_HEIGHT - 1; + av1PictureDesc.baseQIndex = 20; + av1PictureDesc.interpolationFilter = 0; + av1PictureDesc.txMode = 2; + av1PictureDesc.cdefDampingMinus3 = 3; + av1PictureDesc.tileLayout = &av1TileLayout; + av1PictureDesc.loopFilter = &av1LoopFilter; + av1PictureDesc.cdef = &av1Cdef; + av1PictureDesc.loopRestoration = &av1LoopRestoration; + av1PictureDesc.globalMotion = &av1GlobalMotion; + + nri::VideoEncodeRateControlDesc rateControlDesc = {}; + rateControlDesc.mode = nri::VideoEncodeRateControlMode::CQP; + rateControlDesc.qpI = 20; + rateControlDesc.qpP = 22; + rateControlDesc.qpB = 24; + rateControlDesc.frameRateNumerator = 30; + rateControlDesc.frameRateDenominator = 1; + + nri::VideoEncodeDesc encodeDesc = {}; + encodeDesc.session = m_EncodeSession; + encodeDesc.parameters = m_EncodeParameters; + encodeDesc.srcPicture = m_EncodePicture; + encodeDesc.dstBitstream.buffer = m_BitstreamBuffer; + encodeDesc.dstBitstream.offset = ENCODED_SLICE_OFFSET; + encodeDesc.dstBitstream.size = BITSTREAM_SIZE - ENCODED_SLICE_OFFSET; + encodeDesc.bitstreamMetadataSize = ENCODED_SLICE_OFFSET; + encodeDesc.pictureDesc = &pictureDesc; + encodeDesc.rateControlDesc = &rateControlDesc; + encodeDesc.reconstructedPicture = m_ReconstructedPicture; + encodeDesc.metadata = m_MetadataBuffer; + encodeDesc.resolvedMetadata = m_ResolvedMetadataBuffer; + encodeDesc.av1PictureDesc = m_Codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; + + if (!SubmitOneTime(NRI, *m_VideoEncodeQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::BufferBarrierDesc bufferBarriers[3] = {}; + bufferBarriers[0].buffer = m_BitstreamBuffer; + bufferBarriers[0].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + bufferBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[1].buffer = m_MetadataBuffer; + bufferBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[2].buffer = m_ResolvedMetadataBuffer; + bufferBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + + nri::TextureBarrierDesc textureBarriers[2] = {}; + textureBarriers[0].texture = m_EncodeTexture; + textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[0].mipNum = nri::REMAINING; + textureBarriers[0].layerNum = nri::REMAINING; + textureBarriers[0].planes = nri::PlaneBits::ALL; + textureBarriers[1].texture = m_ReconstructedTexture; + textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[1].mipNum = nri::REMAINING; + textureBarriers[1].layerNum = nri::REMAINING; + textureBarriers[1].planes = nri::PlaneBits::ALL; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = bufferBarriers; + barrierDesc.bufferNum = helper::GetCountOf(bufferBarriers); + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + NRI.CmdBarrier(commandBuffer, barrierDesc); + Video.CmdEncodeVideo(commandBuffer, encodeDesc); + bufferBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[0].after = {}; + // D3D12 resolves encode metadata inside CmdEncodeVideo and transitions the raw metadata buffer to encode-read before returning. + bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[1].after = {}; + bufferBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[2].after = {}; + textureBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + NRI.CmdBarrier(commandBuffer, barrierDesc); + + })) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submission failed"; + return false; + } + + if (m_MetadataReadbackPending) + return true; + + NRI.ResetCommandAllocator(*m_MetadataReadbackCommandAllocator); + if (NRI.BeginCommandBuffer(*m_MetadataReadbackCommandBuffer, nullptr) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to begin metadata readback command buffer"; + return false; + } + + nri::BufferBarrierDesc metadataBarriers[4] = {}; + metadataBarriers[0].buffer = m_ResolvedMetadataBuffer; + metadataBarriers[0].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + metadataBarriers[0].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + metadataBarriers[1].buffer = m_ResolvedMetadataReadbackBuffer; + metadataBarriers[1].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + metadataBarriers[2].buffer = m_BitstreamBuffer; + metadataBarriers[2].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + metadataBarriers[2].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + metadataBarriers[3].buffer = m_BitstreamHeaderReadbackBuffer; + metadataBarriers[3].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + + nri::BarrierDesc metadataBarrierDesc = {}; + metadataBarrierDesc.buffers = metadataBarriers; + metadataBarrierDesc.bufferNum = helper::GetCountOf(metadataBarriers); + NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); + Video.CmdResolveVideoEncodeFeedback(*m_MetadataReadbackCommandBuffer, *m_EncodeSession, *m_ResolvedMetadataBuffer, 0); + metadataBarriers[0].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + metadataBarriers[0].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + metadataBarrierDesc.bufferNum = 1; + NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); + metadataBarrierDesc.bufferNum = helper::GetCountOf(metadataBarriers); + NRI.CmdCopyBuffer(*m_MetadataReadbackCommandBuffer, *m_ResolvedMetadataReadbackBuffer, 0, *m_ResolvedMetadataBuffer, 0, RESOLVED_METADATA_SIZE); + NRI.CmdCopyBuffer(*m_MetadataReadbackCommandBuffer, *m_BitstreamHeaderReadbackBuffer, 0, *m_BitstreamBuffer, ENCODED_SLICE_OFFSET, AV1_HEADER_READBACK_SIZE); + metadataBarriers[0].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + metadataBarriers[0].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + metadataBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + metadataBarriers[1].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + metadataBarriers[2].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + metadataBarriers[2].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + metadataBarriers[3].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + metadataBarriers[3].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); + + if (NRI.EndCommandBuffer(*m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to end metadata readback command buffer"; + return false; + } + + m_MetadataReadbackFenceValue++; + nri::FenceSubmitDesc signalFence = {}; + signalFence.fence = m_MetadataReadbackFence; + signalFence.value = m_MetadataReadbackFenceValue; + + const nri::CommandBuffer* commandBuffers[] = {m_MetadataReadbackCommandBuffer}; + nri::QueueSubmitDesc submit = {}; + submit.commandBuffers = commandBuffers; + submit.commandBufferNum = helper::GetCountOf(commandBuffers); + submit.signalFences = &signalFence; + submit.signalFenceNum = 1; + if (NRI.QueueSubmit(*m_GraphicsQueue, submit) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to submit async metadata readback"; + return false; + } + + m_MetadataReadbackPending = true; + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submitted; waiting for async metadata readback"; + return true; +} + +bool Sample::TryDecodePendingMetadata(float timeSec) { + if (!m_MetadataReadbackPending) + return false; + + if (NRI.GetFenceValue(*m_MetadataReadbackFence) < m_MetadataReadbackFenceValue) + return false; + + m_MetadataReadbackPending = false; + + nri::VideoEncodeFeedback feedback = {}; + const nri::Result feedbackResult = Video.GetVideoEncodeFeedback(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, feedback); + if (feedbackResult != nri::Result::SUCCESS) { + if (feedbackResult == nri::Result::UNSUPPORTED && m_Codec == SampleCodec::AV1) { + feedback.encodedBitstreamOffset = 0; + feedback.encodedBitstreamWrittenBytes = AV1_HEADER_READBACK_SIZE; + feedback.writtenSubregionNum = 1; + } else { + if (feedbackResult == nri::Result::UNSUPPORTED) + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode metadata feedback is unsupported"; + else + m_VideoStatus = "Failed to read resolved encode metadata"; + return false; + } + } + + if (feedback.errorFlags || !feedback.encodedBitstreamWrittenBytes) { + char message[160] = {}; + std::snprintf(message, sizeof(message), "Encoder returned errorFlags=0x%llX bytes=%llu", + (unsigned long long)feedback.errorFlags, (unsigned long long)feedback.encodedBitstreamWrittenBytes); + m_VideoStatus = message; + return false; + } + + nri::VideoAV1EncodeDecodeInfo av1DecodeInfo = {}; + if (m_Codec == SampleCodec::AV1) { + const uint8_t* encodedHeader = (const uint8_t*)NRI.MapBuffer(*m_BitstreamHeaderReadbackBuffer, 0, AV1_HEADER_READBACK_SIZE); + if (!encodedHeader && feedbackResult == nri::Result::UNSUPPORTED) { + m_VideoStatus = "Failed to map AV1 encoded header readback"; + return false; + } + + nri::VideoAV1EncodeDecodeInfoDesc av1InfoDesc = {}; + av1InfoDesc.feedback = &feedback; + av1InfoDesc.sequence = &m_AV1Sequence; + av1InfoDesc.encodedPayloadHeader = encodedHeader; + av1InfoDesc.encodedPayloadHeaderSize = encodedHeader ? std::min(AV1_HEADER_READBACK_SIZE, feedback.encodedBitstreamWrittenBytes) : 0; + const nri::Result av1InfoResult = Video.GetVideoEncodeAV1DecodeInfo(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, av1InfoDesc, av1DecodeInfo); + if (encodedHeader) + NRI.UnmapBuffer(*m_BitstreamHeaderReadbackBuffer); + if (av1InfoResult != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to prepare AV1 decode metadata"; + return false; + } + feedback.encodedBitstreamWrittenBytes = av1DecodeInfo.bitstreamOffset + av1DecodeInfo.bitstreamSize; + } + + return DecodeEncodedBitstream(feedback, m_Codec == SampleCodec::AV1 ? &av1DecodeInfo : nullptr, timeSec); +} + +bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo, float timeSec) { + std::vector annexBHeaders; + if (!WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) + return false; + + const uint64_t encodedPayloadSkip = av1DecodeInfo ? av1DecodeInfo->bitstreamOffset : GetEncodedPayloadHeaderSkip(m_Codec, feedback.encodedBitstreamWrittenBytes); + const uint64_t encodedPayloadBytes = av1DecodeInfo ? av1DecodeInfo->bitstreamSize : feedback.encodedBitstreamWrittenBytes - encodedPayloadSkip; + const uint64_t decodeSliceOffset = annexBHeaders.size(); + const uint64_t decodeBitstreamSize = AlignUp(decodeSliceOffset + encodedPayloadBytes, 256); + const uint64_t encodedSourceOffset = ENCODED_SLICE_OFFSET + feedback.encodedBitstreamOffset + encodedPayloadSkip; + if (feedback.encodedBitstreamOffset > BITSTREAM_SIZE - ENCODED_SLICE_OFFSET || + encodedSourceOffset > BITSTREAM_SIZE || + encodedPayloadBytes > BITSTREAM_SIZE - encodedSourceOffset || + decodeBitstreamSize > BITSTREAM_SIZE) { + m_VideoStatus = std::string("Encoded ") + GetCodecName(m_Codec) + " bitstream exceeded decode buffer size"; + return false; + } + + if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + NRI.CmdZeroBuffer(commandBuffer, *m_DecodeBitstreamBuffer, 0, BITSTREAM_SIZE); + if (!annexBHeaders.empty()) + NRI.CmdCopyBuffer(commandBuffer, *m_DecodeBitstreamBuffer, 0, *m_BitstreamHeaderUploadBuffer, 0, annexBHeaders.size()); + + nri::BufferBarrierDesc bufferBarriers[2] = {}; + bufferBarriers[0].buffer = m_BitstreamBuffer; + bufferBarriers[0].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + bufferBarriers[0].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + bufferBarriers[1].buffer = m_DecodeBitstreamBuffer; + bufferBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + bufferBarriers[1].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = bufferBarriers; + barrierDesc.bufferNum = helper::GetCountOf(bufferBarriers); + NRI.CmdBarrier(commandBuffer, barrierDesc); + + NRI.CmdCopyBuffer(commandBuffer, *m_DecodeBitstreamBuffer, decodeSliceOffset, *m_BitstreamBuffer, encodedSourceOffset, encodedPayloadBytes); + + bufferBarriers[0].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + bufferBarriers[0].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + bufferBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + bufferBarriers[1].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + NRI.CmdBarrier(commandBuffer, barrierDesc); + })) { + m_VideoStatus = std::string("Failed to build exact ") + GetCodecName(m_Codec) + " decode bitstream"; + return false; + } + + const uint32_t pictureOffsets[] = {(uint32_t)decodeSliceOffset}; + + nri::VideoH264DecodePictureDesc h264DecodePicture = {}; + h264DecodePicture.flags = nri::VideoH264DecodePictureBits::IDR | nri::VideoH264DecodePictureBits::INTRA | nri::VideoH264DecodePictureBits::REFERENCE; + h264DecodePicture.pictureParameterSetId = m_H264Pps.pictureParameterSetId; + h264DecodePicture.frameNum = 0; + h264DecodePicture.idrPictureId = 1; + h264DecodePicture.topFieldOrderCount = 0; + h264DecodePicture.bottomFieldOrderCount = 0; + h264DecodePicture.sliceOffsets = pictureOffsets; + h264DecodePicture.sliceOffsetNum = helper::GetCountOf(pictureOffsets); + + nri::VideoH265DecodePictureDesc h265DecodePicture = {}; + h265DecodePicture.flags = nri::VideoH265DecodePictureBits::IRAP | nri::VideoH265DecodePictureBits::IDR | nri::VideoH265DecodePictureBits::REFERENCE; + h265DecodePicture.videoParameterSetId = m_H265Vps.videoParameterSetId; + h265DecodePicture.sequenceParameterSetId = m_H265Sps.sequenceParameterSetId; + h265DecodePicture.pictureParameterSetId = m_H265Pps.pictureParameterSetId; + h265DecodePicture.pictureOrderCount = 0; + h265DecodePicture.sliceSegmentOffsets = pictureOffsets; + h265DecodePicture.sliceSegmentOffsetNum = helper::GetCountOf(pictureOffsets); + + nri::VideoAV1EncodeDecodeInfo av1Info = {}; + if (av1DecodeInfo) { + av1Info = *av1DecodeInfo; + av1Info.picture.tiles = av1Info.tiles; + av1Info.picture.tileLayout = &av1Info.tileLayout; + av1Info.picture.quantization = &av1Info.quantization; + av1Info.picture.loopFilter = &av1Info.loopFilter; + av1Info.picture.cdef = &av1Info.cdef; + av1Info.picture.segmentation = av1DecodeInfo->picture.segmentation ? &av1Info.segmentation : nullptr; + av1Info.picture.loopRestoration = &av1Info.loopRestoration; + av1Info.picture.globalMotion = &av1Info.globalMotion; + av1Info.tileLayout.miColumnStarts = av1Info.miColumnStarts; + av1Info.tileLayout.miRowStarts = av1Info.miRowStarts; + av1Info.tileLayout.widthInSuperblocksMinus1 = av1Info.widthInSuperblocksMinus1; + av1Info.tileLayout.heightInSuperblocksMinus1 = av1Info.heightInSuperblocksMinus1; + } + + nri::VideoDecodeDesc decodeDesc = {}; + decodeDesc.session = m_DecodeSession; + decodeDesc.parameters = m_DecodeParameters; + decodeDesc.bitstream.buffer = m_DecodeBitstreamBuffer; + decodeDesc.bitstream.size = decodeBitstreamSize; + decodeDesc.dstPicture = m_DecodePicture; + decodeDesc.dstSlot = 0; + decodeDesc.h264PictureDesc = m_Codec == SampleCodec::H264 ? &h264DecodePicture : nullptr; + decodeDesc.h265PictureDesc = m_Codec == SampleCodec::H265 ? &h265DecodePicture : nullptr; + decodeDesc.av1PictureDesc = av1DecodeInfo ? &av1Info.picture : nullptr; + + nri::VideoDecodePictureStates decodePictureStates = {}; + if (Video.GetVideoDecodePictureStates(*m_DecodePicture, decodePictureStates) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to query video decode picture states"; + return false; + } + + if (!SubmitOneTime(NRI, *m_VideoDecodeQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::TextureBarrierDesc textureBarrier = {}; + textureBarrier.texture = m_DecodeTexture; + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarrier.after = decodePictureStates.decodeWrite; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textures = &textureBarrier; + barrierDesc.textureNum = 1; + NRI.CmdBarrier(commandBuffer, barrierDesc); + Video.CmdDecodeVideo(commandBuffer, decodeDesc); + + if (decodePictureStates.releaseAfterDecode) { + textureBarrier.before = decodePictureStates.decodeWrite; + textureBarrier.after = decodePictureStates.afterDecode; + NRI.CmdBarrier(commandBuffer, barrierDesc); + } + })) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " decode submission failed"; + return false; + } + + if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::TextureBarrierDesc textureBarrier = {}; + textureBarrier.texture = m_DecodeTexture; + textureBarrier.before = decodePictureStates.graphicsBefore; + textureBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textures = &textureBarrier; + barrierDesc.textureNum = 1; + NRI.CmdBarrier(commandBuffer, barrierDesc); + + nri::BufferBarrierDesc nv12BufferBarrier = {}; + nv12BufferBarrier.buffer = m_UploadBuffer; + nv12BufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + nv12BufferBarrier.after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + + nri::BarrierDesc copyBarrierDesc = {}; + copyBarrierDesc.buffers = &nv12BufferBarrier; + copyBarrierDesc.bufferNum = 1; + NRI.CmdBarrier(commandBuffer, copyBarrierDesc); + + nri::TextureRegionDesc lumaRegion = {}; + lumaRegion.width = VIDEO_WIDTH; + lumaRegion.height = VIDEO_HEIGHT; + lumaRegion.depth = 1; + lumaRegion.planes = nri::PlaneBits::PLANE_0; + + nri::TextureDataLayoutDesc lumaLayout = {}; + lumaLayout.rowPitch = m_Nv12Layout.yRowPitchBytes; + lumaLayout.slicePitch = m_Nv12Layout.ySlicePitchBytes; + NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, lumaLayout, *m_DecodeTexture, lumaRegion); + + nri::TextureRegionDesc chromaRegion = {}; + chromaRegion.width = VIDEO_WIDTH; + chromaRegion.height = VIDEO_HEIGHT; + chromaRegion.depth = 1; + chromaRegion.planes = nri::PlaneBits::PLANE_1; + + nri::TextureDataLayoutDesc chromaLayout = {}; + chromaLayout.offset = m_Nv12Layout.uvOffsetBytes; + chromaLayout.rowPitch = m_Nv12Layout.uvRowPitchBytes; + chromaLayout.slicePitch = m_Nv12Layout.uvSlicePitchBytes; + NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, chromaLayout, *m_DecodeTexture, chromaRegion); + + nv12BufferBarrier.before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + nv12BufferBarrier.after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + NRI.CmdBarrier(commandBuffer, copyBarrierDesc); + + textureBarrier.before = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; + textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + NRI.CmdBarrier(commandBuffer, barrierDesc); + })) { + m_VideoStatus = "Failed to copy decoded NV12 for preview"; + return false; + } + + PatternConstants patternConstants = MakePatternConstants(OP_NV12_TO_PREVIEW, timeSec); + if (!GeneratePatternWithCompute(patternConstants, m_DecodePreviewStorage, true)) { + m_VideoStatus = "Failed to convert decoded NV12 to preview texture"; + return false; + } + + m_DecodePreviewReady = true; + char message[128] = {}; + std::snprintf(message, sizeof(message), "%s encode/decode round trip complete, encoded %llu bytes", GetCodecName(m_Codec), (unsigned long long)feedback.encodedBitstreamWrittenBytes); + m_VideoStatus = message; + return true; +} + +bool Sample::TryRunRoundTrip(float timeSec) { + if (m_MetadataReadbackPending) + return TryDecodePendingMetadata(timeSec); + + return TrySubmitEncodeAndMetadataReadback(timeSec); +} + +bool Sample::CanRunRoundTrip() const { + return m_VideoReady && m_GraphicsQueue && m_VideoEncodeQueue && m_VideoDecodeQueue && m_UploadBuffer && m_UploadBufferView && m_SourcePreviewStorage && + m_DecodePreviewStorage && m_GeneratePipelineLayout && m_GenerateComputePipeline && m_GenerateDescriptorSet; +} + +void Sample::LatencySleep(uint32_t frameIndex) { + uint32_t queuedFrameIndex = frameIndex % GetQueuedFrameNum(); + const QueuedFrame& queuedFrame = m_QueuedFrames[queuedFrameIndex]; + + NRI.Wait(*m_FrameFence, frameIndex >= GetQueuedFrameNum() ? 1 + frameIndex - GetQueuedFrameNum() : 0); + NRI.ResetCommandAllocator(*queuedFrame.commandAllocator); +} + +void Sample::DrawTexturePanel(const char* label, nri::Descriptor* texture, const ImVec2& size) { + ImGui::Text("%s", label); + if (!texture) { + ImGui::Text("Not ready"); + return; + } + ImGui::Image((ImTextureID)texture, size); +} + +void Sample::PrepareFrame(uint32_t frameIndex) { + (void)frameIndex; + + const double timeSec = m_Timer.GetTimeStamp() * 0.001 - m_StartTimeSec; + const bool canRunRoundTrip = CanRunRoundTrip(); + + InitializeGeneratedFrames((float)timeSec); + + if (canRunRoundTrip && timeSec - m_LastRoundTripTimeSec >= ROUND_TRIP_INTERVAL_SEC) { + if (TryRunRoundTrip((float)timeSec)) + m_LastRoundTripTimeSec = timeSec; + } + + ImGui::NewFrame(); + { + ImGui::SetNextWindowPos({20.0f, 20.0f}, ImGuiCond_Once); + ImGui::SetNextWindowSize({900.0f, 520.0f}, ImGuiCond_Once); + ImGui::Begin("NRI Video Encode / Decode"); + { + ImGui::Text("Codec: %s, format: NV12, size: %ux%u", GetCodecName(m_Codec), VIDEO_WIDTH, VIDEO_HEIGHT); + ImGui::TextWrapped("Video: %s", m_VideoStatus.c_str()); + ImGui::TextWrapped("Preview: %s", m_PreviewStatus.c_str()); + ImGui::Text("Encode queue: %s, decode queue: %s", m_VideoEncodeQueue ? "yes" : "no", m_VideoDecodeQueue ? "yes" : "no"); + const bool roundTripSupported = CanRunRoundTrip(); + ImGui::Text("Round trip: %s", roundTripSupported ? "running" : "backend must support NRI video encode/decode"); + if (m_VideoReady && !m_DecodePreviewReady) + ImGui::Text("Decode preview: waiting for first decoded frame"); + + ImGui::Separator(); + if (ImGui::BeginTable("PreviewPanels", 2, ImGuiTableFlags_SizingStretchSame)) { + ImGui::TableNextColumn(); + float width = std::max(200.0f, ImGui::GetContentRegionAvail().x); + DrawTexturePanel("Generated source", m_SourcePreviewTextureView, {width, width * float(VIDEO_HEIGHT) / float(VIDEO_WIDTH)}); + + ImGui::TableNextColumn(); + width = std::max(200.0f, ImGui::GetContentRegionAvail().x); + DrawTexturePanel(m_DecodePreviewReady ? "Decoded preview" : "Decoded preview pending", m_DecodePreviewReady ? m_DecodePreviewTextureView : nullptr, {width, width * float(VIDEO_HEIGHT) / float(VIDEO_WIDTH)}); + ImGui::EndTable(); + } + } + ImGui::End(); + } + ImGui::EndFrame(); + ImGui::Render(); +} + +void Sample::RenderFrame(uint32_t frameIndex) { + uint32_t queuedFrameIndex = frameIndex % GetQueuedFrameNum(); + const QueuedFrame& queuedFrame = m_QueuedFrames[queuedFrameIndex]; + + uint32_t recycledSemaphoreIndex = frameIndex % (uint32_t)m_SwapChainTextures.size(); + nri::Fence* swapChainAcquireSemaphore = m_SwapChainTextures[recycledSemaphoreIndex].acquireSemaphore; + + uint32_t currentSwapChainTextureIndex = 0; + NRI.AcquireNextTexture(*m_SwapChain, *swapChainAcquireSemaphore, currentSwapChainTextureIndex); + + const SwapChainTexture& swapChainTexture = m_SwapChainTextures[currentSwapChainTextureIndex]; + + nri::CommandBuffer& commandBuffer = *queuedFrame.commandBuffer; + NRI.BeginCommandBuffer(commandBuffer, nullptr); + { + nri::TextureBarrierDesc textureBarriers = {}; + textureBarriers.texture = swapChainTexture.texture; + textureBarriers.after = {nri::AccessBits::COLOR_ATTACHMENT, nri::Layout::COLOR_ATTACHMENT}; + textureBarriers.layerNum = 1; + textureBarriers.mipNum = 1; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textureNum = 1; + barrierDesc.textures = &textureBarriers; + NRI.CmdBarrier(commandBuffer, barrierDesc); + + nri::AttachmentDesc colorAttachmentDesc = {}; + colorAttachmentDesc.descriptor = swapChainTexture.colorAttachment; + colorAttachmentDesc.clearValue.color.f = {0.03f, 0.03f, 0.03f, 1.0f}; + + nri::RenderingDesc renderingDesc = {}; + renderingDesc.colorNum = 1; + renderingDesc.colors = &colorAttachmentDesc; + + CmdCopyImguiData(commandBuffer, *m_Streamer); + + NRI.CmdBeginRendering(commandBuffer, renderingDesc); + CmdDrawImgui(commandBuffer, swapChainTexture.attachmentFormat, 1.0f, true); + NRI.CmdEndRendering(commandBuffer); + + textureBarriers.before = textureBarriers.after; + textureBarriers.after = {nri::AccessBits::NONE, nri::Layout::PRESENT, nri::StageBits::NONE}; + NRI.CmdBarrier(commandBuffer, barrierDesc); + } + NRI.EndCommandBuffer(commandBuffer); + + nri::FenceSubmitDesc textureAcquiredFence = {}; + textureAcquiredFence.fence = swapChainAcquireSemaphore; + textureAcquiredFence.stages = nri::StageBits::COLOR_ATTACHMENT; + + nri::FenceSubmitDesc renderingFinishedFence = {}; + renderingFinishedFence.fence = swapChainTexture.releaseSemaphore; + + nri::QueueSubmitDesc queueSubmitDesc = {}; + queueSubmitDesc.waitFences = &textureAcquiredFence; + queueSubmitDesc.waitFenceNum = 1; + queueSubmitDesc.commandBuffers = &queuedFrame.commandBuffer; + queueSubmitDesc.commandBufferNum = 1; + queueSubmitDesc.signalFences = &renderingFinishedFence; + queueSubmitDesc.signalFenceNum = 1; + NRI.QueueSubmit(*m_GraphicsQueue, queueSubmitDesc); + + NRI.EndStreamerFrame(*m_Streamer); + NRI.QueuePresent(*m_SwapChain, *swapChainTexture.releaseSemaphore); + + nri::FenceSubmitDesc signalFence = {}; + signalFence.fence = m_FrameFence; + signalFence.value = 1 + frameIndex; + + nri::QueueSubmitDesc signalSubmitDesc = {}; + signalSubmitDesc.signalFences = &signalFence; + signalSubmitDesc.signalFenceNum = 1; + NRI.QueueSubmit(*m_GraphicsQueue, signalSubmitDesc); +} + +SAMPLE_MAIN(Sample, 0); From c6c5ecee899a27a04b86fefbc2f199fe1a140d18 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Wed, 6 May 2026 00:28:26 +0200 Subject: [PATCH 02/28] Update video sample dependencies Point the NRIFramework submodule at the branch that uses NRI video-queues, remove the local NRI override documentation, and apply formatting cleanup to the video sample. --- .gitignore | 2 - .gitmodules | 4 +- README.md | 1 - Source/VideoEncodeDecode.cpp | 72 +++++++++++++----------------------- 4 files changed, 28 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index c10db7b..e251e72 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,3 @@ pso_cache.bin # can be a symbolic link _Data -build -build-video-sample diff --git a/.gitmodules b/.gitmodules index b919ec0..797781a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,5 @@ [submodule "External/NRIFramework"] path = External/NRIFramework - url = https://github.com/NVIDIA-RTX/NRIFramework.git - branch = main + url = https://github.com/Daedie-git/NRIFramework.git + branch = codex/video-encode-decode-sample update = merge diff --git a/README.md b/README.md index 070b8e8..01fab10 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ Or by running scripts only: ### CMake options - `DISABLE_SHADER_COMPILATION` - disable compilation of shaders (shaders can be built on other platform) -- `NRI_SOURCE_OVERRIDE_DIR` - use an alternate NRI source tree, for example an in-review NRI branch - `NRIF_USE_WAYLAND` - use Wayland instead of X11 on Linux ## How to run diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index a2bc461..ed65962 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -43,25 +43,25 @@ enum class SampleCodec : uint8_t { static const char* GetCodecName(SampleCodec codec) { switch (codec) { - case SampleCodec::H265: - return "H.265"; - case SampleCodec::AV1: - return "AV1"; - case SampleCodec::H264: - default: - return "H.264"; + case SampleCodec::H265: + return "H.265"; + case SampleCodec::AV1: + return "AV1"; + case SampleCodec::H264: + default: + return "H.264"; } } static nri::VideoCodec GetNriCodec(SampleCodec codec) { switch (codec) { - case SampleCodec::H265: - return nri::VideoCodec::H265; - case SampleCodec::AV1: - return nri::VideoCodec::AV1; - case SampleCodec::H264: - default: - return nri::VideoCodec::H264; + case SampleCodec::H265: + return nri::VideoCodec::H265; + case SampleCodec::AV1: + return nri::VideoCodec::AV1; + case SampleCodec::H264: + default: + return nri::VideoCodec::H264; } } @@ -72,10 +72,7 @@ static uint64_t GetEncodedPayloadHeaderSkip(SampleCodec codec, uint64_t encodedB static nri::VideoAV1SequenceDesc MakeAV1SequenceDesc() { nri::VideoAV1SequenceDesc desc = {}; - desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | - nri::VideoAV1SequenceBits::ENABLE_CDEF | - nri::VideoAV1SequenceBits::ENABLE_RESTORATION | - nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; + desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | nri::VideoAV1SequenceBits::ENABLE_CDEF | nri::VideoAV1SequenceBits::ENABLE_RESTORATION | nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; desc.bitDepth = 8; desc.subsamplingX = 1; desc.subsamplingY = 1; @@ -166,8 +163,7 @@ static bool SubmitOneTime( nri::CoreInterface& core, nri::Queue& queue, nri::DescriptorPool* descriptorPool, Record&& record) { nri::CommandAllocator* allocator = nullptr; nri::CommandBuffer* commandBuffer = nullptr; - bool ok = core.CreateCommandAllocator(queue, allocator) == nri::Result::SUCCESS && allocator && core.CreateCommandBuffer(*allocator, commandBuffer) == nri::Result::SUCCESS && commandBuffer && - core.BeginCommandBuffer(*commandBuffer, descriptorPool) == nri::Result::SUCCESS; + bool ok = core.CreateCommandAllocator(queue, allocator) == nri::Result::SUCCESS && allocator && core.CreateCommandBuffer(*allocator, commandBuffer) == nri::Result::SUCCESS && commandBuffer && core.BeginCommandBuffer(*commandBuffer, descriptorPool) == nri::Result::SUCCESS; if (ok) { std::forward(record)(*commandBuffer); ok = core.EndCommandBuffer(*commandBuffer) == nri::Result::SUCCESS; @@ -254,11 +250,11 @@ class Sample : public SampleBase { void RenderFrame(uint32_t frameIndex) override; private: - bool InitializeGraphics(nri::GraphicsAPI graphicsAPI); + bool InitializeGraphics(nri::GraphicsAPI graphicsAPI); bool TryInitializePreviewTextures(nri::GraphicsAPI graphicsAPI); - void InitializeGeneratedFrames(float timeSec); + void InitializeGeneratedFrames(float timeSec); bool CanRunRoundTrip() const; - void TryInitializeVideo(nri::GraphicsAPI graphicsAPI); + void TryInitializeVideo(nri::GraphicsAPI graphicsAPI); PatternConstants MakePatternConstants(PatternOperation operation, float timeSec) const; bool GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage = false); bool WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders); @@ -916,8 +912,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { vps.decPicBufMgr.maxNumReorderPics[0] = 1; nri::VideoH265SequenceParameterSetDesc h265Sps = {}; - h265Sps.flags = nri::VideoH265SequenceParameterSetBits::TEMPORAL_ID_NESTING | nri::VideoH265SequenceParameterSetBits::AMP_ENABLED | - nri::VideoH265SequenceParameterSetBits::SAMPLE_ADAPTIVE_OFFSET_ENABLED; + h265Sps.flags = nri::VideoH265SequenceParameterSetBits::TEMPORAL_ID_NESTING | nri::VideoH265SequenceParameterSetBits::AMP_ENABLED | nri::VideoH265SequenceParameterSetBits::SAMPLE_ADAPTIVE_OFFSET_ENABLED; h265Sps.videoParameterSetId = vps.videoParameterSetId; h265Sps.maxSubLayersMinus1 = vps.maxSubLayersMinus1; h265Sps.sequenceParameterSetId = 0; @@ -935,9 +930,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { h265Sps.decPicBufMgr = vps.decPicBufMgr; nri::VideoH265PictureParameterSetDesc h265Pps = {}; - h265Pps.flags = nri::VideoH265PictureParameterSetBits::CABAC_INIT_PRESENT | nri::VideoH265PictureParameterSetBits::TRANSFORM_SKIP_ENABLED | - nri::VideoH265PictureParameterSetBits::CU_QP_DELTA_ENABLED | nri::VideoH265PictureParameterSetBits::SLICE_CHROMA_QP_OFFSETS_PRESENT | - nri::VideoH265PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; + h265Pps.flags = nri::VideoH265PictureParameterSetBits::CABAC_INIT_PRESENT | nri::VideoH265PictureParameterSetBits::TRANSFORM_SKIP_ENABLED | nri::VideoH265PictureParameterSetBits::CU_QP_DELTA_ENABLED | nri::VideoH265PictureParameterSetBits::SLICE_CHROMA_QP_OFFSETS_PRESENT | nri::VideoH265PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; h265Pps.pictureParameterSetId = 0; h265Pps.sequenceParameterSetId = h265Sps.sequenceParameterSetId; h265Pps.videoParameterSetId = vps.videoParameterSetId; @@ -1205,8 +1198,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (NRI.CreateCommandAllocator(*m_GraphicsQueue, m_MetadataReadbackCommandAllocator) != nri::Result::SUCCESS || - NRI.CreateCommandBuffer(*m_MetadataReadbackCommandAllocator, m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { + if (NRI.CreateCommandAllocator(*m_GraphicsQueue, m_MetadataReadbackCommandAllocator) != nri::Result::SUCCESS || NRI.CreateCommandBuffer(*m_MetadataReadbackCommandAllocator, m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create metadata readback command buffer"; return; } @@ -1355,12 +1347,7 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { av1PictureDesc.currentFrameId = 0; av1PictureDesc.refreshFrameFlags = 0xFF; av1PictureDesc.primaryReferenceName = nri::VideoAV1ReferenceName::NONE; - av1PictureDesc.flags = nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE | - nri::VideoAV1PictureBits::DISABLE_CDF_UPDATE | - nri::VideoAV1PictureBits::ALLOW_SCREEN_CONTENT_TOOLS | - nri::VideoAV1PictureBits::FORCE_INTEGER_MV | - nri::VideoAV1PictureBits::SHOW_FRAME | - nri::VideoAV1PictureBits::SHOWABLE_FRAME; + av1PictureDesc.flags = nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE | nri::VideoAV1PictureBits::DISABLE_CDF_UPDATE | nri::VideoAV1PictureBits::ALLOW_SCREEN_CONTENT_TOOLS | nri::VideoAV1PictureBits::FORCE_INTEGER_MV | nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME; av1PictureDesc.renderWidthMinus1 = VIDEO_WIDTH - 1; av1PictureDesc.renderHeightMinus1 = VIDEO_HEIGHT - 1; av1PictureDesc.baseQIndex = 20; @@ -1441,7 +1428,6 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { barrierDesc.textures = textureBarriers; barrierDesc.textureNum = helper::GetCountOf(textureBarriers); NRI.CmdBarrier(commandBuffer, barrierDesc); - })) { m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submission failed"; return false; @@ -1585,10 +1571,7 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co const uint64_t decodeSliceOffset = annexBHeaders.size(); const uint64_t decodeBitstreamSize = AlignUp(decodeSliceOffset + encodedPayloadBytes, 256); const uint64_t encodedSourceOffset = ENCODED_SLICE_OFFSET + feedback.encodedBitstreamOffset + encodedPayloadSkip; - if (feedback.encodedBitstreamOffset > BITSTREAM_SIZE - ENCODED_SLICE_OFFSET || - encodedSourceOffset > BITSTREAM_SIZE || - encodedPayloadBytes > BITSTREAM_SIZE - encodedSourceOffset || - decodeBitstreamSize > BITSTREAM_SIZE) { + if (feedback.encodedBitstreamOffset > BITSTREAM_SIZE - ENCODED_SLICE_OFFSET || encodedSourceOffset > BITSTREAM_SIZE || encodedPayloadBytes > BITSTREAM_SIZE - encodedSourceOffset || decodeBitstreamSize > BITSTREAM_SIZE) { m_VideoStatus = std::string("Encoded ") + GetCodecName(m_Codec) + " bitstream exceeded decode buffer size"; return false; } @@ -1783,8 +1766,7 @@ bool Sample::TryRunRoundTrip(float timeSec) { } bool Sample::CanRunRoundTrip() const { - return m_VideoReady && m_GraphicsQueue && m_VideoEncodeQueue && m_VideoDecodeQueue && m_UploadBuffer && m_UploadBufferView && m_SourcePreviewStorage && - m_DecodePreviewStorage && m_GeneratePipelineLayout && m_GenerateComputePipeline && m_GenerateDescriptorSet; + return m_VideoReady && m_GraphicsQueue && m_VideoEncodeQueue && m_VideoDecodeQueue && m_UploadBuffer && m_UploadBufferView && m_SourcePreviewStorage && m_DecodePreviewStorage && m_GeneratePipelineLayout && m_GenerateComputePipeline && m_GenerateDescriptorSet; } void Sample::LatencySleep(uint32_t frameIndex) { @@ -1804,9 +1786,7 @@ void Sample::DrawTexturePanel(const char* label, nri::Descriptor* texture, const ImGui::Image((ImTextureID)texture, size); } -void Sample::PrepareFrame(uint32_t frameIndex) { - (void)frameIndex; - +void Sample::PrepareFrame(uint32_t) { const double timeSec = m_Timer.GetTimeStamp() * 0.001 - m_StartTimeSec; const bool canRunRoundTrip = CanRunRoundTrip(); From 85b03c2b5685ae132ba8dfc5e141e90fe3ad592b Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Wed, 6 May 2026 00:51:02 +0200 Subject: [PATCH 03/28] Use upstream NRIFramework submodule URL --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 797781a..b919ec0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,5 @@ [submodule "External/NRIFramework"] path = External/NRIFramework - url = https://github.com/Daedie-git/NRIFramework.git - branch = codex/video-encode-decode-sample + url = https://github.com/NVIDIA-RTX/NRIFramework.git + branch = main update = merge From cb6cf7a16e4679489554db29a94fd5013e195d25 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Wed, 6 May 2026 08:54:23 +0200 Subject: [PATCH 04/28] Use simplified NRI video API --- CMakeLists.txt | 19 +++---------------- Source/VideoEncodeDecode.cpp | 23 ++++++++--------------- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b62c798..c9300bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,23 +190,10 @@ add_sample(Resources c) add_sample(SceneViewer cpp) add_sample(Triangle cpp) -set(NRI_VIDEO_HEADER "${NRI_SOURCE_DIR}/Include/Extensions/NRIVideo.h") -set(NRI_VIDEO_VERSION 0) -if(EXISTS "${NRI_VIDEO_HEADER}") - file(STRINGS "${NRI_VIDEO_HEADER}" NRI_VIDEO_VERSION_LINE REGEX "^#define NRI_VIDEO_VERSION ") - if(NRI_VIDEO_VERSION_LINE MATCHES "^#define NRI_VIDEO_VERSION +([0-9]+)") - set(NRI_VIDEO_VERSION "${CMAKE_MATCH_1}") - endif() -endif() +add_sample(VideoEncodeDecode cpp) -if(NRI_VIDEO_VERSION GREATER_EQUAL 1) - add_sample(VideoEncodeDecode cpp) - - if(WIN32) - target_link_libraries(VideoEncodeDecode PRIVATE d3d12) - endif() -else() - message(STATUS "Skipping VideoEncodeDecode: selected NRI source does not provide NRIVideo version 1") +if(WIN32) + target_link_libraries(VideoEncodeDecode PRIVATE d3d12) endif() # Wrapper depends on Vulkan SDK availability diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index ed65962..6a9908b 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -856,8 +856,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { decodeSessionDesc.usage = nri::VideoUsage::DECODE; decodeSessionDesc.maxReferenceNum = 16; - if (Video.GetVideoQueue(*m_Device, encodeSessionDesc, m_VideoEncodeQueue) != nri::Result::SUCCESS || Video.GetVideoQueue(*m_Device, decodeSessionDesc, m_VideoDecodeQueue) != nri::Result::SUCCESS) { - m_VideoStatus = std::string("Failed to get ") + GetCodecName(m_Codec) + "-capable video queues"; + if (NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_ENCODE, 0, m_VideoEncodeQueue) != nri::Result::SUCCESS || NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_DECODE, 0, m_VideoDecodeQueue) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to get video queues"; return; } @@ -983,31 +983,24 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { encodeTextureDesc.height = VIDEO_HEIGHT; encodeTextureDesc.mipNum = 1; encodeTextureDesc.layerNum = 1; + encodeTextureDesc.videoCodec = GetNriCodec(m_Codec); nri::TextureDesc decodeTextureDesc = encodeTextureDesc; decodeTextureDesc.usage = nri::TextureUsageBits::VIDEO_DECODE; - nri::VideoTextureDesc encodeVideoTextureDesc = {}; - encodeVideoTextureDesc.textureDesc = encodeTextureDesc; - encodeVideoTextureDesc.codec = GetNriCodec(m_Codec); - - nri::VideoTextureDesc decodeVideoTextureDesc = {}; - decodeVideoTextureDesc.textureDesc = decodeTextureDesc; - decodeVideoTextureDesc.codec = GetNriCodec(m_Codec); - - if (Video.CreateCommittedVideoTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeVideoTextureDesc, m_EncodeTexture) != nri::Result::SUCCESS) { + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeTextureDesc, m_EncodeTexture) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create NV12 encode texture"; return; } NRI.SetDebugName(m_EncodeTexture, "VideoEncodeTexture"); - if (Video.CreateCommittedVideoTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeVideoTextureDesc, m_ReconstructedTexture) != nri::Result::SUCCESS) { + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeTextureDesc, m_ReconstructedTexture) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create NV12 reconstructed texture"; return; } NRI.SetDebugName(m_ReconstructedTexture, "VideoReconstructedTexture"); - if (Video.CreateCommittedVideoTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, decodeVideoTextureDesc, m_DecodeTexture) != nri::Result::SUCCESS) { + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, decodeTextureDesc, m_DecodeTexture) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create NV12 decode texture"; return; } @@ -1163,7 +1156,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (Video.CreateCommittedVideoBitstreamBuffer(*m_Device, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create encode bitstream buffer"; return; } @@ -1173,7 +1166,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (Video.CreateCommittedVideoBitstreamBuffer(*m_Device, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { + if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_UPLOAD, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create decode bitstream buffer"; return; } From cabd90d915a33dc4419b31175835aa697b5b6eb7 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Wed, 6 May 2026 09:47:05 +0200 Subject: [PATCH 05/28] Choose video bitstream memory per backend --- Source/VideoEncodeDecode.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 6a9908b..5956cdf 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -235,6 +235,16 @@ static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, }); } +static nri::Result CreateVideoBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, nri::GraphicsAPI graphicsAPI, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { + nri::MemoryLocation memoryLocation = nri::MemoryLocation::DEVICE; + if (bufferDesc.usage & nri::BufferUsageBits::VIDEO_ENCODE) + memoryLocation = graphicsAPI == nri::GraphicsAPI::VK ? nri::MemoryLocation::HOST_READBACK : nri::MemoryLocation::DEVICE; + else if (bufferDesc.usage & nri::BufferUsageBits::VIDEO_DECODE) + memoryLocation = nri::MemoryLocation::HOST_UPLOAD; + + return core.CreateCommittedBuffer(device, memoryLocation, priority, bufferDesc, buffer); +} + } // namespace class Sample : public SampleBase { @@ -1156,7 +1166,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { + if (CreateVideoBitstreamBuffer(NRI, *m_Device, graphicsAPI, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create encode bitstream buffer"; return; } @@ -1166,7 +1176,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_UPLOAD, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { + if (CreateVideoBitstreamBuffer(NRI, *m_Device, graphicsAPI, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create decode bitstream buffer"; return; } From e37bf1d1a02e03ac4203bea59d1252b0122715f7 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Wed, 6 May 2026 12:15:37 +0200 Subject: [PATCH 06/28] Use readback encode bitstream buffers --- Source/VideoEncodeDecode.cpp | 37 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 5956cdf..7a4bed7 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -235,14 +235,12 @@ static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, }); } -static nri::Result CreateVideoBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, nri::GraphicsAPI graphicsAPI, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { - nri::MemoryLocation memoryLocation = nri::MemoryLocation::DEVICE; - if (bufferDesc.usage & nri::BufferUsageBits::VIDEO_ENCODE) - memoryLocation = graphicsAPI == nri::GraphicsAPI::VK ? nri::MemoryLocation::HOST_READBACK : nri::MemoryLocation::DEVICE; - else if (bufferDesc.usage & nri::BufferUsageBits::VIDEO_DECODE) - memoryLocation = nri::MemoryLocation::HOST_UPLOAD; - - return core.CreateCommittedBuffer(device, memoryLocation, priority, bufferDesc, buffer); +static nri::Result CreateEncodeBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { + return core.CreateCommittedBuffer(device, nri::MemoryLocation::HOST_READBACK, priority, bufferDesc, buffer); +} + +static nri::Result CreateDecodeBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { + return core.CreateCommittedBuffer(device, nri::MemoryLocation::HOST_UPLOAD, priority, bufferDesc, buffer); } } // namespace @@ -1139,7 +1137,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { nri::BufferDesc bitstreamBufferDesc = {}; bitstreamBufferDesc.size = BITSTREAM_SIZE; - bitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE | nri::BufferUsageBits::VIDEO_DECODE; + bitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; nri::BufferDesc decodeBitstreamBufferDesc = {}; decodeBitstreamBufferDesc.size = BITSTREAM_SIZE; @@ -1166,7 +1164,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (CreateVideoBitstreamBuffer(NRI, *m_Device, graphicsAPI, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { + if (CreateEncodeBitstreamBuffer(NRI, *m_Device, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create encode bitstream buffer"; return; } @@ -1176,7 +1174,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (CreateVideoBitstreamBuffer(NRI, *m_Device, graphicsAPI, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { + if (CreateDecodeBitstreamBuffer(NRI, *m_Device, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create decode bitstream buffer"; return; } @@ -1387,14 +1385,11 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { encodeDesc.av1PictureDesc = m_Codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; if (!SubmitOneTime(NRI, *m_VideoEncodeQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::BufferBarrierDesc bufferBarriers[3] = {}; - bufferBarriers[0].buffer = m_BitstreamBuffer; - bufferBarriers[0].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + nri::BufferBarrierDesc bufferBarriers[2] = {}; + bufferBarriers[0].buffer = m_MetadataBuffer; bufferBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[1].buffer = m_MetadataBuffer; + bufferBarriers[1].buffer = m_ResolvedMetadataBuffer; bufferBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[2].buffer = m_ResolvedMetadataBuffer; - bufferBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; nri::TextureBarrierDesc textureBarriers[2] = {}; textureBarriers[0].texture = m_EncodeTexture; @@ -1417,13 +1412,11 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { barrierDesc.textureNum = helper::GetCountOf(textureBarriers); NRI.CmdBarrier(commandBuffer, barrierDesc); Video.CmdEncodeVideo(commandBuffer, encodeDesc); - bufferBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[0].after = {}; // D3D12 resolves encode metadata inside CmdEncodeVideo and transitions the raw metadata buffer to encode-read before returning. - bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[0].after = {}; + bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; bufferBarriers[1].after = {}; - bufferBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[2].after = {}; textureBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; From e90cfcd4e0a38c14682e78a9ee1c515e82ff2c7a Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Wed, 6 May 2026 15:48:18 +0200 Subject: [PATCH 07/28] Update video sample for queue selection cleanup --- Source/VideoEncodeDecode.cpp | 45 +++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 7a4bed7..70fb9d7 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -336,6 +336,7 @@ class Sample : public SampleBase { double m_StartTimeSec = 0.0; double m_LastRoundTripTimeSec = -1.0; bool m_VideoReady = false; + bool m_VideoQueuesRequested = false; bool m_DecodePreviewReady = false; bool m_PreviewTexturesShaderReadable = false; bool m_MetadataReadbackPending = false; @@ -348,19 +349,19 @@ Sample::~Sample() { if (Video.DestroyVideoPicture) { if (m_DecodePicture) - Video.DestroyVideoPicture(*m_DecodePicture); + Video.DestroyVideoPicture(m_DecodePicture); if (m_ReconstructedPicture) - Video.DestroyVideoPicture(*m_ReconstructedPicture); + Video.DestroyVideoPicture(m_ReconstructedPicture); if (m_EncodePicture) - Video.DestroyVideoPicture(*m_EncodePicture); + Video.DestroyVideoPicture(m_EncodePicture); if (m_DecodeParameters) - Video.DestroyVideoSessionParameters(*m_DecodeParameters); + Video.DestroyVideoSessionParameters(m_DecodeParameters); if (m_EncodeParameters) - Video.DestroyVideoSessionParameters(*m_EncodeParameters); + Video.DestroyVideoSessionParameters(m_EncodeParameters); if (m_DecodeSession) - Video.DestroyVideoSession(*m_DecodeSession); + Video.DestroyVideoSession(m_DecodeSession); if (m_EncodeSession) - Video.DestroyVideoSession(*m_EncodeSession); + Video.DestroyVideoSession(m_EncodeSession); } if (m_MetadataReadbackCommandBuffer) @@ -465,12 +466,23 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { uint32_t adapterDescsNum = helper::GetCountOf(adapterDesc); NRI_ABORT_ON_FAILURE(nri::nriEnumerateAdapters(adapterDesc, adapterDescsNum)); + const nri::AdapterDesc& selectedAdapter = adapterDesc[std::min(m_AdapterIndex, adapterDescsNum - 1)]; + m_VideoQueuesRequested = graphicsAPI != nri::GraphicsAPI::D3D11 && + selectedAdapter.queueNum[(uint32_t)nri::QueueType::VIDEO_ENCODE] && + selectedAdapter.queueNum[(uint32_t)nri::QueueType::VIDEO_DECODE]; + + nri::QueueFamilyDesc queueFamilies[3] = {}; + uint32_t queueFamilyNum = 0; + queueFamilies[queueFamilyNum].queueNum = 1; + queueFamilies[queueFamilyNum++].queueType = nri::QueueType::GRAPHICS; + if (m_VideoQueuesRequested) { + queueFamilies[queueFamilyNum].queueNum = 1; + queueFamilies[queueFamilyNum++].queueType = nri::QueueType::VIDEO_ENCODE; + queueFamilies[queueFamilyNum].queueNum = 1; + queueFamilies[queueFamilyNum++].queueType = nri::QueueType::VIDEO_DECODE; + } + nri::DeviceCreationDesc deviceCreationDesc = {}; - nri::QueueFamilyDesc queueFamilies[] = { - {nullptr, 1, nri::QueueType::GRAPHICS}, - {nullptr, 1, nri::QueueType::VIDEO_ENCODE}, - {nullptr, 1, nri::QueueType::VIDEO_DECODE}, - }; deviceCreationDesc.graphicsAPI = graphicsAPI; deviceCreationDesc.enableGraphicsAPIValidation = m_DebugAPI; @@ -478,10 +490,10 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { deviceCreationDesc.enableD3D11CommandBufferEmulation = D3D11_ENABLE_COMMAND_BUFFER_EMULATION; deviceCreationDesc.disableD3D12EnhancedBarriers = D3D12_DISABLE_ENHANCED_BARRIERS; deviceCreationDesc.vkBindingOffsets = VK_BINDING_OFFSETS; - deviceCreationDesc.adapterDesc = &adapterDesc[std::min(m_AdapterIndex, adapterDescsNum - 1)]; + deviceCreationDesc.adapterDesc = &selectedAdapter; deviceCreationDesc.allocationCallbacks = m_AllocationCallbacks; deviceCreationDesc.queueFamilies = queueFamilies; - deviceCreationDesc.queueFamilyNum = helper::GetCountOf(queueFamilies); + deviceCreationDesc.queueFamilyNum = queueFamilyNum; NRI_ABORT_ON_FAILURE(nri::nriCreateDevice(deviceCreationDesc, m_Device)); NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::CoreInterface), (nri::CoreInterface*)&NRI)); @@ -842,6 +854,11 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } + if (!m_VideoQueuesRequested) { + m_VideoStatus = "Adapter has no NRI video encode/decode queues"; + return; + } + if (!deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_ENCODE] || !deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_DECODE]) { m_VideoStatus = "Adapter has no NRI video encode/decode queues"; return; From 47f6bb7d76ab99deae72ad44328afff68c28a374 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 7 May 2026 14:30:23 +0200 Subject: [PATCH 08/28] Update video encode decode sample --- README.md | 8 + Source/VideoEncodeDecode.cpp | 378 ++++++++++++++++++++++++++--------- 2 files changed, 286 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index 01fab10..296bf78 100644 --- a/README.md +++ b/README.md @@ -59,3 +59,11 @@ The executables from `_Bin` directory load resources from `_Data`, therefore the - Triangle - simple textured triangle rendering (also multiview demonstration in _FLEXIBLE_ mode) - VideoEncodeDecode - H.264/H.265/AV1 NV12 video encode/decode round trip; built only when the selected NRI source provides the NRIVideo extension - Wrapper - shows how to wrap native D3D11/D3D12/VK objects into *NRI* entities + +### VideoEncodeDecode AV1 notes + +`VideoEncodeDecode --codec=AV1 --av1Frame=P` encodes a static generated source image as an IDR reference followed by a P frame. The source is intentionally frozen for this permutation so the left generated image and right decoded image can be compared directly while validating AV1 reference/DPB handling. It is not intended to demonstrate animated AV1 P-frame motion. + +The framework `--width` and `--height` arguments control the sample output window size only. Use `--videoWidth` and `--videoHeight` to select the NV12 encode/decode surface size. The default video size is 1920x1080. + +The sample uses CQP rate control by default. Use `--qpI`, `--qpP`, and `--qpB` to change the per-frame-type quantizers, and `--av1BaseQIndex` to change AV1's frame base quantizer. H.264/H.265 QP values must be in the 0..51 range; AV1 quantizer values use the 0..255 range. Lower values generally produce higher quality and larger bitstreams. diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 70fb9d7..956af94 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -17,8 +17,8 @@ namespace { -constexpr uint32_t VIDEO_WIDTH = 1920; -constexpr uint32_t VIDEO_HEIGHT = 1088; +constexpr uint32_t DEFAULT_VIDEO_WIDTH = 1920; +constexpr uint32_t DEFAULT_VIDEO_HEIGHT = 1080; constexpr double ROUND_TRIP_INTERVAL_SEC = 1.0 / 60.0; constexpr uint64_t BITSTREAM_SIZE = 2 * 1024 * 1024; constexpr uint64_t ENCODED_SLICE_OFFSET = 4096; @@ -26,10 +26,6 @@ constexpr uint64_t AV1_HEADER_READBACK_SIZE = 4096; constexpr uint64_t METADATA_SIZE = 4 * 1024 * 1024; constexpr uint64_t RESOLVED_METADATA_SIZE = 4096; -static_assert(VIDEO_WIDTH % 4 == 0, "Compute-backed NV12 writer expects width divisible by 4"); -static_assert(VIDEO_WIDTH % 16 == 0, "H.264 macroblock width should stay aligned"); -static_assert(VIDEO_HEIGHT % 16 == 0, "H.264 macroblock height should stay aligned"); - enum PatternOperation : uint32_t { OP_GENERATE_PATTERN = 0, OP_NV12_TO_PREVIEW = 1, @@ -70,14 +66,14 @@ static uint64_t GetEncodedPayloadHeaderSkip(SampleCodec codec, uint64_t encodedB return std::min(headerSize, encodedBitstreamBytes); } -static nri::VideoAV1SequenceDesc MakeAV1SequenceDesc() { +static nri::VideoAV1SequenceDesc MakeAV1SequenceDesc(uint32_t width, uint32_t height) { nri::VideoAV1SequenceDesc desc = {}; desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | nri::VideoAV1SequenceBits::ENABLE_CDEF | nri::VideoAV1SequenceBits::ENABLE_RESTORATION | nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; desc.bitDepth = 8; desc.subsamplingX = 1; desc.subsamplingY = 1; - desc.maxFrameWidthMinus1 = VIDEO_WIDTH - 1; - desc.maxFrameHeightMinus1 = VIDEO_HEIGHT - 1; + desc.maxFrameWidthMinus1 = (uint16_t)(width - 1); + desc.maxFrameHeightMinus1 = (uint16_t)(height - 1); desc.frameWidthBitsMinus1 = 15; desc.frameHeightBitsMinus1 = 15; desc.orderHintBitsMinus1 = 7; @@ -96,8 +92,8 @@ struct QueuedFrame { }; struct PatternConstants { - uint32_t width = VIDEO_WIDTH; - uint32_t height = VIDEO_HEIGHT; + uint32_t width = DEFAULT_VIDEO_WIDTH; + uint32_t height = DEFAULT_VIDEO_HEIGHT; uint32_t yOffsetBytes = 0; uint32_t yRowPitchBytes = 0; uint32_t uvRowPitchBytes = 0; @@ -109,28 +105,28 @@ struct PatternConstants { }; struct Nv12BufferLayout { - uint32_t yRowPitchBytes = VIDEO_WIDTH; - uint32_t ySlicePitchBytes = VIDEO_WIDTH * VIDEO_HEIGHT; - uint64_t uvOffsetBytes = uint64_t(VIDEO_WIDTH) * VIDEO_HEIGHT; - uint32_t uvRowPitchBytes = VIDEO_WIDTH; - uint32_t uvSlicePitchBytes = VIDEO_WIDTH * VIDEO_HEIGHT / 2; - uint64_t totalSizeBytes = uint64_t(VIDEO_WIDTH) * VIDEO_HEIGHT * 3 / 2; + uint32_t yRowPitchBytes = DEFAULT_VIDEO_WIDTH; + uint32_t ySlicePitchBytes = DEFAULT_VIDEO_WIDTH * DEFAULT_VIDEO_HEIGHT; + uint64_t uvOffsetBytes = uint64_t(DEFAULT_VIDEO_WIDTH) * DEFAULT_VIDEO_HEIGHT; + uint32_t uvRowPitchBytes = DEFAULT_VIDEO_WIDTH; + uint32_t uvSlicePitchBytes = DEFAULT_VIDEO_WIDTH * DEFAULT_VIDEO_HEIGHT / 2; + uint64_t totalSizeBytes = uint64_t(DEFAULT_VIDEO_WIDTH) * DEFAULT_VIDEO_HEIGHT * 3 / 2; }; static uint64_t AlignUp(uint64_t value, uint64_t alignment) { return alignment == 0 ? value : ((value + alignment - 1) / alignment) * alignment; } -static Nv12BufferLayout MakeNv12BufferLayout(const nri::DeviceDesc& deviceDesc) { +static Nv12BufferLayout MakeNv12BufferLayout(const nri::DeviceDesc& deviceDesc, uint32_t width, uint32_t height) { const uint32_t rowAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureRow, 1u); const uint32_t sliceAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureSlice, 1u); Nv12BufferLayout layout = {}; - layout.yRowPitchBytes = (uint32_t)AlignUp(VIDEO_WIDTH, rowAlignment); - layout.ySlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.yRowPitchBytes) * VIDEO_HEIGHT, sliceAlignment); + layout.yRowPitchBytes = (uint32_t)AlignUp(width, rowAlignment); + layout.ySlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.yRowPitchBytes) * height, sliceAlignment); layout.uvOffsetBytes = layout.ySlicePitchBytes; - layout.uvRowPitchBytes = (uint32_t)AlignUp(VIDEO_WIDTH, rowAlignment); - layout.uvSlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.uvRowPitchBytes) * (VIDEO_HEIGHT / 2), sliceAlignment); + layout.uvRowPitchBytes = (uint32_t)AlignUp(width, rowAlignment); + layout.uvSlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.uvRowPitchBytes) * (height / 2), sliceAlignment); layout.totalSizeBytes = layout.uvOffsetBytes + layout.uvSlicePitchBytes; return layout; } @@ -182,7 +178,7 @@ static bool SubmitOneTime( return ok; } -static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, const Nv12BufferLayout& layout, nri::Buffer& src, nri::Texture& dst) { +static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, const Nv12BufferLayout& layout, nri::Buffer& src, nri::Texture& dst, uint32_t width, uint32_t height) { return SubmitOneTime(core, queue, [&](nri::CommandBuffer& commandBuffer) { nri::BufferBarrierDesc bufferBarrier = {}; bufferBarrier.buffer = &src; @@ -205,8 +201,8 @@ static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, core.CmdBarrier(commandBuffer, barrierDesc); nri::TextureRegionDesc lumaRegion = {}; - lumaRegion.width = VIDEO_WIDTH; - lumaRegion.height = VIDEO_HEIGHT; + lumaRegion.width = (nri::Dim_t)width; + lumaRegion.height = (nri::Dim_t)height; lumaRegion.depth = 1; lumaRegion.planes = nri::PlaneBits::PLANE_0; @@ -216,8 +212,8 @@ static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, core.CmdUploadBufferToTexture(commandBuffer, dst, lumaRegion, src, lumaLayout); nri::TextureRegionDesc chromaRegion = {}; - chromaRegion.width = VIDEO_WIDTH; - chromaRegion.height = VIDEO_HEIGHT; + chromaRegion.width = (nri::Dim_t)width; + chromaRegion.height = (nri::Dim_t)height; chromaRegion.depth = 1; chromaRegion.planes = nri::PlaneBits::PLANE_1; @@ -291,7 +287,9 @@ class Sample : public SampleBase { nri::VideoSessionParameters* m_DecodeParameters = nullptr; nri::Texture* m_EncodeTexture = nullptr; nri::Texture* m_ReconstructedTexture = nullptr; + nri::Texture* m_AV1PReconstructedTexture = nullptr; nri::Texture* m_DecodeTexture = nullptr; + nri::Texture* m_AV1PDecodeTexture = nullptr; nri::Texture* m_SourcePreviewTexture = nullptr; nri::Texture* m_DecodePreviewTexture = nullptr; nri::Buffer* m_UploadBuffer = nullptr; @@ -313,7 +311,9 @@ class Sample : public SampleBase { nri::Buffer* m_ResolvedMetadataReadbackBuffer = nullptr; nri::VideoPicture* m_EncodePicture = nullptr; nri::VideoPicture* m_ReconstructedPicture = nullptr; + nri::VideoPicture* m_AV1PReconstructedPicture = nullptr; nri::VideoPicture* m_DecodePicture = nullptr; + nri::VideoPicture* m_AV1PDecodePicture = nullptr; nri::CommandAllocator* m_MetadataReadbackCommandAllocator = nullptr; nri::CommandBuffer* m_MetadataReadbackCommandBuffer = nullptr; nri::Fence* m_MetadataReadbackFence = nullptr; @@ -326,6 +326,13 @@ class Sample : public SampleBase { std::string m_VideoStatus = "Initializing video"; std::string m_PreviewStatus = "Initializing preview"; std::string m_CodecArg = "H264"; + std::string m_AV1FrameArg = "IDR"; + uint32_t m_VideoWidth = DEFAULT_VIDEO_WIDTH; + uint32_t m_VideoHeight = DEFAULT_VIDEO_HEIGHT; + uint32_t m_QpI = 20; + uint32_t m_QpP = 22; + uint32_t m_QpB = 24; + uint32_t m_AV1BaseQIndex = 20; SampleCodec m_Codec = SampleCodec::H264; nri::VideoH264SequenceParameterSetDesc m_H264Sps = {}; nri::VideoH264PictureParameterSetDesc m_H264Pps = {}; @@ -340,6 +347,8 @@ class Sample : public SampleBase { bool m_DecodePreviewReady = false; bool m_PreviewTexturesShaderReadable = false; bool m_MetadataReadbackPending = false; + bool m_AV1PFrameVisual = false; + uint32_t m_AV1PFrameStage = 0; uint64_t m_MetadataReadbackFenceValue = 0; }; @@ -348,8 +357,12 @@ Sample::~Sample() { NRI.DeviceWaitIdle(m_Device); if (Video.DestroyVideoPicture) { + if (m_AV1PDecodePicture) + Video.DestroyVideoPicture(m_AV1PDecodePicture); if (m_DecodePicture) Video.DestroyVideoPicture(m_DecodePicture); + if (m_AV1PReconstructedPicture) + Video.DestroyVideoPicture(m_AV1PReconstructedPicture); if (m_ReconstructedPicture) Video.DestroyVideoPicture(m_ReconstructedPicture); if (m_EncodePicture) @@ -408,8 +421,12 @@ Sample::~Sample() { NRI.DestroyTexture(m_DecodePreviewTexture); if (m_DecodeTexture) NRI.DestroyTexture(m_DecodeTexture); + if (m_AV1PDecodeTexture) + NRI.DestroyTexture(m_AV1PDecodeTexture); if (m_ReconstructedTexture) NRI.DestroyTexture(m_ReconstructedTexture); + if (m_AV1PReconstructedTexture) + NRI.DestroyTexture(m_AV1PReconstructedTexture); if (m_EncodeTexture) NRI.DestroyTexture(m_EncodeTexture); @@ -440,14 +457,41 @@ Sample::~Sample() { void Sample::InitCmdLine(cmdline::parser& cmdLine) { cmdLine.add("codec", 0, "video codec: H264, H265, or AV1", false, m_CodecArg, cmdline::oneof("H264", "H265", "AV1")); + cmdLine.add("av1Frame", 0, "AV1 visual frame permutation: IDR or P", false, m_AV1FrameArg, cmdline::oneof("IDR", "P")); + cmdLine.add("videoWidth", 0, "NV12 video encode/decode width", false, m_VideoWidth); + cmdLine.add("videoHeight", 0, "NV12 video encode/decode height", false, m_VideoHeight); + cmdLine.add("qpI", 0, "CQP quantizer for I/IDR frames", false, m_QpI); + cmdLine.add("qpP", 0, "CQP quantizer for P frames", false, m_QpP); + cmdLine.add("qpB", 0, "CQP quantizer for B frames", false, m_QpB); + cmdLine.add("av1BaseQIndex", 0, "AV1 base quantizer index", false, m_AV1BaseQIndex); } void Sample::ReadCmdLine(cmdline::parser& cmdLine) { m_CodecArg = cmdLine.get("codec"); + m_AV1FrameArg = cmdLine.get("av1Frame"); + m_VideoWidth = cmdLine.get("videoWidth"); + m_VideoHeight = cmdLine.get("videoHeight"); + m_QpI = cmdLine.get("qpI"); + m_QpP = cmdLine.get("qpP"); + m_QpB = cmdLine.get("qpB"); + m_AV1BaseQIndex = cmdLine.get("av1BaseQIndex"); m_Codec = m_CodecArg == "H265" ? SampleCodec::H265 : (m_CodecArg == "AV1" ? SampleCodec::AV1 : SampleCodec::H264); + m_AV1PFrameVisual = m_Codec == SampleCodec::AV1 && m_AV1FrameArg == "P"; } bool Sample::Initialize(nri::GraphicsAPI graphicsAPI, bool) { + if (!m_VideoWidth || !m_VideoHeight || (m_VideoWidth % 4) != 0 || (m_VideoHeight % 2) != 0 || m_VideoWidth > 65535 || m_VideoHeight > 65535) { + m_VideoStatus = "Video size must be non-zero, width must be divisible by 4, height must be even, and both dimensions must fit 16-bit video descriptors"; + std::fprintf(stderr, "%s\n", m_VideoStatus.c_str()); + return false; + } + const uint32_t maxCodecQp = m_Codec == SampleCodec::AV1 ? 255 : 51; + if (m_QpI > maxCodecQp || m_QpP > maxCodecQp || m_QpB > maxCodecQp || m_AV1BaseQIndex > 255) { + m_VideoStatus = m_Codec == SampleCodec::AV1 ? "AV1 quantizers must be in the 0..255 range" : "H.264/H.265 QP values must be in the 0..51 range"; + std::fprintf(stderr, "%s\n", m_VideoStatus.c_str()); + return false; + } + m_GraphicsAPI = graphicsAPI; if (!InitializeGraphics(graphicsAPI)) return false; @@ -501,7 +545,7 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::StreamerInterface), (nri::StreamerInterface*)&NRI)); NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::SwapChainInterface), (nri::SwapChainInterface*)&NRI)); - m_Nv12Layout = MakeNv12BufferLayout(NRI.GetDeviceDesc(*m_Device)); + m_Nv12Layout = MakeNv12BufferLayout(NRI.GetDeviceDesc(*m_Device), m_VideoWidth, m_VideoHeight); nri::StreamerDesc streamerDesc = {}; streamerDesc.dynamicBufferMemoryLocation = nri::MemoryLocation::HOST_UPLOAD; @@ -560,8 +604,8 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { PatternConstants Sample::MakePatternConstants(PatternOperation operation, float timeSec) const { PatternConstants patternConstants = {}; - patternConstants.width = VIDEO_WIDTH; - patternConstants.height = VIDEO_HEIGHT; + patternConstants.width = m_VideoWidth; + patternConstants.height = m_VideoHeight; patternConstants.yRowPitchBytes = m_Nv12Layout.yRowPitchBytes; patternConstants.uvRowPitchBytes = m_Nv12Layout.uvRowPitchBytes; patternConstants.uvOffsetBytes = (uint32_t)m_Nv12Layout.uvOffsetBytes; @@ -582,7 +626,7 @@ void Sample::InitializeGeneratedFrames(float timeSec) { return; } - if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture)) { + if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture, m_VideoWidth, m_VideoHeight)) { m_PreviewStatus = "Failed to upload NV12 source to video texture"; return; } @@ -673,8 +717,8 @@ bool Sample::TryInitializePreviewTextures(nri::GraphicsAPI) { nri::TextureDesc previewTextureDesc = {}; previewTextureDesc.type = nri::TextureType::TEXTURE_2D; previewTextureDesc.format = nri::Format::RGBA8_UNORM; - previewTextureDesc.width = VIDEO_WIDTH; - previewTextureDesc.height = VIDEO_HEIGHT; + previewTextureDesc.width = (nri::Dim_t)m_VideoWidth; + previewTextureDesc.height = (nri::Dim_t)m_VideoHeight; previewTextureDesc.mipNum = 1; previewTextureDesc.layerNum = 1; previewTextureDesc.usage = nri::TextureUsageBits::SHADER_RESOURCE | nri::TextureUsageBits::SHADER_RESOURCE_STORAGE; @@ -873,8 +917,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { encodeSessionDesc.usage = nri::VideoUsage::ENCODE; encodeSessionDesc.codec = GetNriCodec(m_Codec); encodeSessionDesc.format = nri::Format::NV12_UNORM; - encodeSessionDesc.width = VIDEO_WIDTH; - encodeSessionDesc.height = VIDEO_HEIGHT; + encodeSessionDesc.width = m_VideoWidth; + encodeSessionDesc.height = m_VideoHeight; encodeSessionDesc.maxReferenceNum = 1; nri::VideoSessionDesc decodeSessionDesc = encodeSessionDesc; @@ -906,8 +950,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { sps.pictureOrderCountType = 0; sps.log2MaxPictureOrderCountLsbMinus4 = 0; sps.referenceFrameNum = 1; - sps.pictureWidthInMbsMinus1 = VIDEO_WIDTH / 16 - 1; - sps.pictureHeightInMapUnitsMinus1 = VIDEO_HEIGHT / 16 - 1; + sps.pictureWidthInMbsMinus1 = (uint16_t)((m_VideoWidth + 15) / 16 - 1); + sps.pictureHeightInMapUnitsMinus1 = (uint16_t)((m_VideoHeight + 15) / 16 - 1); nri::VideoH264PictureParameterSetDesc pps = {}; pps.flags = nri::VideoH264PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; @@ -942,8 +986,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { h265Sps.maxSubLayersMinus1 = vps.maxSubLayersMinus1; h265Sps.sequenceParameterSetId = 0; h265Sps.chromaFormatIdc = 1; - h265Sps.pictureWidthInLumaSamples = VIDEO_WIDTH; - h265Sps.pictureHeightInLumaSamples = VIDEO_HEIGHT; + h265Sps.pictureWidthInLumaSamples = m_VideoWidth; + h265Sps.pictureHeightInLumaSamples = m_VideoHeight; h265Sps.log2MaxPictureOrderCountLsbMinus4 = 3; h265Sps.log2MinLumaCodingBlockSizeMinus3 = 0; h265Sps.log2DiffMaxMinLumaCodingBlockSize = 2; @@ -974,7 +1018,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { h265Parameters.maxSequenceParameterSetNum = 1; h265Parameters.maxPictureParameterSetNum = 1; - m_AV1Sequence = MakeAV1SequenceDesc(); + m_AV1Sequence = MakeAV1SequenceDesc(m_VideoWidth, m_VideoHeight); nri::VideoAV1SessionParametersDesc av1Parameters = {}; av1Parameters.sequence = m_AV1Sequence; @@ -1004,8 +1048,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { encodeTextureDesc.type = nri::TextureType::TEXTURE_2D; encodeTextureDesc.usage = nri::TextureUsageBits::VIDEO_ENCODE; encodeTextureDesc.format = nri::Format::NV12_UNORM; - encodeTextureDesc.width = VIDEO_WIDTH; - encodeTextureDesc.height = VIDEO_HEIGHT; + encodeTextureDesc.width = (nri::Dim_t)m_VideoWidth; + encodeTextureDesc.height = (nri::Dim_t)m_VideoHeight; encodeTextureDesc.mipNum = 1; encodeTextureDesc.layerNum = 1; encodeTextureDesc.videoCodec = GetNriCodec(m_Codec); @@ -1025,19 +1069,39 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { } NRI.SetDebugName(m_ReconstructedTexture, "VideoReconstructedTexture"); + if (m_AV1PFrameVisual) { + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeTextureDesc, m_AV1PReconstructedTexture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create second NV12 reconstructed texture"; + return; + } + NRI.SetDebugName(m_AV1PReconstructedTexture, "VideoAV1PReconstructedTexture"); + } + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, decodeTextureDesc, m_DecodeTexture) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create NV12 decode texture"; return; } NRI.SetDebugName(m_DecodeTexture, "VideoDecodeTexture"); + if (m_AV1PFrameVisual) { + if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, decodeTextureDesc, m_AV1PDecodeTexture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create second NV12 decode texture"; + return; + } + NRI.SetDebugName(m_AV1PDecodeTexture, "VideoAV1PDecodeTexture"); + } + if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::TextureBarrierDesc textureBarriers[3] = {}; + nri::TextureBarrierDesc textureBarriers[5] = {}; textureBarriers[0].texture = m_EncodeTexture; textureBarriers[1].texture = m_ReconstructedTexture; textureBarriers[2].texture = m_DecodeTexture; + textureBarriers[3].texture = m_AV1PReconstructedTexture; + textureBarriers[4].texture = m_AV1PDecodeTexture; for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { + if (!textureBarrier.texture) + continue; textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::UNDEFINED, nri::StageBits::ALL}; textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarrier.mipNum = nri::REMAINING; @@ -1047,7 +1111,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { nri::BarrierDesc barrierDesc = {}; barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + barrierDesc.textureNum = m_AV1PFrameVisual ? helper::GetCountOf(textureBarriers) : 3; NRI.CmdBarrier(commandBuffer, barrierDesc); })) { m_VideoStatus = "Failed to initialize video texture layouts"; @@ -1225,15 +1289,15 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { encodePictureDesc.texture = m_EncodeTexture; encodePictureDesc.usage = nri::VideoPictureUsage::ENCODE_INPUT; encodePictureDesc.format = nri::Format::NV12_UNORM; - encodePictureDesc.width = VIDEO_WIDTH; - encodePictureDesc.height = VIDEO_HEIGHT; + encodePictureDesc.width = m_VideoWidth; + encodePictureDesc.height = m_VideoHeight; nri::VideoPictureDesc decodePictureDesc = {}; decodePictureDesc.texture = m_DecodeTexture; decodePictureDesc.usage = nri::VideoPictureUsage::DECODE_OUTPUT; decodePictureDesc.format = nri::Format::NV12_UNORM; - decodePictureDesc.width = VIDEO_WIDTH; - decodePictureDesc.height = VIDEO_HEIGHT; + decodePictureDesc.width = m_VideoWidth; + decodePictureDesc.height = m_VideoHeight; nri::VideoPictureDesc reconstructedPictureDesc = encodePictureDesc; reconstructedPictureDesc.texture = m_ReconstructedTexture; @@ -1249,11 +1313,27 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } + if (m_AV1PFrameVisual) { + reconstructedPictureDesc.texture = m_AV1PReconstructedTexture; + if (Video.CreateVideoPicture(*m_Device, reconstructedPictureDesc, m_AV1PReconstructedPicture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create second reconstructed picture"; + return; + } + } + if (Video.CreateVideoPicture(*m_Device, decodePictureDesc, m_DecodePicture) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create decode picture"; return; } + if (m_AV1PFrameVisual) { + decodePictureDesc.texture = m_AV1PDecodeTexture; + if (Video.CreateVideoPicture(*m_Device, decodePictureDesc, m_AV1PDecodePicture) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to create second decode picture"; + return; + } + } + m_VideoReady = true; m_VideoStatus = std::string("NRI video queues and ") + GetCodecName(m_Codec) + " encode/decode objects initialized"; } @@ -1309,13 +1389,14 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { return false; } - PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); + const bool av1PFrame = m_AV1PFrameVisual && m_AV1PFrameStage == 1; + PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, m_AV1PFrameVisual ? 0.0f : timeSec); if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { m_VideoStatus = "Failed to generate NV12 source pattern via compute"; return false; } - if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture)) { + if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture, m_VideoWidth, m_VideoHeight)) { m_VideoStatus = "Failed to upload NV12 source to video texture"; return false; } @@ -1333,13 +1414,15 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { } nri::VideoEncodePictureDesc pictureDesc = {}; - pictureDesc.frameType = nri::VideoEncodeFrameType::IDR; - pictureDesc.idrPictureId = 1; - - uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((VIDEO_WIDTH + 7) >> 3))}; - uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((VIDEO_HEIGHT + 7) >> 3))}; - uint16_t av1WidthInSuperblocksMinus1[] = {(uint16_t)(((VIDEO_WIDTH + 63) / 64) - 1)}; - uint16_t av1HeightInSuperblocksMinus1[] = {(uint16_t)(((VIDEO_HEIGHT + 63) / 64) - 1)}; + pictureDesc.frameType = av1PFrame ? nri::VideoEncodeFrameType::P : nri::VideoEncodeFrameType::IDR; + pictureDesc.frameIndex = av1PFrame ? 1 : 0; + pictureDesc.pictureOrderCount = av1PFrame ? 1 : 0; + pictureDesc.idrPictureId = av1PFrame ? 0 : 1; + + uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((m_VideoWidth + 7) >> 3))}; + uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((m_VideoHeight + 7) >> 3))}; + uint16_t av1WidthInSuperblocksMinus1[] = {(uint16_t)(((m_VideoWidth + 63) / 64) - 1)}; + uint16_t av1HeightInSuperblocksMinus1[] = {(uint16_t)(((m_VideoHeight + 63) / 64) - 1)}; nri::VideoAV1TileLayoutDesc av1TileLayout = {}; av1TileLayout.columnNum = 1; av1TileLayout.rowNum = 1; @@ -1362,13 +1445,15 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { params[5] = 1 << 16; } nri::VideoAV1PictureDesc av1PictureDesc = {}; - av1PictureDesc.currentFrameId = 0; - av1PictureDesc.refreshFrameFlags = 0xFF; - av1PictureDesc.primaryReferenceName = nri::VideoAV1ReferenceName::NONE; - av1PictureDesc.flags = nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE | nri::VideoAV1PictureBits::DISABLE_CDF_UPDATE | nri::VideoAV1PictureBits::ALLOW_SCREEN_CONTENT_TOOLS | nri::VideoAV1PictureBits::FORCE_INTEGER_MV | nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME; - av1PictureDesc.renderWidthMinus1 = VIDEO_WIDTH - 1; - av1PictureDesc.renderHeightMinus1 = VIDEO_HEIGHT - 1; - av1PictureDesc.baseQIndex = 20; + av1PictureDesc.currentFrameId = av1PFrame ? 1 : 0; + av1PictureDesc.orderHint = av1PFrame ? 1 : 0; + av1PictureDesc.refreshFrameFlags = av1PFrame ? 0x1 : 0xFF; + av1PictureDesc.primaryReferenceName = av1PFrame ? nri::VideoAV1ReferenceName::LAST : nri::VideoAV1ReferenceName::NONE; + av1PictureDesc.flags = av1PFrame ? nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME + : nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE | nri::VideoAV1PictureBits::DISABLE_CDF_UPDATE | nri::VideoAV1PictureBits::ALLOW_SCREEN_CONTENT_TOOLS | nri::VideoAV1PictureBits::FORCE_INTEGER_MV | nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME; + av1PictureDesc.renderWidthMinus1 = (uint16_t)(m_VideoWidth - 1); + av1PictureDesc.renderHeightMinus1 = (uint16_t)(m_VideoHeight - 1); + av1PictureDesc.baseQIndex = (uint8_t)m_AV1BaseQIndex; av1PictureDesc.interpolationFilter = 0; av1PictureDesc.txMode = 2; av1PictureDesc.cdefDampingMinus3 = 3; @@ -1377,12 +1462,35 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { av1PictureDesc.cdef = &av1Cdef; av1PictureDesc.loopRestoration = &av1LoopRestoration; av1PictureDesc.globalMotion = &av1GlobalMotion; + nri::VideoReference av1Reference = {m_ReconstructedPicture, 0}; + nri::VideoAV1ReferenceDesc av1References[8] = {}; + if (av1PFrame) { + const nri::VideoAV1ReferenceName av1ReferenceNames[] = { + nri::VideoAV1ReferenceName::LAST, + nri::VideoAV1ReferenceName::LAST2, + nri::VideoAV1ReferenceName::LAST3, + nri::VideoAV1ReferenceName::GOLDEN, + nri::VideoAV1ReferenceName::BWDREF, + nri::VideoAV1ReferenceName::ALTREF2, + nri::VideoAV1ReferenceName::ALTREF, + }; + for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { + av1References[i].name = av1ReferenceNames[i]; + av1References[i].refFrameIndex = 0; + av1References[i].frameType = nri::VideoEncodeFrameType::IDR; + av1References[i].orderHint = 0; + av1References[i].frameId = 0; + av1References[i].slot = 0; + } + av1PictureDesc.references = av1References; + av1PictureDesc.referenceNum = helper::GetCountOf(av1ReferenceNames); + } nri::VideoEncodeRateControlDesc rateControlDesc = {}; rateControlDesc.mode = nri::VideoEncodeRateControlMode::CQP; - rateControlDesc.qpI = 20; - rateControlDesc.qpP = 22; - rateControlDesc.qpB = 24; + rateControlDesc.qpI = (uint8_t)m_QpI; + rateControlDesc.qpP = (uint8_t)m_QpP; + rateControlDesc.qpB = (uint8_t)m_QpB; rateControlDesc.frameRateNumerator = 30; rateControlDesc.frameRateDenominator = 1; @@ -1397,6 +1505,12 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { encodeDesc.pictureDesc = &pictureDesc; encodeDesc.rateControlDesc = &rateControlDesc; encodeDesc.reconstructedPicture = m_ReconstructedPicture; + if (av1PFrame) { + encodeDesc.reconstructedPicture = m_AV1PReconstructedPicture; + encodeDesc.references = &av1Reference; + encodeDesc.referenceNum = 1; + encodeDesc.reconstructedSlot = 1; + } encodeDesc.metadata = m_MetadataBuffer; encodeDesc.resolvedMetadata = m_ResolvedMetadataBuffer; encodeDesc.av1PictureDesc = m_Codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; @@ -1408,25 +1522,31 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { bufferBarriers[1].buffer = m_ResolvedMetadataBuffer; bufferBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - nri::TextureBarrierDesc textureBarriers[2] = {}; + nri::TextureBarrierDesc textureBarriers[3] = {}; textureBarriers[0].texture = m_EncodeTexture; textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; textureBarriers[0].mipNum = nri::REMAINING; textureBarriers[0].layerNum = nri::REMAINING; textureBarriers[0].planes = nri::PlaneBits::ALL; - textureBarriers[1].texture = m_ReconstructedTexture; + textureBarriers[1].texture = av1PFrame ? m_AV1PReconstructedTexture : m_ReconstructedTexture; textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; textureBarriers[1].mipNum = nri::REMAINING; textureBarriers[1].layerNum = nri::REMAINING; textureBarriers[1].planes = nri::PlaneBits::ALL; + textureBarriers[2].texture = m_ReconstructedTexture; + textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].mipNum = nri::REMAINING; + textureBarriers[2].layerNum = nri::REMAINING; + textureBarriers[2].planes = nri::PlaneBits::ALL; nri::BarrierDesc barrierDesc = {}; barrierDesc.buffers = bufferBarriers; barrierDesc.bufferNum = helper::GetCountOf(bufferBarriers); barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; NRI.CmdBarrier(commandBuffer, barrierDesc); Video.CmdEncodeVideo(commandBuffer, encodeDesc); // D3D12 resolves encode metadata inside CmdEncodeVideo and transitions the raw metadata buffer to encode-read before returning. @@ -1438,8 +1558,10 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = helper::GetCountOf(textureBarriers); + barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; NRI.CmdBarrier(commandBuffer, barrierDesc); })) { m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submission failed"; @@ -1511,8 +1633,7 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { } m_MetadataReadbackPending = true; - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submitted; waiting for async metadata readback"; - return true; + return false; } bool Sample::TryDecodePendingMetadata(float timeSec) { @@ -1562,16 +1683,25 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { av1InfoDesc.encodedPayloadHeader = encodedHeader; av1InfoDesc.encodedPayloadHeaderSize = encodedHeader ? std::min(AV1_HEADER_READBACK_SIZE, feedback.encodedBitstreamWrittenBytes) : 0; const nri::Result av1InfoResult = Video.GetVideoEncodeAV1DecodeInfo(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, av1InfoDesc, av1DecodeInfo); - if (encodedHeader) - NRI.UnmapBuffer(*m_BitstreamHeaderReadbackBuffer); if (av1InfoResult != nri::Result::SUCCESS) { + if (encodedHeader) + NRI.UnmapBuffer(*m_BitstreamHeaderReadbackBuffer); m_VideoStatus = "Failed to prepare AV1 decode metadata"; return false; } + if (encodedHeader) + NRI.UnmapBuffer(*m_BitstreamHeaderReadbackBuffer); feedback.encodedBitstreamWrittenBytes = av1DecodeInfo.bitstreamOffset + av1DecodeInfo.bitstreamSize; } - return DecodeEncodedBitstream(feedback, m_Codec == SampleCodec::AV1 ? &av1DecodeInfo : nullptr, timeSec); + const bool decoded = DecodeEncodedBitstream(feedback, m_Codec == SampleCodec::AV1 ? &av1DecodeInfo : nullptr, timeSec); + if (decoded && m_AV1PFrameVisual && m_AV1PFrameStage == 0) { + m_AV1PFrameStage = 1; + m_DecodePreviewReady = false; + return TrySubmitEncodeAndMetadataReadback(timeSec); + } + + return decoded; } bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo, float timeSec) { @@ -1656,42 +1786,87 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co av1Info.tileLayout.widthInSuperblocksMinus1 = av1Info.widthInSuperblocksMinus1; av1Info.tileLayout.heightInSuperblocksMinus1 = av1Info.heightInSuperblocksMinus1; } + nri::VideoReference av1DecodeReference = {m_DecodePicture, 0}; + nri::VideoAV1ReferenceDesc av1DecodeReferences[8] = {}; + uint8_t av1DecodeOrderHints[8] = {}; + const bool av1PFrame = m_AV1PFrameVisual && m_AV1PFrameStage == 1; + if (av1PFrame) { + const nri::VideoAV1ReferenceName av1ReferenceNames[] = { + nri::VideoAV1ReferenceName::LAST, + nri::VideoAV1ReferenceName::LAST2, + nri::VideoAV1ReferenceName::LAST3, + nri::VideoAV1ReferenceName::GOLDEN, + nri::VideoAV1ReferenceName::BWDREF, + nri::VideoAV1ReferenceName::ALTREF2, + nri::VideoAV1ReferenceName::ALTREF, + }; + for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { + av1DecodeReferences[i].name = av1ReferenceNames[i]; + av1DecodeReferences[i].refFrameIndex = 0; + av1DecodeReferences[i].frameType = nri::VideoEncodeFrameType::IDR; + av1DecodeReferences[i].orderHint = 0; + av1DecodeReferences[i].frameId = 0; + av1DecodeReferences[i].slot = 0; + av1DecodeReferences[i].savedOrderHints = av1DecodeOrderHints; + } + av1Info.picture.frameType = nri::VideoEncodeFrameType::P; + av1Info.picture.orderHint = 1; + av1Info.picture.refreshFrameFlags = 0x1; + av1Info.picture.primaryReferenceName = nri::VideoAV1ReferenceName::LAST; + av1Info.picture.currentFrameId = 1; + av1Info.picture.flags = nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME; + av1Info.picture.orderHints = av1DecodeOrderHints; + av1Info.picture.references = av1DecodeReferences; + av1Info.picture.referenceNum = helper::GetCountOf(av1ReferenceNames); + } nri::VideoDecodeDesc decodeDesc = {}; decodeDesc.session = m_DecodeSession; decodeDesc.parameters = m_DecodeParameters; decodeDesc.bitstream.buffer = m_DecodeBitstreamBuffer; decodeDesc.bitstream.size = decodeBitstreamSize; - decodeDesc.dstPicture = m_DecodePicture; - decodeDesc.dstSlot = 0; + decodeDesc.dstPicture = av1PFrame ? m_AV1PDecodePicture : m_DecodePicture; + decodeDesc.references = av1PFrame ? &av1DecodeReference : nullptr; + decodeDesc.referenceNum = av1PFrame ? 1u : 0u; + decodeDesc.dstSlot = av1PFrame ? 1u : 0u; decodeDesc.h264PictureDesc = m_Codec == SampleCodec::H264 ? &h264DecodePicture : nullptr; decodeDesc.h265PictureDesc = m_Codec == SampleCodec::H265 ? &h265DecodePicture : nullptr; decodeDesc.av1PictureDesc = av1DecodeInfo ? &av1Info.picture : nullptr; nri::VideoDecodePictureStates decodePictureStates = {}; - if (Video.GetVideoDecodePictureStates(*m_DecodePicture, decodePictureStates) != nri::Result::SUCCESS) { + if (Video.GetVideoDecodePictureStates(*(av1PFrame ? m_AV1PDecodePicture : m_DecodePicture), decodePictureStates) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to query video decode picture states"; return false; } if (!SubmitOneTime(NRI, *m_VideoDecodeQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::TextureBarrierDesc textureBarrier = {}; - textureBarrier.texture = m_DecodeTexture; - textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarrier.after = decodePictureStates.decodeWrite; - textureBarrier.mipNum = nri::REMAINING; - textureBarrier.layerNum = nri::REMAINING; - textureBarrier.planes = nri::PlaneBits::ALL; + nri::TextureBarrierDesc textureBarriers[2] = {}; + textureBarriers[0].texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; + textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[0].after = decodePictureStates.decodeWrite; + textureBarriers[0].mipNum = nri::REMAINING; + textureBarriers[0].layerNum = nri::REMAINING; + textureBarriers[0].planes = nri::PlaneBits::ALL; + textureBarriers[1].texture = m_DecodeTexture; + textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[1].after = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[1].mipNum = nri::REMAINING; + textureBarriers[1].layerNum = nri::REMAINING; + textureBarriers[1].planes = nri::PlaneBits::ALL; nri::BarrierDesc barrierDesc = {}; - barrierDesc.textures = &textureBarrier; - barrierDesc.textureNum = 1; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = av1PFrame ? 2 : 1; NRI.CmdBarrier(commandBuffer, barrierDesc); Video.CmdDecodeVideo(commandBuffer, decodeDesc); if (decodePictureStates.releaseAfterDecode) { - textureBarrier.before = decodePictureStates.decodeWrite; - textureBarrier.after = decodePictureStates.afterDecode; + textureBarriers[0].before = decodePictureStates.decodeWrite; + textureBarriers[0].after = decodePictureStates.afterDecode; + if (av1PFrame) { + textureBarriers[1].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + } NRI.CmdBarrier(commandBuffer, barrierDesc); } })) { @@ -1701,7 +1876,7 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { nri::TextureBarrierDesc textureBarrier = {}; - textureBarrier.texture = m_DecodeTexture; + textureBarrier.texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; textureBarrier.before = decodePictureStates.graphicsBefore; textureBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; textureBarrier.mipNum = nri::REMAINING; @@ -1724,19 +1899,19 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co NRI.CmdBarrier(commandBuffer, copyBarrierDesc); nri::TextureRegionDesc lumaRegion = {}; - lumaRegion.width = VIDEO_WIDTH; - lumaRegion.height = VIDEO_HEIGHT; + lumaRegion.width = (nri::Dim_t)m_VideoWidth; + lumaRegion.height = (nri::Dim_t)m_VideoHeight; lumaRegion.depth = 1; lumaRegion.planes = nri::PlaneBits::PLANE_0; nri::TextureDataLayoutDesc lumaLayout = {}; lumaLayout.rowPitch = m_Nv12Layout.yRowPitchBytes; lumaLayout.slicePitch = m_Nv12Layout.ySlicePitchBytes; - NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, lumaLayout, *m_DecodeTexture, lumaRegion); + NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, lumaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), lumaRegion); nri::TextureRegionDesc chromaRegion = {}; - chromaRegion.width = VIDEO_WIDTH; - chromaRegion.height = VIDEO_HEIGHT; + chromaRegion.width = (nri::Dim_t)m_VideoWidth; + chromaRegion.height = (nri::Dim_t)m_VideoHeight; chromaRegion.depth = 1; chromaRegion.planes = nri::PlaneBits::PLANE_1; @@ -1744,7 +1919,7 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co chromaLayout.offset = m_Nv12Layout.uvOffsetBytes; chromaLayout.rowPitch = m_Nv12Layout.uvRowPitchBytes; chromaLayout.slicePitch = m_Nv12Layout.uvSlicePitchBytes; - NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, chromaLayout, *m_DecodeTexture, chromaRegion); + NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, chromaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), chromaRegion); nv12BufferBarrier.before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; nv12BufferBarrier.after = {nri::AccessBits::NONE, nri::StageBits::NONE}; @@ -1816,7 +1991,10 @@ void Sample::PrepareFrame(uint32_t) { ImGui::SetNextWindowSize({900.0f, 520.0f}, ImGuiCond_Once); ImGui::Begin("NRI Video Encode / Decode"); { - ImGui::Text("Codec: %s, format: NV12, size: %ux%u", GetCodecName(m_Codec), VIDEO_WIDTH, VIDEO_HEIGHT); + ImGui::Text("Codec: %s, format: NV12, size: %ux%u", GetCodecName(m_Codec), m_VideoWidth, m_VideoHeight); + ImGui::Text("CQP: I=%u, P=%u, B=%u%s", m_QpI, m_QpP, m_QpB, m_Codec == SampleCodec::AV1 ? ", AV1 baseQIndex follows below" : ""); + if (m_Codec == SampleCodec::AV1) + ImGui::Text("AV1: frame=%s, baseQIndex=%u", m_AV1FrameArg.c_str(), m_AV1BaseQIndex); ImGui::TextWrapped("Video: %s", m_VideoStatus.c_str()); ImGui::TextWrapped("Preview: %s", m_PreviewStatus.c_str()); ImGui::Text("Encode queue: %s, decode queue: %s", m_VideoEncodeQueue ? "yes" : "no", m_VideoDecodeQueue ? "yes" : "no"); @@ -1829,11 +2007,11 @@ void Sample::PrepareFrame(uint32_t) { if (ImGui::BeginTable("PreviewPanels", 2, ImGuiTableFlags_SizingStretchSame)) { ImGui::TableNextColumn(); float width = std::max(200.0f, ImGui::GetContentRegionAvail().x); - DrawTexturePanel("Generated source", m_SourcePreviewTextureView, {width, width * float(VIDEO_HEIGHT) / float(VIDEO_WIDTH)}); + DrawTexturePanel("Generated source", m_SourcePreviewTextureView, {width, width * float(m_VideoHeight) / float(m_VideoWidth)}); ImGui::TableNextColumn(); width = std::max(200.0f, ImGui::GetContentRegionAvail().x); - DrawTexturePanel(m_DecodePreviewReady ? "Decoded preview" : "Decoded preview pending", m_DecodePreviewReady ? m_DecodePreviewTextureView : nullptr, {width, width * float(VIDEO_HEIGHT) / float(VIDEO_WIDTH)}); + DrawTexturePanel(m_DecodePreviewReady ? "Decoded preview" : "Decoded preview pending", m_DecodePreviewReady ? m_DecodePreviewTextureView : nullptr, {width, width * float(m_VideoHeight) / float(m_VideoWidth)}); ImGui::EndTable(); } } From 91f66f9de1811a9da69d65a4fe7af0192fdc0ec1 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 7 May 2026 15:29:18 +0200 Subject: [PATCH 09/28] Fix D3D12 AV1 decode preview --- Source/VideoEncodeDecode.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 956af94..d19fd7e 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -1671,7 +1671,8 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { nri::VideoAV1EncodeDecodeInfo av1DecodeInfo = {}; if (m_Codec == SampleCodec::AV1) { - const uint8_t* encodedHeader = (const uint8_t*)NRI.MapBuffer(*m_BitstreamHeaderReadbackBuffer, 0, AV1_HEADER_READBACK_SIZE); + const bool needsEncodedHeaderReadback = feedbackResult == nri::Result::UNSUPPORTED; + const uint8_t* encodedHeader = needsEncodedHeaderReadback ? (const uint8_t*)NRI.MapBuffer(*m_BitstreamHeaderReadbackBuffer, 0, AV1_HEADER_READBACK_SIZE) : nullptr; if (!encodedHeader && feedbackResult == nri::Result::UNSUPPORTED) { m_VideoStatus = "Failed to map AV1 encoded header readback"; return false; From e29fbcb4bdf3aa36096a3cb2947e90b1ddebce56 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 7 May 2026 15:39:02 +0200 Subject: [PATCH 10/28] Keep AV1 decode info backend neutral --- Source/VideoEncodeDecode.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index d19fd7e..e6f85b7 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -1671,9 +1671,8 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { nri::VideoAV1EncodeDecodeInfo av1DecodeInfo = {}; if (m_Codec == SampleCodec::AV1) { - const bool needsEncodedHeaderReadback = feedbackResult == nri::Result::UNSUPPORTED; - const uint8_t* encodedHeader = needsEncodedHeaderReadback ? (const uint8_t*)NRI.MapBuffer(*m_BitstreamHeaderReadbackBuffer, 0, AV1_HEADER_READBACK_SIZE) : nullptr; - if (!encodedHeader && feedbackResult == nri::Result::UNSUPPORTED) { + const uint8_t* encodedHeader = (const uint8_t*)NRI.MapBuffer(*m_BitstreamHeaderReadbackBuffer, 0, AV1_HEADER_READBACK_SIZE); + if (!encodedHeader) { m_VideoStatus = "Failed to map AV1 encoded header readback"; return false; } From 43a0e6ff577ed02e19858768400c1d88d129a92c Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 7 May 2026 15:53:09 +0200 Subject: [PATCH 11/28] Use NRI AV1 metadata without header readback --- Source/VideoEncodeDecode.cpp | 53 ++++-------------------------------- 1 file changed, 6 insertions(+), 47 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index e6f85b7..f6ede04 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -22,7 +22,6 @@ constexpr uint32_t DEFAULT_VIDEO_HEIGHT = 1080; constexpr double ROUND_TRIP_INTERVAL_SEC = 1.0 / 60.0; constexpr uint64_t BITSTREAM_SIZE = 2 * 1024 * 1024; constexpr uint64_t ENCODED_SLICE_OFFSET = 4096; -constexpr uint64_t AV1_HEADER_READBACK_SIZE = 4096; constexpr uint64_t METADATA_SIZE = 4 * 1024 * 1024; constexpr uint64_t RESOLVED_METADATA_SIZE = 4096; @@ -303,7 +302,6 @@ class Sample : public SampleBase { nri::DescriptorPool* m_GenerateDescriptorPool = nullptr; nri::DescriptorSet* m_GenerateDescriptorSet = nullptr; nri::Buffer* m_BitstreamHeaderUploadBuffer = nullptr; - nri::Buffer* m_BitstreamHeaderReadbackBuffer = nullptr; nri::Buffer* m_BitstreamBuffer = nullptr; nri::Buffer* m_DecodeBitstreamBuffer = nullptr; nri::Buffer* m_MetadataBuffer = nullptr; @@ -401,8 +399,6 @@ Sample::~Sample() { NRI.DestroyBuffer(m_DecodeBitstreamBuffer); if (m_BitstreamBuffer) NRI.DestroyBuffer(m_BitstreamBuffer); - if (m_BitstreamHeaderReadbackBuffer) - NRI.DestroyBuffer(m_BitstreamHeaderReadbackBuffer); if (m_BitstreamHeaderUploadBuffer) NRI.DestroyBuffer(m_BitstreamHeaderUploadBuffer); if (m_UploadBuffer) @@ -1236,10 +1232,6 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { resolvedMetadataReadbackBufferDesc.size = RESOLVED_METADATA_SIZE; resolvedMetadataReadbackBufferDesc.usage = nri::BufferUsageBits::NONE; - nri::BufferDesc bitstreamHeaderReadbackBufferDesc = {}; - bitstreamHeaderReadbackBufferDesc.size = AV1_HEADER_READBACK_SIZE; - bitstreamHeaderReadbackBufferDesc.usage = nri::BufferUsageBits::NONE; - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_UPLOAD, 0.0f, bitstreamHeaderUploadBufferDesc, m_BitstreamHeaderUploadBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create bitstream header upload buffer"; return; @@ -1250,11 +1242,6 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_READBACK, 0.0f, bitstreamHeaderReadbackBufferDesc, m_BitstreamHeaderReadbackBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create bitstream header readback buffer"; - return; - } - if (CreateDecodeBitstreamBuffer(NRI, *m_Device, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create decode bitstream buffer"; return; @@ -1577,17 +1564,12 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { return false; } - nri::BufferBarrierDesc metadataBarriers[4] = {}; + nri::BufferBarrierDesc metadataBarriers[2] = {}; metadataBarriers[0].buffer = m_ResolvedMetadataBuffer; metadataBarriers[0].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; metadataBarriers[0].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; metadataBarriers[1].buffer = m_ResolvedMetadataReadbackBuffer; metadataBarriers[1].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - metadataBarriers[2].buffer = m_BitstreamBuffer; - metadataBarriers[2].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; - metadataBarriers[2].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; - metadataBarriers[3].buffer = m_BitstreamHeaderReadbackBuffer; - metadataBarriers[3].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; nri::BarrierDesc metadataBarrierDesc = {}; metadataBarrierDesc.buffers = metadataBarriers; @@ -1600,15 +1582,10 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); metadataBarrierDesc.bufferNum = helper::GetCountOf(metadataBarriers); NRI.CmdCopyBuffer(*m_MetadataReadbackCommandBuffer, *m_ResolvedMetadataReadbackBuffer, 0, *m_ResolvedMetadataBuffer, 0, RESOLVED_METADATA_SIZE); - NRI.CmdCopyBuffer(*m_MetadataReadbackCommandBuffer, *m_BitstreamHeaderReadbackBuffer, 0, *m_BitstreamBuffer, ENCODED_SLICE_OFFSET, AV1_HEADER_READBACK_SIZE); metadataBarriers[0].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; metadataBarriers[0].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; metadataBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; metadataBarriers[1].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; - metadataBarriers[2].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; - metadataBarriers[2].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; - metadataBarriers[3].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - metadataBarriers[3].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); if (NRI.EndCommandBuffer(*m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { @@ -1648,17 +1625,11 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { nri::VideoEncodeFeedback feedback = {}; const nri::Result feedbackResult = Video.GetVideoEncodeFeedback(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, feedback); if (feedbackResult != nri::Result::SUCCESS) { - if (feedbackResult == nri::Result::UNSUPPORTED && m_Codec == SampleCodec::AV1) { - feedback.encodedBitstreamOffset = 0; - feedback.encodedBitstreamWrittenBytes = AV1_HEADER_READBACK_SIZE; - feedback.writtenSubregionNum = 1; - } else { - if (feedbackResult == nri::Result::UNSUPPORTED) - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode metadata feedback is unsupported"; - else - m_VideoStatus = "Failed to read resolved encode metadata"; - return false; - } + if (feedbackResult == nri::Result::UNSUPPORTED) + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode metadata feedback is unsupported"; + else + m_VideoStatus = "Failed to read resolved encode metadata"; + return false; } if (feedback.errorFlags || !feedback.encodedBitstreamWrittenBytes) { @@ -1671,26 +1642,14 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { nri::VideoAV1EncodeDecodeInfo av1DecodeInfo = {}; if (m_Codec == SampleCodec::AV1) { - const uint8_t* encodedHeader = (const uint8_t*)NRI.MapBuffer(*m_BitstreamHeaderReadbackBuffer, 0, AV1_HEADER_READBACK_SIZE); - if (!encodedHeader) { - m_VideoStatus = "Failed to map AV1 encoded header readback"; - return false; - } - nri::VideoAV1EncodeDecodeInfoDesc av1InfoDesc = {}; av1InfoDesc.feedback = &feedback; av1InfoDesc.sequence = &m_AV1Sequence; - av1InfoDesc.encodedPayloadHeader = encodedHeader; - av1InfoDesc.encodedPayloadHeaderSize = encodedHeader ? std::min(AV1_HEADER_READBACK_SIZE, feedback.encodedBitstreamWrittenBytes) : 0; const nri::Result av1InfoResult = Video.GetVideoEncodeAV1DecodeInfo(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, av1InfoDesc, av1DecodeInfo); if (av1InfoResult != nri::Result::SUCCESS) { - if (encodedHeader) - NRI.UnmapBuffer(*m_BitstreamHeaderReadbackBuffer); m_VideoStatus = "Failed to prepare AV1 decode metadata"; return false; } - if (encodedHeader) - NRI.UnmapBuffer(*m_BitstreamHeaderReadbackBuffer); feedback.encodedBitstreamWrittenBytes = av1DecodeInfo.bitstreamOffset + av1DecodeInfo.bitstreamSize; } From 119ae0cfd2bab478e0c9ca7e3e0fc8bad036e243 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 21 May 2026 12:05:51 +0200 Subject: [PATCH 12/28] Exercise video EOS helper in encode sample --- Source/VideoEncodeDecode.cpp | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index f6ede04..6bea690 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -261,6 +261,7 @@ class Sample : public SampleBase { PatternConstants MakePatternConstants(PatternOperation operation, float timeSec) const; bool GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage = false); bool WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders); + bool WriteAnnexBEndOfStream(std::vector& annexBEndOfStream); bool TrySubmitEncodeAndMetadataReadback(float timeSec); bool TryDecodePendingMetadata(float timeSec); bool DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo, float timeSec); @@ -1370,6 +1371,29 @@ bool Sample::WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeader return true; } +bool Sample::WriteAnnexBEndOfStream(std::vector& annexBEndOfStream) { + annexBEndOfStream.clear(); + if (m_Codec == SampleCodec::AV1) + return true; + + nri::VideoAnnexBEndOfStreamDesc annexBDesc = {}; + annexBDesc.codec = GetNriCodec(m_Codec); + if (Video.WriteVideoAnnexBEndOfStream(annexBDesc) != nri::Result::SUCCESS || annexBDesc.writtenSize == 0) { + m_VideoStatus = std::string("Failed to query ") + GetCodecName(m_Codec) + " Annex-B end-of-stream size"; + return false; + } + + annexBEndOfStream.resize((size_t)annexBDesc.writtenSize); + annexBDesc.dst = annexBEndOfStream.data(); + annexBDesc.dstSize = annexBEndOfStream.size(); + if (Video.WriteVideoAnnexBEndOfStream(annexBDesc) != nri::Result::SUCCESS) { + m_VideoStatus = std::string("Failed to build ") + GetCodecName(m_Codec) + " Annex-B end-of-stream marker"; + return false; + } + + return true; +} + bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { if (!CanRunRoundTrip()) { m_VideoStatus = std::string(GetCodecName(m_Codec)) + " round trip is not currently supported in this configuration"; @@ -1667,6 +1691,9 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co std::vector annexBHeaders; if (!WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) return false; + std::vector annexBEndOfStream; + if (!WriteAnnexBEndOfStream(annexBEndOfStream)) + return false; const uint64_t encodedPayloadSkip = av1DecodeInfo ? av1DecodeInfo->bitstreamOffset : GetEncodedPayloadHeaderSkip(m_Codec, feedback.encodedBitstreamWrittenBytes); const uint64_t encodedPayloadBytes = av1DecodeInfo ? av1DecodeInfo->bitstreamSize : feedback.encodedBitstreamWrittenBytes - encodedPayloadSkip; @@ -1900,7 +1927,7 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co m_DecodePreviewReady = true; char message[128] = {}; - std::snprintf(message, sizeof(message), "%s encode/decode round trip complete, encoded %llu bytes", GetCodecName(m_Codec), (unsigned long long)feedback.encodedBitstreamWrittenBytes); + std::snprintf(message, sizeof(message), "%s encode/decode round trip complete, encoded %llu bytes, EOS %llu bytes", GetCodecName(m_Codec), (unsigned long long)feedback.encodedBitstreamWrittenBytes, (unsigned long long)annexBEndOfStream.size()); m_VideoStatus = message; return true; } From c57ef4568220660079c1a97cb447f73ba1820ff8 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 21 May 2026 12:27:14 +0200 Subject: [PATCH 13/28] Use video capability helpers in sample --- Source/VideoEncodeDecode.cpp | 62 ++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 6bea690..043d5fb 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -116,6 +116,10 @@ static uint64_t AlignUp(uint64_t value, uint64_t alignment) { return alignment == 0 ? value : ((value + alignment - 1) / alignment) * alignment; } +static bool IsAligned(uint64_t value, uint64_t alignment) { + return alignment == 0 || value % alignment == 0; +} + static Nv12BufferLayout MakeNv12BufferLayout(const nri::DeviceDesc& deviceDesc, uint32_t width, uint32_t height) { const uint32_t rowAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureRow, 1u); const uint32_t sliceAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureSlice, 1u); @@ -922,6 +926,29 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { decodeSessionDesc.usage = nri::VideoUsage::DECODE; decodeSessionDesc.maxReferenceNum = 16; + nri::VideoCapabilities encodeCapabilities = {}; + if (Video.GetVideoCapabilities(*m_Device, encodeSessionDesc, encodeCapabilities) != nri::Result::SUCCESS) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode is unsupported for the requested size"; + return; + } + + nri::VideoCapabilities decodeCapabilities = {}; + if (Video.GetVideoCapabilities(*m_Device, decodeSessionDesc, decodeCapabilities) != nri::Result::SUCCESS) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " decode is unsupported for the requested size"; + return; + } + + const uint64_t encodePayloadCapacity = BITSTREAM_SIZE - ENCODED_SLICE_OFFSET; + if (!IsAligned(ENCODED_SLICE_OFFSET, encodeCapabilities.bitstreamOffsetAlignment) || !IsAligned(encodePayloadCapacity, encodeCapabilities.bitstreamSizeAlignment) || encodePayloadCapacity > encodeCapabilities.bitstreamSizeMax) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode bitstream buffer does not satisfy video capabilities"; + return; + } + + if (BITSTREAM_SIZE > decodeCapabilities.bitstreamSizeMax || !IsAligned(BITSTREAM_SIZE, decodeCapabilities.bitstreamSizeAlignment)) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " decode bitstream buffer does not satisfy video capabilities"; + return; + } + if (NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_ENCODE, 0, m_VideoEncodeQueue) != nri::Result::SUCCESS || NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_DECODE, 0, m_VideoDecodeQueue) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to get video queues"; return; @@ -1526,7 +1553,15 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { encodeDesc.resolvedMetadata = m_ResolvedMetadataBuffer; encodeDesc.av1PictureDesc = m_Codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; + nri::VideoEncodePictureStates encodePictureStates = {}; + if (Video.GetVideoEncodePictureStates(*(av1PFrame ? m_AV1PReconstructedPicture : m_ReconstructedPicture), encodePictureStates) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to query video encode picture states"; + return false; + } + if (!SubmitOneTime(NRI, *m_VideoEncodeQueue, [&](nri::CommandBuffer& commandBuffer) { + const nri::AccessLayoutStage encodeReferenceRead = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + nri::BufferBarrierDesc bufferBarriers[2] = {}; bufferBarriers[0].buffer = m_MetadataBuffer; bufferBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; @@ -1536,19 +1571,19 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { nri::TextureBarrierDesc textureBarriers[3] = {}; textureBarriers[0].texture = m_EncodeTexture; textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[0].after = encodePictureStates.encodeRead; textureBarriers[0].mipNum = nri::REMAINING; textureBarriers[0].layerNum = nri::REMAINING; textureBarriers[0].planes = nri::PlaneBits::ALL; textureBarriers[1].texture = av1PFrame ? m_AV1PReconstructedTexture : m_ReconstructedTexture; textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[1].after = encodePictureStates.encodeWrite; textureBarriers[1].mipNum = nri::REMAINING; textureBarriers[1].layerNum = nri::REMAINING; textureBarriers[1].planes = nri::PlaneBits::ALL; textureBarriers[2].texture = m_ReconstructedTexture; - textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].before = encodePictureStates.releaseAfterEncode ? encodePictureStates.afterEncode : encodePictureStates.encodeWrite; + textureBarriers[2].after = encodeReferenceRead; textureBarriers[2].mipNum = nri::REMAINING; textureBarriers[2].layerNum = nri::REMAINING; textureBarriers[2].planes = nri::PlaneBits::ALL; @@ -1565,14 +1600,21 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { bufferBarriers[0].after = {}; bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; bufferBarriers[1].after = {}; - textureBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[0].before = encodePictureStates.encodeRead; textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[2].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + uint32_t textureBarrierNum = 1; + if (encodePictureStates.releaseAfterEncode) { + textureBarriers[1].before = encodePictureStates.encodeWrite; + textureBarriers[1].after = encodePictureStates.afterEncode; + textureBarrierNum = 2; + if (av1PFrame) { + textureBarriers[2].before = encodeReferenceRead; + textureBarriers[2].after = encodePictureStates.afterEncode; + textureBarrierNum = 3; + } + } barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; + barrierDesc.textureNum = textureBarrierNum; NRI.CmdBarrier(commandBuffer, barrierDesc); })) { m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submission failed"; From 3d698c80cd224dff504cbec43e45d34915d11278 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 28 May 2026 17:47:07 +0200 Subject: [PATCH 14/28] fix(video): stabilize codec sample --- Source/VideoEncodeDecode.cpp | 137 ++++++++++++++--------------------- 1 file changed, 54 insertions(+), 83 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 043d5fb..1f72db4 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -116,10 +116,6 @@ static uint64_t AlignUp(uint64_t value, uint64_t alignment) { return alignment == 0 ? value : ((value + alignment - 1) / alignment) * alignment; } -static bool IsAligned(uint64_t value, uint64_t alignment) { - return alignment == 0 || value % alignment == 0; -} - static Nv12BufferLayout MakeNv12BufferLayout(const nri::DeviceDesc& deviceDesc, uint32_t width, uint32_t height) { const uint32_t rowAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureRow, 1u); const uint32_t sliceAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureSlice, 1u); @@ -259,7 +255,7 @@ class Sample : public SampleBase { private: bool InitializeGraphics(nri::GraphicsAPI graphicsAPI); bool TryInitializePreviewTextures(nri::GraphicsAPI graphicsAPI); - void InitializeGeneratedFrames(float timeSec); + void InitializeGeneratedFrames(float timeSec, bool uploadEncodeTexture = false); bool CanRunRoundTrip() const; void TryInitializeVideo(nri::GraphicsAPI graphicsAPI); PatternConstants MakePatternConstants(PatternOperation operation, float timeSec) const; @@ -472,6 +468,10 @@ void Sample::ReadCmdLine(cmdline::parser& cmdLine) { m_AV1FrameArg = cmdLine.get("av1Frame"); m_VideoWidth = cmdLine.get("videoWidth"); m_VideoHeight = cmdLine.get("videoHeight"); + if (!cmdLine.exist("videoWidth") && cmdLine.exist("width")) + m_VideoWidth = m_OutputResolution.x; + if (!cmdLine.exist("videoHeight") && cmdLine.exist("height")) + m_VideoHeight = m_OutputResolution.y; m_QpI = cmdLine.get("qpI"); m_QpP = cmdLine.get("qpP"); m_QpB = cmdLine.get("qpB"); @@ -615,13 +615,13 @@ PatternConstants Sample::MakePatternConstants(PatternOperation operation, float return patternConstants; } -void Sample::InitializeGeneratedFrames(float timeSec) { +void Sample::InitializeGeneratedFrames(float timeSec, bool uploadEncodeTexture) { if (!m_SourcePreviewTexture || !m_SourcePreviewStorage || !m_UploadBuffer || !m_UploadBufferView) return; PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); - if (m_EncodeTexture) { + if (m_EncodeTexture && uploadEncodeTexture) { if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { m_PreviewStatus = "Failed to generate source pattern via compute"; return; @@ -926,29 +926,6 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { decodeSessionDesc.usage = nri::VideoUsage::DECODE; decodeSessionDesc.maxReferenceNum = 16; - nri::VideoCapabilities encodeCapabilities = {}; - if (Video.GetVideoCapabilities(*m_Device, encodeSessionDesc, encodeCapabilities) != nri::Result::SUCCESS) { - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode is unsupported for the requested size"; - return; - } - - nri::VideoCapabilities decodeCapabilities = {}; - if (Video.GetVideoCapabilities(*m_Device, decodeSessionDesc, decodeCapabilities) != nri::Result::SUCCESS) { - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " decode is unsupported for the requested size"; - return; - } - - const uint64_t encodePayloadCapacity = BITSTREAM_SIZE - ENCODED_SLICE_OFFSET; - if (!IsAligned(ENCODED_SLICE_OFFSET, encodeCapabilities.bitstreamOffsetAlignment) || !IsAligned(encodePayloadCapacity, encodeCapabilities.bitstreamSizeAlignment) || encodePayloadCapacity > encodeCapabilities.bitstreamSizeMax) { - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode bitstream buffer does not satisfy video capabilities"; - return; - } - - if (BITSTREAM_SIZE > decodeCapabilities.bitstreamSizeMax || !IsAligned(BITSTREAM_SIZE, decodeCapabilities.bitstreamSizeAlignment)) { - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " decode bitstream buffer does not satisfy video capabilities"; - return; - } - if (NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_ENCODE, 0, m_VideoEncodeQueue) != nri::Result::SUCCESS || NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_DECODE, 0, m_VideoDecodeQueue) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to get video queues"; return; @@ -1428,7 +1405,7 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { } const bool av1PFrame = m_AV1PFrameVisual && m_AV1PFrameStage == 1; - PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, m_AV1PFrameVisual ? 0.0f : timeSec); + PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { m_VideoStatus = "Failed to generate NV12 source pattern via compute"; return false; @@ -1553,15 +1530,7 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { encodeDesc.resolvedMetadata = m_ResolvedMetadataBuffer; encodeDesc.av1PictureDesc = m_Codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; - nri::VideoEncodePictureStates encodePictureStates = {}; - if (Video.GetVideoEncodePictureStates(*(av1PFrame ? m_AV1PReconstructedPicture : m_ReconstructedPicture), encodePictureStates) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to query video encode picture states"; - return false; - } - if (!SubmitOneTime(NRI, *m_VideoEncodeQueue, [&](nri::CommandBuffer& commandBuffer) { - const nri::AccessLayoutStage encodeReferenceRead = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - nri::BufferBarrierDesc bufferBarriers[2] = {}; bufferBarriers[0].buffer = m_MetadataBuffer; bufferBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; @@ -1571,19 +1540,19 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { nri::TextureBarrierDesc textureBarriers[3] = {}; textureBarriers[0].texture = m_EncodeTexture; textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[0].after = encodePictureStates.encodeRead; + textureBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; textureBarriers[0].mipNum = nri::REMAINING; textureBarriers[0].layerNum = nri::REMAINING; textureBarriers[0].planes = nri::PlaneBits::ALL; textureBarriers[1].texture = av1PFrame ? m_AV1PReconstructedTexture : m_ReconstructedTexture; textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].after = encodePictureStates.encodeWrite; + textureBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; textureBarriers[1].mipNum = nri::REMAINING; textureBarriers[1].layerNum = nri::REMAINING; textureBarriers[1].planes = nri::PlaneBits::ALL; textureBarriers[2].texture = m_ReconstructedTexture; - textureBarriers[2].before = encodePictureStates.releaseAfterEncode ? encodePictureStates.afterEncode : encodePictureStates.encodeWrite; - textureBarriers[2].after = encodeReferenceRead; + textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; textureBarriers[2].mipNum = nri::REMAINING; textureBarriers[2].layerNum = nri::REMAINING; textureBarriers[2].planes = nri::PlaneBits::ALL; @@ -1600,21 +1569,14 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { bufferBarriers[0].after = {}; bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; bufferBarriers[1].after = {}; - textureBarriers[0].before = encodePictureStates.encodeRead; + textureBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - uint32_t textureBarrierNum = 1; - if (encodePictureStates.releaseAfterEncode) { - textureBarriers[1].before = encodePictureStates.encodeWrite; - textureBarriers[1].after = encodePictureStates.afterEncode; - textureBarrierNum = 2; - if (av1PFrame) { - textureBarriers[2].before = encodeReferenceRead; - textureBarriers[2].after = encodePictureStates.afterEncode; - textureBarrierNum = 3; - } - } + textureBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = textureBarrierNum; + barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; NRI.CmdBarrier(commandBuffer, barrierDesc); })) { m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submission failed"; @@ -1711,6 +1673,28 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { nri::VideoAV1EncodeDecodeInfoDesc av1InfoDesc = {}; av1InfoDesc.feedback = &feedback; av1InfoDesc.sequence = &m_AV1Sequence; + nri::VideoAV1ReferenceDesc av1InfoReferences[8] = {}; + if (m_AV1PFrameVisual && m_AV1PFrameStage == 1) { + const nri::VideoAV1ReferenceName av1ReferenceNames[] = { + nri::VideoAV1ReferenceName::LAST, + nri::VideoAV1ReferenceName::LAST2, + nri::VideoAV1ReferenceName::LAST3, + nri::VideoAV1ReferenceName::GOLDEN, + nri::VideoAV1ReferenceName::BWDREF, + nri::VideoAV1ReferenceName::ALTREF2, + nri::VideoAV1ReferenceName::ALTREF, + }; + for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { + av1InfoReferences[i].name = av1ReferenceNames[i]; + av1InfoReferences[i].refFrameIndex = 0; + av1InfoReferences[i].frameType = nri::VideoEncodeFrameType::IDR; + av1InfoReferences[i].orderHint = 0; + av1InfoReferences[i].frameId = 0; + av1InfoReferences[i].slot = 0; + } + av1InfoDesc.references = av1InfoReferences; + av1InfoDesc.referenceNum = helper::GetCountOf(av1ReferenceNames); + } const nri::Result av1InfoResult = Video.GetVideoEncodeAV1DecodeInfo(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, av1InfoDesc, av1DecodeInfo); if (av1InfoResult != nri::Result::SUCCESS) { m_VideoStatus = "Failed to prepare AV1 decode metadata"; @@ -1747,35 +1731,22 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co return false; } - if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { - NRI.CmdZeroBuffer(commandBuffer, *m_DecodeBitstreamBuffer, 0, BITSTREAM_SIZE); - if (!annexBHeaders.empty()) - NRI.CmdCopyBuffer(commandBuffer, *m_DecodeBitstreamBuffer, 0, *m_BitstreamHeaderUploadBuffer, 0, annexBHeaders.size()); - - nri::BufferBarrierDesc bufferBarriers[2] = {}; - bufferBarriers[0].buffer = m_BitstreamBuffer; - bufferBarriers[0].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; - bufferBarriers[0].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; - bufferBarriers[1].buffer = m_DecodeBitstreamBuffer; - bufferBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - bufferBarriers[1].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - - nri::BarrierDesc barrierDesc = {}; - barrierDesc.buffers = bufferBarriers; - barrierDesc.bufferNum = helper::GetCountOf(bufferBarriers); - NRI.CmdBarrier(commandBuffer, barrierDesc); - - NRI.CmdCopyBuffer(commandBuffer, *m_DecodeBitstreamBuffer, decodeSliceOffset, *m_BitstreamBuffer, encodedSourceOffset, encodedPayloadBytes); - - bufferBarriers[0].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; - bufferBarriers[0].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; - bufferBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - bufferBarriers[1].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; - NRI.CmdBarrier(commandBuffer, barrierDesc); - })) { - m_VideoStatus = std::string("Failed to build exact ") + GetCodecName(m_Codec) + " decode bitstream"; + const uint8_t* encodedPayload = (const uint8_t*)NRI.MapBuffer(*m_BitstreamBuffer, encodedSourceOffset, encodedPayloadBytes); + uint8_t* decodeBitstream = (uint8_t*)NRI.MapBuffer(*m_DecodeBitstreamBuffer, 0, decodeBitstreamSize); + if (!encodedPayload || !decodeBitstream) { + if (encodedPayload) + NRI.UnmapBuffer(*m_BitstreamBuffer); + if (decodeBitstream) + NRI.UnmapBuffer(*m_DecodeBitstreamBuffer); + m_VideoStatus = std::string("Failed to map exact ") + GetCodecName(m_Codec) + " decode bitstream"; return false; } + std::memset(decodeBitstream, 0, (size_t)decodeBitstreamSize); + if (!annexBHeaders.empty()) + std::memcpy(decodeBitstream, annexBHeaders.data(), annexBHeaders.size()); + std::memcpy(decodeBitstream + decodeSliceOffset, encodedPayload, (size_t)encodedPayloadBytes); + NRI.UnmapBuffer(*m_BitstreamBuffer); + NRI.UnmapBuffer(*m_DecodeBitstreamBuffer); const uint32_t pictureOffsets[] = {(uint32_t)decodeSliceOffset}; From cfe03bff023990c58b3e7bc3e8352f162f764712 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Thu, 28 May 2026 18:03:57 +0200 Subject: [PATCH 15/28] fix(video): refresh AV1 P preview --- Source/VideoEncodeDecode.cpp | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 1f72db4..979d897 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -1706,7 +1706,6 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { const bool decoded = DecodeEncodedBitstream(feedback, m_Codec == SampleCodec::AV1 ? &av1DecodeInfo : nullptr, timeSec); if (decoded && m_AV1PFrameVisual && m_AV1PFrameStage == 0) { m_AV1PFrameStage = 1; - m_DecodePreviewReady = false; return TrySubmitEncodeAndMetadataReadback(timeSec); } @@ -1784,39 +1783,16 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co av1Info.tileLayout.miRowStarts = av1Info.miRowStarts; av1Info.tileLayout.widthInSuperblocksMinus1 = av1Info.widthInSuperblocksMinus1; av1Info.tileLayout.heightInSuperblocksMinus1 = av1Info.heightInSuperblocksMinus1; + if (av1Info.picture.references && av1Info.picture.referenceNum) + av1Info.picture.references = av1Info.references; } nri::VideoReference av1DecodeReference = {m_DecodePicture, 0}; - nri::VideoAV1ReferenceDesc av1DecodeReferences[8] = {}; uint8_t av1DecodeOrderHints[8] = {}; const bool av1PFrame = m_AV1PFrameVisual && m_AV1PFrameStage == 1; if (av1PFrame) { - const nri::VideoAV1ReferenceName av1ReferenceNames[] = { - nri::VideoAV1ReferenceName::LAST, - nri::VideoAV1ReferenceName::LAST2, - nri::VideoAV1ReferenceName::LAST3, - nri::VideoAV1ReferenceName::GOLDEN, - nri::VideoAV1ReferenceName::BWDREF, - nri::VideoAV1ReferenceName::ALTREF2, - nri::VideoAV1ReferenceName::ALTREF, - }; - for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { - av1DecodeReferences[i].name = av1ReferenceNames[i]; - av1DecodeReferences[i].refFrameIndex = 0; - av1DecodeReferences[i].frameType = nri::VideoEncodeFrameType::IDR; - av1DecodeReferences[i].orderHint = 0; - av1DecodeReferences[i].frameId = 0; - av1DecodeReferences[i].slot = 0; - av1DecodeReferences[i].savedOrderHints = av1DecodeOrderHints; - } - av1Info.picture.frameType = nri::VideoEncodeFrameType::P; - av1Info.picture.orderHint = 1; - av1Info.picture.refreshFrameFlags = 0x1; - av1Info.picture.primaryReferenceName = nri::VideoAV1ReferenceName::LAST; - av1Info.picture.currentFrameId = 1; - av1Info.picture.flags = nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME; av1Info.picture.orderHints = av1DecodeOrderHints; - av1Info.picture.references = av1DecodeReferences; - av1Info.picture.referenceNum = helper::GetCountOf(av1ReferenceNames); + for (uint32_t i = 0; i < av1Info.picture.referenceNum && i < helper::GetCountOf(av1Info.references); i++) + av1Info.references[i].savedOrderHints = av1DecodeOrderHints; } nri::VideoDecodeDesc decodeDesc = {}; @@ -1939,6 +1915,9 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co } m_DecodePreviewReady = true; + if (av1PFrame) + m_AV1PFrameStage = 0; + char message[128] = {}; std::snprintf(message, sizeof(message), "%s encode/decode round trip complete, encoded %llu bytes, EOS %llu bytes", GetCodecName(m_Codec), (unsigned long long)feedback.encodedBitstreamWrittenBytes, (unsigned long long)annexBEndOfStream.size()); m_VideoStatus = message; From fb83ed429a53f22352622e67d3488a1a945c3fd3 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sat, 13 Jun 2026 19:17:21 +0200 Subject: [PATCH 16/28] Fix Vulkan video sample decode preview --- Source/VideoEncodeDecode.cpp | 317 +++++++++++++++++++++++------------ 1 file changed, 214 insertions(+), 103 deletions(-) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 979d897..496c2ed 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -36,6 +36,12 @@ enum class SampleCodec : uint8_t { AV1, }; +struct Av1SequenceOptions { + bool enableCdef = true; + bool enableRestoration = true; + uint8_t seqForceScreenContentTools = 2; +}; + static const char* GetCodecName(SampleCodec codec) { switch (codec) { case SampleCodec::H265: @@ -60,24 +66,33 @@ static nri::VideoCodec GetNriCodec(SampleCodec codec) { } } -static uint64_t GetEncodedPayloadHeaderSkip(SampleCodec codec, uint64_t encodedBitstreamBytes) { - const uint64_t headerSize = codec == SampleCodec::H264 ? 1 : 0; - return std::min(headerSize, encodedBitstreamBytes); +static uint8_t GetAv1FrameSizeBitsMinus1(uint32_t value) { + uint32_t bits = 0; + uint32_t maxValue = value ? value - 1 : 0; + do { + bits++; + maxValue >>= 1; + } while (maxValue); + return (uint8_t)(bits - 1); } -static nri::VideoAV1SequenceDesc MakeAV1SequenceDesc(uint32_t width, uint32_t height) { +static nri::VideoAV1SequenceDesc MakeAV1SequenceDesc(uint32_t width, uint32_t height, const Av1SequenceOptions& options) { nri::VideoAV1SequenceDesc desc = {}; - desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | nri::VideoAV1SequenceBits::ENABLE_CDEF | nri::VideoAV1SequenceBits::ENABLE_RESTORATION | nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; + desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; + if (options.enableCdef) + desc.flags |= nri::VideoAV1SequenceBits::ENABLE_CDEF; + if (options.enableRestoration) + desc.flags |= nri::VideoAV1SequenceBits::ENABLE_RESTORATION; desc.bitDepth = 8; desc.subsamplingX = 1; desc.subsamplingY = 1; desc.maxFrameWidthMinus1 = (uint16_t)(width - 1); desc.maxFrameHeightMinus1 = (uint16_t)(height - 1); - desc.frameWidthBitsMinus1 = 15; - desc.frameHeightBitsMinus1 = 15; + desc.frameWidthBitsMinus1 = GetAv1FrameSizeBitsMinus1(width); + desc.frameHeightBitsMinus1 = GetAv1FrameSizeBitsMinus1(height); desc.orderHintBitsMinus1 = 7; desc.seqForceIntegerMv = 2; - desc.seqForceScreenContentTools = 2; + desc.seqForceScreenContentTools = options.seqForceScreenContentTools; desc.colorPrimaries = 1; desc.transferCharacteristics = 1; desc.matrixCoefficients = 1; @@ -259,7 +274,7 @@ class Sample : public SampleBase { bool CanRunRoundTrip() const; void TryInitializeVideo(nri::GraphicsAPI graphicsAPI); PatternConstants MakePatternConstants(PatternOperation operation, float timeSec) const; - bool GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage = false); + bool GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage = false, nri::AccessStage uploadBufferBefore = {}); bool WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders); bool WriteAnnexBEndOfStream(std::vector& annexBEndOfStream); bool TrySubmitEncodeAndMetadataReadback(float timeSec); @@ -328,6 +343,10 @@ class Sample : public SampleBase { std::string m_AV1FrameArg = "IDR"; uint32_t m_VideoWidth = DEFAULT_VIDEO_WIDTH; uint32_t m_VideoHeight = DEFAULT_VIDEO_HEIGHT; + uint32_t m_CodedVideoWidth = DEFAULT_VIDEO_WIDTH; + uint32_t m_CodedVideoHeight = DEFAULT_VIDEO_HEIGHT; + uint32_t m_DecodeBitstreamSizeAlignment = 256; + uint32_t m_DecodeFrameIndex = 0; uint32_t m_QpI = 20; uint32_t m_QpP = 22; uint32_t m_QpB = 24; @@ -340,11 +359,12 @@ class Sample : public SampleBase { nri::VideoH265PictureParameterSetDesc m_H265Pps = {}; nri::VideoAV1SequenceDesc m_AV1Sequence = {}; double m_StartTimeSec = 0.0; - double m_LastRoundTripTimeSec = -1.0; + double m_LastRoundTripTimeSec = -ROUND_TRIP_INTERVAL_SEC; bool m_VideoReady = false; bool m_VideoQueuesRequested = false; bool m_DecodePreviewReady = false; bool m_PreviewTexturesShaderReadable = false; + bool m_SourcePreviewReady = false; bool m_MetadataReadbackPending = false; bool m_AV1PFrameVisual = false; uint32_t m_AV1PFrameStage = 0; @@ -512,9 +532,7 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { NRI_ABORT_ON_FAILURE(nri::nriEnumerateAdapters(adapterDesc, adapterDescsNum)); const nri::AdapterDesc& selectedAdapter = adapterDesc[std::min(m_AdapterIndex, adapterDescsNum - 1)]; - m_VideoQueuesRequested = graphicsAPI != nri::GraphicsAPI::D3D11 && - selectedAdapter.queueNum[(uint32_t)nri::QueueType::VIDEO_ENCODE] && - selectedAdapter.queueNum[(uint32_t)nri::QueueType::VIDEO_DECODE]; + m_VideoQueuesRequested = graphicsAPI != nri::GraphicsAPI::D3D11 && selectedAdapter.queueNum[(uint32_t)nri::QueueType::VIDEO_ENCODE] && selectedAdapter.queueNum[(uint32_t)nri::QueueType::VIDEO_DECODE]; nri::QueueFamilyDesc queueFamilies[3] = {}; uint32_t queueFamilyNum = 0; @@ -533,7 +551,8 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { deviceCreationDesc.enableGraphicsAPIValidation = m_DebugAPI; deviceCreationDesc.enableNRIValidation = m_DebugNRI; deviceCreationDesc.enableD3D11CommandBufferEmulation = D3D11_ENABLE_COMMAND_BUFFER_EMULATION; - deviceCreationDesc.disableD3D12EnhancedBarriers = D3D12_DISABLE_ENHANCED_BARRIERS; + // D3D12 video encode command lists fail to close with enhanced barriers enabled. + deviceCreationDesc.disableD3D12EnhancedBarriers = graphicsAPI == nri::GraphicsAPI::D3D12 ? true : D3D12_DISABLE_ENHANCED_BARRIERS; deviceCreationDesc.vkBindingOffsets = VK_BINDING_OFFSETS; deviceCreationDesc.adapterDesc = &selectedAdapter; deviceCreationDesc.allocationCallbacks = m_AllocationCallbacks; @@ -545,7 +564,6 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::HelperInterface), (nri::HelperInterface*)&NRI)); NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::StreamerInterface), (nri::StreamerInterface*)&NRI)); NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::SwapChainInterface), (nri::SwapChainInterface*)&NRI)); - m_Nv12Layout = MakeNv12BufferLayout(NRI.GetDeviceDesc(*m_Device), m_VideoWidth, m_VideoHeight); nri::StreamerDesc streamerDesc = {}; @@ -594,7 +612,7 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { swapChainTexture.attachmentFormat = m_SwapChainFormat; } - m_QueuedFrames.resize(GetQueuedFrameNum()); + m_QueuedFrames.resize(GetQueuedFrameNum() + swapChainTextureNum); for (QueuedFrame& queuedFrame : m_QueuedFrames) { NRI_ABORT_ON_FAILURE(NRI.CreateCommandAllocator(*m_GraphicsQueue, queuedFrame.commandAllocator)); NRI_ABORT_ON_FAILURE(NRI.CreateCommandBuffer(*queuedFrame.commandAllocator, queuedFrame.commandBuffer)); @@ -632,6 +650,7 @@ void Sample::InitializeGeneratedFrames(float timeSec, bool uploadEncodeTexture) return; } m_PreviewStatus = "Source preview is generated by compute"; + m_SourcePreviewReady = true; return; } @@ -641,9 +660,10 @@ void Sample::InitializeGeneratedFrames(float timeSec, bool uploadEncodeTexture) } m_PreviewStatus = "Source preview is generated by compute"; + m_SourcePreviewReady = true; } -bool Sample::GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage) { +bool Sample::GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage, nri::AccessStage uploadBufferBefore) { if (!m_GeneratePipelineLayout || !m_GenerateComputePipeline || !m_GenerateDescriptorSet || !m_UploadBufferView || !previewTexture) return false; if ((constants.width % 4) != 0 || (constants.height % 2) != 0) @@ -657,12 +677,12 @@ bool Sample::GeneratePatternWithCompute(const PatternConstants& constants, nri:: NRI, *m_GraphicsQueue, m_GenerateDescriptorPool, - [this, &constants, dispatchX, dispatchY, previewTexturesShaderReadable, returnSourceBufferToShaderStorage](nri::CommandBuffer& commandBuffer) { + [this, &constants, dispatchX, dispatchY, previewTexturesShaderReadable, returnSourceBufferToShaderStorage, uploadBufferBefore](nri::CommandBuffer& commandBuffer) { nri::SetDescriptorSetDesc descriptorSet = {0, m_GenerateDescriptorSet}; nri::BufferBarrierDesc bufferBarrier = {}; bufferBarrier.buffer = m_UploadBuffer; - bufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + bufferBarrier.before = uploadBufferBefore; bufferBarrier.after = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; nri::TextureBarrierDesc textureBarriers[2] = {}; @@ -915,7 +935,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { } nri::VideoSessionDesc encodeSessionDesc = {}; - encodeSessionDesc.usage = nri::VideoUsage::ENCODE; + encodeSessionDesc.type = nri::VideoSessionType::ENCODE; encodeSessionDesc.codec = GetNriCodec(m_Codec); encodeSessionDesc.format = nri::Format::NV12_UNORM; encodeSessionDesc.width = m_VideoWidth; @@ -923,8 +943,39 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { encodeSessionDesc.maxReferenceNum = 1; nri::VideoSessionDesc decodeSessionDesc = encodeSessionDesc; - decodeSessionDesc.usage = nri::VideoUsage::DECODE; - decodeSessionDesc.maxReferenceNum = 16; + decodeSessionDesc.type = nri::VideoSessionType::DECODE; + decodeSessionDesc.maxReferenceNum = m_Codec == SampleCodec::AV1 ? 1 : 16; + + nri::VideoCapabilities encodeCapabilities = {}; + nri::VideoCapabilities decodeCapabilities = {}; + nri::Result encodeCapabilitiesResult = Video.GetVideoCapabilities(*m_Device, encodeSessionDesc, encodeCapabilities); + nri::Result decodeCapabilitiesResult = Video.GetVideoCapabilities(*m_Device, decodeSessionDesc, decodeCapabilities); + const bool hasVideoGranularity = encodeCapabilities.pictureAccessGranularityWidth && encodeCapabilities.pictureAccessGranularityHeight && decodeCapabilities.pictureAccessGranularityWidth && decodeCapabilities.pictureAccessGranularityHeight; + if (!hasVideoGranularity) { + m_VideoStatus = std::string("Failed to query ") + GetCodecName(m_Codec) + " video capabilities"; + return; + } + + const uint32_t codedWidthAlignment = std::max({encodeCapabilities.pictureAccessGranularityWidth, decodeCapabilities.pictureAccessGranularityWidth, 1u}); + const uint32_t codedHeightAlignment = std::max({encodeCapabilities.pictureAccessGranularityHeight, decodeCapabilities.pictureAccessGranularityHeight, 1u}); + m_DecodeBitstreamSizeAlignment = std::max(decodeCapabilities.bitstreamSizeAlignment, 1u); + m_CodedVideoWidth = (uint32_t)AlignUp(std::max({m_VideoWidth, encodeCapabilities.widthMin, decodeCapabilities.widthMin}), codedWidthAlignment); + m_CodedVideoHeight = (uint32_t)AlignUp(std::max({m_VideoHeight, encodeCapabilities.heightMin, decodeCapabilities.heightMin}), codedHeightAlignment); + if ((encodeCapabilities.widthMax && m_CodedVideoWidth > encodeCapabilities.widthMax) || (decodeCapabilities.widthMax && m_CodedVideoWidth > decodeCapabilities.widthMax) || (encodeCapabilities.heightMax && m_CodedVideoHeight > encodeCapabilities.heightMax) || (decodeCapabilities.heightMax && m_CodedVideoHeight > decodeCapabilities.heightMax)) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " coded video size exceeds backend capabilities after granularity alignment"; + return; + } + + encodeSessionDesc.width = m_CodedVideoWidth; + encodeSessionDesc.height = m_CodedVideoHeight; + decodeSessionDesc.width = m_CodedVideoWidth; + decodeSessionDesc.height = m_CodedVideoHeight; + encodeCapabilitiesResult = Video.GetVideoCapabilities(*m_Device, encodeSessionDesc, encodeCapabilities); + decodeCapabilitiesResult = Video.GetVideoCapabilities(*m_Device, decodeSessionDesc, decodeCapabilities); + if (encodeCapabilitiesResult != nri::Result::SUCCESS || decodeCapabilitiesResult != nri::Result::SUCCESS) { + m_VideoStatus = std::string(GetCodecName(m_Codec)) + " coded video size is unsupported"; + return; + } if (NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_ENCODE, 0, m_VideoEncodeQueue) != nri::Result::SUCCESS || NRI.GetQueue(*m_Device, nri::QueueType::VIDEO_DECODE, 0, m_VideoDecodeQueue) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to get video queues"; @@ -951,8 +1002,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { sps.pictureOrderCountType = 0; sps.log2MaxPictureOrderCountLsbMinus4 = 0; sps.referenceFrameNum = 1; - sps.pictureWidthInMbsMinus1 = (uint16_t)((m_VideoWidth + 15) / 16 - 1); - sps.pictureHeightInMapUnitsMinus1 = (uint16_t)((m_VideoHeight + 15) / 16 - 1); + sps.pictureWidthInMbsMinus1 = (uint16_t)((m_CodedVideoWidth + 15) / 16 - 1); + sps.pictureHeightInMapUnitsMinus1 = (uint16_t)((m_CodedVideoHeight + 15) / 16 - 1); nri::VideoH264PictureParameterSetDesc pps = {}; pps.flags = nri::VideoH264PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; @@ -987,8 +1038,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { h265Sps.maxSubLayersMinus1 = vps.maxSubLayersMinus1; h265Sps.sequenceParameterSetId = 0; h265Sps.chromaFormatIdc = 1; - h265Sps.pictureWidthInLumaSamples = m_VideoWidth; - h265Sps.pictureHeightInLumaSamples = m_VideoHeight; + h265Sps.pictureWidthInLumaSamples = m_CodedVideoWidth; + h265Sps.pictureHeightInLumaSamples = m_CodedVideoHeight; h265Sps.log2MaxPictureOrderCountLsbMinus4 = 3; h265Sps.log2MinLumaCodingBlockSizeMinus3 = 0; h265Sps.log2DiffMaxMinLumaCodingBlockSize = 2; @@ -1019,7 +1070,13 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { h265Parameters.maxSequenceParameterSetNum = 1; h265Parameters.maxPictureParameterSetNum = 1; - m_AV1Sequence = MakeAV1SequenceDesc(m_VideoWidth, m_VideoHeight); + Av1SequenceOptions av1SequenceOptions = {}; + if (graphicsAPI == nri::GraphicsAPI::VK) { + av1SequenceOptions.enableCdef = false; + av1SequenceOptions.enableRestoration = false; + av1SequenceOptions.seqForceScreenContentTools = 0; + } + m_AV1Sequence = MakeAV1SequenceDesc(m_CodedVideoWidth, m_CodedVideoHeight, av1SequenceOptions); nri::VideoAV1SessionParametersDesc av1Parameters = {}; av1Parameters.sequence = m_AV1Sequence; @@ -1049,8 +1106,8 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { encodeTextureDesc.type = nri::TextureType::TEXTURE_2D; encodeTextureDesc.usage = nri::TextureUsageBits::VIDEO_ENCODE; encodeTextureDesc.format = nri::Format::NV12_UNORM; - encodeTextureDesc.width = (nri::Dim_t)m_VideoWidth; - encodeTextureDesc.height = (nri::Dim_t)m_VideoHeight; + encodeTextureDesc.width = (nri::Dim_t)m_CodedVideoWidth; + encodeTextureDesc.height = (nri::Dim_t)m_CodedVideoHeight; encodeTextureDesc.mipNum = 1; encodeTextureDesc.layerNum = 1; encodeTextureDesc.videoCodec = GetNriCodec(m_Codec); @@ -1094,11 +1151,14 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { nri::TextureBarrierDesc textureBarriers[5] = {}; - textureBarriers[0].texture = m_EncodeTexture; - textureBarriers[1].texture = m_ReconstructedTexture; - textureBarriers[2].texture = m_DecodeTexture; - textureBarriers[3].texture = m_AV1PReconstructedTexture; - textureBarriers[4].texture = m_AV1PDecodeTexture; + uint32_t textureBarrierNum = 0; + textureBarriers[textureBarrierNum++].texture = m_EncodeTexture; + textureBarriers[textureBarrierNum++].texture = m_ReconstructedTexture; + textureBarriers[textureBarrierNum++].texture = m_DecodeTexture; + if (m_AV1PReconstructedTexture) + textureBarriers[textureBarrierNum++].texture = m_AV1PReconstructedTexture; + if (m_AV1PDecodeTexture) + textureBarriers[textureBarrierNum++].texture = m_AV1PDecodeTexture; for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { if (!textureBarrier.texture) @@ -1112,7 +1172,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { nri::BarrierDesc barrierDesc = {}; barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = m_AV1PFrameVisual ? helper::GetCountOf(textureBarriers) : 3; + barrierDesc.textureNum = textureBarrierNum; NRI.CmdBarrier(commandBuffer, barrierDesc); })) { m_VideoStatus = "Failed to initialize video texture layouts"; @@ -1251,7 +1311,6 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { m_VideoStatus = "Failed to create decode bitstream buffer"; return; } - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, metadataBufferDesc, m_MetadataBuffer) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to create encode metadata buffer"; return; @@ -1280,16 +1339,14 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { nri::VideoPictureDesc encodePictureDesc = {}; encodePictureDesc.texture = m_EncodeTexture; encodePictureDesc.usage = nri::VideoPictureUsage::ENCODE_INPUT; - encodePictureDesc.format = nri::Format::NV12_UNORM; - encodePictureDesc.width = m_VideoWidth; - encodePictureDesc.height = m_VideoHeight; + encodePictureDesc.width = (nri::Dim_t)m_CodedVideoWidth; + encodePictureDesc.height = (nri::Dim_t)m_CodedVideoHeight; nri::VideoPictureDesc decodePictureDesc = {}; decodePictureDesc.texture = m_DecodeTexture; decodePictureDesc.usage = nri::VideoPictureUsage::DECODE_OUTPUT; - decodePictureDesc.format = nri::Format::NV12_UNORM; - decodePictureDesc.width = m_VideoWidth; - decodePictureDesc.height = m_VideoHeight; + decodePictureDesc.width = (nri::Dim_t)m_CodedVideoWidth; + decodePictureDesc.height = (nri::Dim_t)m_CodedVideoHeight; nri::VideoPictureDesc reconstructedPictureDesc = encodePictureDesc; reconstructedPictureDesc.texture = m_ReconstructedTexture; @@ -1415,7 +1472,6 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { m_VideoStatus = "Failed to upload NV12 source to video texture"; return false; } - std::vector annexBHeaders; if (!WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) return false; @@ -1434,10 +1490,10 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { pictureDesc.pictureOrderCount = av1PFrame ? 1 : 0; pictureDesc.idrPictureId = av1PFrame ? 0 : 1; - uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((m_VideoWidth + 7) >> 3))}; - uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((m_VideoHeight + 7) >> 3))}; - uint16_t av1WidthInSuperblocksMinus1[] = {(uint16_t)(((m_VideoWidth + 63) / 64) - 1)}; - uint16_t av1HeightInSuperblocksMinus1[] = {(uint16_t)(((m_VideoHeight + 63) / 64) - 1)}; + uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((m_CodedVideoWidth + 7) >> 3))}; + uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((m_CodedVideoHeight + 7) >> 3))}; + uint16_t av1WidthInSuperblocksMinus1[] = {(uint16_t)(((m_CodedVideoWidth + 63) / 64) - 1)}; + uint16_t av1HeightInSuperblocksMinus1[] = {(uint16_t)(((m_CodedVideoHeight + 63) / 64) - 1)}; nri::VideoAV1TileLayoutDesc av1TileLayout = {}; av1TileLayout.columnNum = 1; av1TileLayout.rowNum = 1; @@ -1464,19 +1520,22 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { av1PictureDesc.orderHint = av1PFrame ? 1 : 0; av1PictureDesc.refreshFrameFlags = av1PFrame ? 0x1 : 0xFF; av1PictureDesc.primaryReferenceName = av1PFrame ? nri::VideoAV1ReferenceName::LAST : nri::VideoAV1ReferenceName::NONE; - av1PictureDesc.flags = av1PFrame ? nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME - : nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE | nri::VideoAV1PictureBits::DISABLE_CDF_UPDATE | nri::VideoAV1PictureBits::ALLOW_SCREEN_CONTENT_TOOLS | nri::VideoAV1PictureBits::FORCE_INTEGER_MV | nri::VideoAV1PictureBits::SHOW_FRAME | nri::VideoAV1PictureBits::SHOWABLE_FRAME; - av1PictureDesc.renderWidthMinus1 = (uint16_t)(m_VideoWidth - 1); - av1PictureDesc.renderHeightMinus1 = (uint16_t)(m_VideoHeight - 1); + nri::VideoAV1PictureBits av1CommonPictureFlags = nri::VideoAV1PictureBits::SHOW_FRAME; + if (m_GraphicsAPI == nri::GraphicsAPI::D3D12) + av1CommonPictureFlags |= nri::VideoAV1PictureBits::SEGMENTATION_ENABLED; + av1PictureDesc.flags = av1PFrame ? av1CommonPictureFlags + : av1CommonPictureFlags | nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE; + av1PictureDesc.renderWidthMinus1 = (uint16_t)((m_GraphicsAPI == nri::GraphicsAPI::VK ? m_CodedVideoWidth : m_VideoWidth) - 1); + av1PictureDesc.renderHeightMinus1 = (uint16_t)((m_GraphicsAPI == nri::GraphicsAPI::VK ? m_CodedVideoHeight : m_VideoHeight) - 1); av1PictureDesc.baseQIndex = (uint8_t)m_AV1BaseQIndex; av1PictureDesc.interpolationFilter = 0; av1PictureDesc.txMode = 2; av1PictureDesc.cdefDampingMinus3 = 3; - av1PictureDesc.tileLayout = &av1TileLayout; - av1PictureDesc.loopFilter = &av1LoopFilter; - av1PictureDesc.cdef = &av1Cdef; - av1PictureDesc.loopRestoration = &av1LoopRestoration; - av1PictureDesc.globalMotion = &av1GlobalMotion; + av1PictureDesc.tileLayout = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1TileLayout; + av1PictureDesc.loopFilter = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1LoopFilter; + av1PictureDesc.cdef = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1Cdef; + av1PictureDesc.loopRestoration = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1LoopRestoration; + av1PictureDesc.globalMotion = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1GlobalMotion; nri::VideoReference av1Reference = {m_ReconstructedPicture, 0}; nri::VideoAV1ReferenceDesc av1References[8] = {}; if (av1PFrame) { @@ -1527,31 +1586,38 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { encodeDesc.reconstructedSlot = 1; } encodeDesc.metadata = m_MetadataBuffer; - encodeDesc.resolvedMetadata = m_ResolvedMetadataBuffer; + encodeDesc.resolvedMetadata = m_GraphicsAPI == nri::GraphicsAPI::VK ? m_ResolvedMetadataReadbackBuffer : m_ResolvedMetadataBuffer; encodeDesc.av1PictureDesc = m_Codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; + nri::VideoEncodePictureStates srcPictureStates = {}; + nri::VideoEncodePictureStates reconstructedPictureStates = {}; + if (Video.GetVideoEncodePictureStates(*m_EncodePicture, srcPictureStates) != nri::Result::SUCCESS || Video.GetVideoEncodePictureStates(*(av1PFrame ? m_AV1PReconstructedPicture : m_ReconstructedPicture), reconstructedPictureStates) != nri::Result::SUCCESS) { + m_VideoStatus = "Failed to query video encode picture states"; + return false; + } + if (!SubmitOneTime(NRI, *m_VideoEncodeQueue, [&](nri::CommandBuffer& commandBuffer) { nri::BufferBarrierDesc bufferBarriers[2] = {}; bufferBarriers[0].buffer = m_MetadataBuffer; bufferBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[1].buffer = m_ResolvedMetadataBuffer; + bufferBarriers[1].buffer = encodeDesc.resolvedMetadata; bufferBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; nri::TextureBarrierDesc textureBarriers[3] = {}; textureBarriers[0].texture = m_EncodeTexture; textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[0].after = srcPictureStates.encodeRead; textureBarriers[0].mipNum = nri::REMAINING; textureBarriers[0].layerNum = nri::REMAINING; textureBarriers[0].planes = nri::PlaneBits::ALL; textureBarriers[1].texture = av1PFrame ? m_AV1PReconstructedTexture : m_ReconstructedTexture; textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[1].after = reconstructedPictureStates.encodeWrite; textureBarriers[1].mipNum = nri::REMAINING; textureBarriers[1].layerNum = nri::REMAINING; textureBarriers[1].planes = nri::PlaneBits::ALL; textureBarriers[2].texture = m_ReconstructedTexture; - textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].before = reconstructedPictureStates.graphicsBefore; textureBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; textureBarriers[2].mipNum = nri::REMAINING; textureBarriers[2].layerNum = nri::REMAINING; @@ -1569,14 +1635,14 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { bufferBarriers[0].after = {}; bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; bufferBarriers[1].after = {}; - textureBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_SRC, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[0].before = srcPictureStates.encodeRead; textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[1].before = reconstructedPictureStates.encodeWrite; textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[2].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[2].after = reconstructedPictureStates.afterEncode; barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; + barrierDesc.textureNum = av1PFrame && reconstructedPictureStates.releaseAfterEncode ? helper::GetCountOf(textureBarriers) : 2; NRI.CmdBarrier(commandBuffer, barrierDesc); })) { m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submission failed"; @@ -1586,6 +1652,11 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { if (m_MetadataReadbackPending) return true; + if (m_GraphicsAPI == nri::GraphicsAPI::VK) { + m_MetadataReadbackPending = true; + return false; + } + NRI.ResetCommandAllocator(*m_MetadataReadbackCommandAllocator); if (NRI.BeginCommandBuffer(*m_MetadataReadbackCommandBuffer, nullptr) != nri::Result::SUCCESS) { m_VideoStatus = "Failed to begin metadata readback command buffer"; @@ -1595,7 +1666,7 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { nri::BufferBarrierDesc metadataBarriers[2] = {}; metadataBarriers[0].buffer = m_ResolvedMetadataBuffer; metadataBarriers[0].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; - metadataBarriers[0].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + metadataBarriers[0].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; metadataBarriers[1].buffer = m_ResolvedMetadataReadbackBuffer; metadataBarriers[1].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; @@ -1603,12 +1674,6 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { metadataBarrierDesc.buffers = metadataBarriers; metadataBarrierDesc.bufferNum = helper::GetCountOf(metadataBarriers); NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); - Video.CmdResolveVideoEncodeFeedback(*m_MetadataReadbackCommandBuffer, *m_EncodeSession, *m_ResolvedMetadataBuffer, 0); - metadataBarriers[0].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - metadataBarriers[0].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; - metadataBarrierDesc.bufferNum = 1; - NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); - metadataBarrierDesc.bufferNum = helper::GetCountOf(metadataBarriers); NRI.CmdCopyBuffer(*m_MetadataReadbackCommandBuffer, *m_ResolvedMetadataReadbackBuffer, 0, *m_ResolvedMetadataBuffer, 0, RESOLVED_METADATA_SIZE); metadataBarriers[0].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; metadataBarriers[0].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; @@ -1642,11 +1707,14 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { } bool Sample::TryDecodePendingMetadata(float timeSec) { - if (!m_MetadataReadbackPending) + if (!m_MetadataReadbackPending) { return false; + } - if (NRI.GetFenceValue(*m_MetadataReadbackFence) < m_MetadataReadbackFenceValue) + const uint64_t completedFence = NRI.GetFenceValue(*m_MetadataReadbackFence); + if (completedFence < m_MetadataReadbackFenceValue) { return false; + } m_MetadataReadbackPending = false; @@ -1695,7 +1763,17 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { av1InfoDesc.references = av1InfoReferences; av1InfoDesc.referenceNum = helper::GetCountOf(av1ReferenceNames); } + const uint8_t* encodedPayloadHeader = nullptr; + const uint64_t encodedPayloadOffset = ENCODED_SLICE_OFFSET + feedback.encodedBitstreamOffset; + const uint64_t encodedPayloadSize = feedback.encodedBitstreamWrittenBytes; + if (m_GraphicsAPI == nri::GraphicsAPI::VK && encodedPayloadOffset <= BITSTREAM_SIZE && encodedPayloadSize <= BITSTREAM_SIZE - encodedPayloadOffset) { + encodedPayloadHeader = (const uint8_t*)NRI.MapBuffer(*m_BitstreamBuffer, encodedPayloadOffset, encodedPayloadSize); + av1InfoDesc.encodedPayloadHeader = encodedPayloadHeader; + av1InfoDesc.encodedPayloadHeaderSize = encodedPayloadHeader ? encodedPayloadSize : 0; + } const nri::Result av1InfoResult = Video.GetVideoEncodeAV1DecodeInfo(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, av1InfoDesc, av1DecodeInfo); + if (encodedPayloadHeader) + NRI.UnmapBuffer(*m_BitstreamBuffer); if (av1InfoResult != nri::Result::SUCCESS) { m_VideoStatus = "Failed to prepare AV1 decode metadata"; return false; @@ -1714,16 +1792,18 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo, float timeSec) { std::vector annexBHeaders; - if (!WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) + if (!av1DecodeInfo && !WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) return false; std::vector annexBEndOfStream; if (!WriteAnnexBEndOfStream(annexBEndOfStream)) return false; + if (m_GraphicsAPI == nri::GraphicsAPI::VK) + annexBEndOfStream.clear(); - const uint64_t encodedPayloadSkip = av1DecodeInfo ? av1DecodeInfo->bitstreamOffset : GetEncodedPayloadHeaderSkip(m_Codec, feedback.encodedBitstreamWrittenBytes); + const uint64_t encodedPayloadSkip = av1DecodeInfo ? av1DecodeInfo->bitstreamOffset : 0; const uint64_t encodedPayloadBytes = av1DecodeInfo ? av1DecodeInfo->bitstreamSize : feedback.encodedBitstreamWrittenBytes - encodedPayloadSkip; const uint64_t decodeSliceOffset = annexBHeaders.size(); - const uint64_t decodeBitstreamSize = AlignUp(decodeSliceOffset + encodedPayloadBytes, 256); + const uint64_t decodeBitstreamSize = AlignUp(decodeSliceOffset + encodedPayloadBytes + annexBEndOfStream.size(), m_DecodeBitstreamSizeAlignment); const uint64_t encodedSourceOffset = ENCODED_SLICE_OFFSET + feedback.encodedBitstreamOffset + encodedPayloadSkip; if (feedback.encodedBitstreamOffset > BITSTREAM_SIZE - ENCODED_SLICE_OFFSET || encodedSourceOffset > BITSTREAM_SIZE || encodedPayloadBytes > BITSTREAM_SIZE - encodedSourceOffset || decodeBitstreamSize > BITSTREAM_SIZE) { m_VideoStatus = std::string("Encoded ") + GetCodecName(m_Codec) + " bitstream exceeded decode buffer size"; @@ -1743,28 +1823,36 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co std::memset(decodeBitstream, 0, (size_t)decodeBitstreamSize); if (!annexBHeaders.empty()) std::memcpy(decodeBitstream, annexBHeaders.data(), annexBHeaders.size()); - std::memcpy(decodeBitstream + decodeSliceOffset, encodedPayload, (size_t)encodedPayloadBytes); + const uint64_t encodedCopyBytes = encodedPayloadBytes; + std::memcpy(decodeBitstream + decodeSliceOffset, encodedPayload, (size_t)encodedCopyBytes); + if (!annexBEndOfStream.empty()) + std::memcpy(decodeBitstream + decodeSliceOffset + encodedCopyBytes, annexBEndOfStream.data(), annexBEndOfStream.size()); + const uint64_t decodeBitstreamRange = AlignUp(decodeSliceOffset + encodedCopyBytes + annexBEndOfStream.size(), m_DecodeBitstreamSizeAlignment); + const uint32_t pictureOffsets[] = {(uint32_t)decodeSliceOffset}; NRI.UnmapBuffer(*m_BitstreamBuffer); NRI.UnmapBuffer(*m_DecodeBitstreamBuffer); - const uint32_t pictureOffsets[] = {(uint32_t)decodeSliceOffset}; + const uint32_t decodeFrameIndex = m_DecodeFrameIndex++; + const uint32_t decodeSlot = m_Codec == SampleCodec::AV1 ? 0 : decodeFrameIndex % 16; nri::VideoH264DecodePictureDesc h264DecodePicture = {}; h264DecodePicture.flags = nri::VideoH264DecodePictureBits::IDR | nri::VideoH264DecodePictureBits::INTRA | nri::VideoH264DecodePictureBits::REFERENCE; + h264DecodePicture.sequenceParameterSetId = m_H264Sps.sequenceParameterSetId; h264DecodePicture.pictureParameterSetId = m_H264Pps.pictureParameterSetId; - h264DecodePicture.frameNum = 0; - h264DecodePicture.idrPictureId = 1; + h264DecodePicture.frameNum = (uint16_t)(decodeFrameIndex & 0xF); + h264DecodePicture.idrPictureId = (uint16_t)(1 + (decodeFrameIndex & 0xFFFF)); h264DecodePicture.topFieldOrderCount = 0; h264DecodePicture.bottomFieldOrderCount = 0; h264DecodePicture.sliceOffsets = pictureOffsets; h264DecodePicture.sliceOffsetNum = helper::GetCountOf(pictureOffsets); + h264DecodePicture.referenceSlot = decodeSlot; nri::VideoH265DecodePictureDesc h265DecodePicture = {}; h265DecodePicture.flags = nri::VideoH265DecodePictureBits::IRAP | nri::VideoH265DecodePictureBits::IDR | nri::VideoH265DecodePictureBits::REFERENCE; h265DecodePicture.videoParameterSetId = m_H265Vps.videoParameterSetId; h265DecodePicture.sequenceParameterSetId = m_H265Sps.sequenceParameterSetId; h265DecodePicture.pictureParameterSetId = m_H265Pps.pictureParameterSetId; - h265DecodePicture.pictureOrderCount = 0; + h265DecodePicture.pictureOrderCount = (int32_t)decodeFrameIndex; h265DecodePicture.sliceSegmentOffsets = pictureOffsets; h265DecodePicture.sliceSegmentOffsetNum = helper::GetCountOf(pictureOffsets); @@ -1799,11 +1887,11 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co decodeDesc.session = m_DecodeSession; decodeDesc.parameters = m_DecodeParameters; decodeDesc.bitstream.buffer = m_DecodeBitstreamBuffer; - decodeDesc.bitstream.size = decodeBitstreamSize; + decodeDesc.bitstream.size = decodeBitstreamRange; decodeDesc.dstPicture = av1PFrame ? m_AV1PDecodePicture : m_DecodePicture; decodeDesc.references = av1PFrame ? &av1DecodeReference : nullptr; decodeDesc.referenceNum = av1PFrame ? 1u : 0u; - decodeDesc.dstSlot = av1PFrame ? 1u : 0u; + decodeDesc.dstSlot = av1PFrame ? 1u : decodeSlot; decodeDesc.h264PictureDesc = m_Codec == SampleCodec::H264 ? &h264DecodePicture : nullptr; decodeDesc.h265PictureDesc = m_Codec == SampleCodec::H265 ? &h265DecodePicture : nullptr; decodeDesc.av1PictureDesc = av1DecodeInfo ? &av1Info.picture : nullptr; @@ -1813,25 +1901,41 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co m_VideoStatus = "Failed to query video decode picture states"; return false; } + if (m_GraphicsAPI == nri::GraphicsAPI::VK && !av1DecodeInfo) { + decodePictureStates.decodeWrite = {nri::AccessBits::VIDEO_DECODE_WRITE, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + decodePictureStates.graphicsBefore = {nri::AccessBits::VIDEO_DECODE, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + } if (!SubmitOneTime(NRI, *m_VideoDecodeQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::TextureBarrierDesc textureBarriers[2] = {}; - textureBarriers[0].texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; - textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[0].after = decodePictureStates.decodeWrite; - textureBarriers[0].mipNum = nri::REMAINING; - textureBarriers[0].layerNum = nri::REMAINING; - textureBarriers[0].planes = nri::PlaneBits::ALL; - textureBarriers[1].texture = m_DecodeTexture; - textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].after = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; - textureBarriers[1].mipNum = nri::REMAINING; - textureBarriers[1].layerNum = nri::REMAINING; - textureBarriers[1].planes = nri::PlaneBits::ALL; + nri::BufferBarrierDesc bufferBarrier = {}; + bufferBarrier.buffer = m_DecodeBitstreamBuffer; + bufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + bufferBarrier.after = {nri::AccessBits::VIDEO_DECODE_READ, nri::StageBits::VIDEO_DECODE}; + + nri::TextureBarrierDesc textureBarriers[3] = {}; + uint32_t textureBarrierNum = 0; + textureBarriers[textureBarrierNum].texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; + textureBarriers[textureBarrierNum].before = {nri::AccessBits::NONE, m_GraphicsAPI == nri::GraphicsAPI::VK && !av1DecodeInfo ? nri::Layout::UNDEFINED : nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[textureBarrierNum].after = decodePictureStates.decodeWrite; + textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; + textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; + textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; + textureBarrierNum++; + if (av1PFrame) { + textureBarriers[textureBarrierNum].texture = m_DecodeTexture; + textureBarriers[textureBarrierNum].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[textureBarrierNum].after = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; + textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; + textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; + textureBarrierNum++; + } nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = m_GraphicsAPI == nri::GraphicsAPI::D3D12 ? nullptr : &bufferBarrier; + barrierDesc.bufferNum = m_GraphicsAPI == nri::GraphicsAPI::D3D12 ? 0 : 1; barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = av1PFrame ? 2 : 1; + barrierDesc.textureNum = textureBarrierNum; NRI.CmdBarrier(commandBuffer, barrierDesc); Video.CmdDecodeVideo(commandBuffer, decodeDesc); @@ -1842,6 +1946,8 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co textureBarriers[1].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; } + barrierDesc.buffers = nullptr; + barrierDesc.bufferNum = 0; NRI.CmdBarrier(commandBuffer, barrierDesc); } })) { @@ -1849,7 +1955,8 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co return false; } - if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::Queue* readbackQueue = m_GraphicsAPI == nri::GraphicsAPI::VK ? m_VideoDecodeQueue : m_GraphicsQueue; + if (!SubmitOneTime(NRI, *readbackQueue, [&](nri::CommandBuffer& commandBuffer) { nri::TextureBarrierDesc textureBarrier = {}; textureBarrier.texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; textureBarrier.before = decodePictureStates.graphicsBefore; @@ -1936,10 +2043,13 @@ bool Sample::CanRunRoundTrip() const { } void Sample::LatencySleep(uint32_t frameIndex) { - uint32_t queuedFrameIndex = frameIndex % GetQueuedFrameNum(); + const uint32_t commandFrameNum = (uint32_t)m_QueuedFrames.size(); + const uint32_t queuedFrameNum = GetQueuedFrameNum(); + uint32_t queuedFrameIndex = frameIndex % commandFrameNum; const QueuedFrame& queuedFrame = m_QueuedFrames[queuedFrameIndex]; + const uint64_t waitValue = frameIndex >= queuedFrameNum ? 1 + frameIndex - queuedFrameNum : 0; - NRI.Wait(*m_FrameFence, frameIndex >= GetQueuedFrameNum() ? 1 + frameIndex - GetQueuedFrameNum() : 0); + NRI.Wait(*m_FrameFence, waitValue); NRI.ResetCommandAllocator(*queuedFrame.commandAllocator); } @@ -1956,7 +2066,8 @@ void Sample::PrepareFrame(uint32_t) { const double timeSec = m_Timer.GetTimeStamp() * 0.001 - m_StartTimeSec; const bool canRunRoundTrip = CanRunRoundTrip(); - InitializeGeneratedFrames((float)timeSec); + if (!m_SourcePreviewReady) + InitializeGeneratedFrames((float)timeSec); if (canRunRoundTrip && timeSec - m_LastRoundTripTimeSec >= ROUND_TRIP_INTERVAL_SEC) { if (TryRunRoundTrip((float)timeSec)) @@ -2000,7 +2111,7 @@ void Sample::PrepareFrame(uint32_t) { } void Sample::RenderFrame(uint32_t frameIndex) { - uint32_t queuedFrameIndex = frameIndex % GetQueuedFrameNum(); + uint32_t queuedFrameIndex = frameIndex % (uint32_t)m_QueuedFrames.size(); const QueuedFrame& queuedFrame = m_QueuedFrames[queuedFrameIndex]; uint32_t recycledSemaphoreIndex = frameIndex % (uint32_t)m_SwapChainTextures.size(); From 8623cb988c60692b4064d902de37c6d2d336f318 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 11:36:20 +0200 Subject: [PATCH 17/28] Refactor video encode decode sample --- CMakeLists.txt | 7 + Source/VideoEncodeDecode.cpp | 1214 ++------------------------ Source/VideoEncodeDecode/Decoder.cpp | 474 ++++++++++ Source/VideoEncodeDecode/Decoder.h | 57 ++ Source/VideoEncodeDecode/Encoder.cpp | 647 ++++++++++++++ Source/VideoEncodeDecode/Encoder.h | 79 ++ Source/VideoEncodeDecode/Shared.h | 372 ++++++++ 7 files changed, 1704 insertions(+), 1146 deletions(-) create mode 100644 Source/VideoEncodeDecode/Decoder.cpp create mode 100644 Source/VideoEncodeDecode/Decoder.h create mode 100644 Source/VideoEncodeDecode/Encoder.cpp create mode 100644 Source/VideoEncodeDecode/Encoder.h create mode 100644 Source/VideoEncodeDecode/Shared.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c9300bf..433f2ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,6 +191,13 @@ add_sample(SceneViewer cpp) add_sample(Triangle cpp) add_sample(VideoEncodeDecode cpp) +target_sources(VideoEncodeDecode PRIVATE + Source/VideoEncodeDecode/Decoder.cpp + Source/VideoEncodeDecode/Decoder.h + Source/VideoEncodeDecode/Encoder.cpp + Source/VideoEncodeDecode/Encoder.h + Source/VideoEncodeDecode/Shared.h +) if(WIN32) target_link_libraries(VideoEncodeDecode PRIVATE d3d12) diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 496c2ed..37337f3 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -5,6 +5,8 @@ #endif #include "NRIFramework.h" +#include "VideoEncodeDecode/Decoder.h" +#include "VideoEncodeDecode/Encoder.h" #include "Extensions/NRIVideo.h" @@ -12,6 +14,7 @@ #include #include #include +#include #include #include @@ -36,12 +39,6 @@ enum class SampleCodec : uint8_t { AV1, }; -struct Av1SequenceOptions { - bool enableCdef = true; - bool enableRestoration = true; - uint8_t seqForceScreenContentTools = 2; -}; - static const char* GetCodecName(SampleCodec codec) { switch (codec) { case SampleCodec::H265: @@ -66,40 +63,6 @@ static nri::VideoCodec GetNriCodec(SampleCodec codec) { } } -static uint8_t GetAv1FrameSizeBitsMinus1(uint32_t value) { - uint32_t bits = 0; - uint32_t maxValue = value ? value - 1 : 0; - do { - bits++; - maxValue >>= 1; - } while (maxValue); - return (uint8_t)(bits - 1); -} - -static nri::VideoAV1SequenceDesc MakeAV1SequenceDesc(uint32_t width, uint32_t height, const Av1SequenceOptions& options) { - nri::VideoAV1SequenceDesc desc = {}; - desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; - if (options.enableCdef) - desc.flags |= nri::VideoAV1SequenceBits::ENABLE_CDEF; - if (options.enableRestoration) - desc.flags |= nri::VideoAV1SequenceBits::ENABLE_RESTORATION; - desc.bitDepth = 8; - desc.subsamplingX = 1; - desc.subsamplingY = 1; - desc.maxFrameWidthMinus1 = (uint16_t)(width - 1); - desc.maxFrameHeightMinus1 = (uint16_t)(height - 1); - desc.frameWidthBitsMinus1 = GetAv1FrameSizeBitsMinus1(width); - desc.frameHeightBitsMinus1 = GetAv1FrameSizeBitsMinus1(height); - desc.orderHintBitsMinus1 = 7; - desc.seqForceIntegerMv = 2; - desc.seqForceScreenContentTools = options.seqForceScreenContentTools; - desc.colorPrimaries = 1; - desc.transferCharacteristics = 1; - desc.matrixCoefficients = 1; - desc.chromaSamplePosition = 1; - return desc; -} - struct QueuedFrame { nri::CommandAllocator* commandAllocator = nullptr; nri::CommandBuffer* commandBuffer = nullptr; @@ -245,14 +208,6 @@ static bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, }); } -static nri::Result CreateEncodeBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { - return core.CreateCommittedBuffer(device, nri::MemoryLocation::HOST_READBACK, priority, bufferDesc, buffer); -} - -static nri::Result CreateDecodeBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { - return core.CreateCommittedBuffer(device, nri::MemoryLocation::HOST_UPLOAD, priority, bufferDesc, buffer); -} - } // namespace class Sample : public SampleBase { @@ -275,17 +230,19 @@ class Sample : public SampleBase { void TryInitializeVideo(nri::GraphicsAPI graphicsAPI); PatternConstants MakePatternConstants(PatternOperation operation, float timeSec) const; bool GeneratePatternWithCompute(const PatternConstants& constants, nri::Descriptor* previewTexture, bool returnSourceBufferToShaderStorage = false, nri::AccessStage uploadBufferBefore = {}); - bool WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders); - bool WriteAnnexBEndOfStream(std::vector& annexBEndOfStream); bool TrySubmitEncodeAndMetadataReadback(float timeSec); bool TryDecodePendingMetadata(float timeSec); - bool DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo, float timeSec); bool TryRunRoundTrip(float timeSec); void DrawTexturePanel(const char* label, nri::Descriptor* texture, const ImVec2& size); private: NRIInterface NRI = {}; nri::VideoInterface Video = {}; + std::unique_ptr m_Encoder; + std::unique_ptr m_Decoder; + video_sample::VideoConfig m_VideoConfig = {}; + video_sample::VideoSize m_VideoSize = {}; + video_sample::CodecParameters m_CodecParameters = {}; nri::GraphicsAPI m_GraphicsAPI = nri::GraphicsAPI::NONE; nri::Device* m_Device = nullptr; @@ -296,15 +253,6 @@ class Sample : public SampleBase { nri::Queue* m_VideoDecodeQueue = nullptr; nri::Fence* m_FrameFence = nullptr; - nri::VideoSession* m_EncodeSession = nullptr; - nri::VideoSession* m_DecodeSession = nullptr; - nri::VideoSessionParameters* m_EncodeParameters = nullptr; - nri::VideoSessionParameters* m_DecodeParameters = nullptr; - nri::Texture* m_EncodeTexture = nullptr; - nri::Texture* m_ReconstructedTexture = nullptr; - nri::Texture* m_AV1PReconstructedTexture = nullptr; - nri::Texture* m_DecodeTexture = nullptr; - nri::Texture* m_AV1PDecodeTexture = nullptr; nri::Texture* m_SourcePreviewTexture = nullptr; nri::Texture* m_DecodePreviewTexture = nullptr; nri::Buffer* m_UploadBuffer = nullptr; @@ -317,25 +265,11 @@ class Sample : public SampleBase { nri::Pipeline* m_GenerateComputePipeline = nullptr; nri::DescriptorPool* m_GenerateDescriptorPool = nullptr; nri::DescriptorSet* m_GenerateDescriptorSet = nullptr; - nri::Buffer* m_BitstreamHeaderUploadBuffer = nullptr; - nri::Buffer* m_BitstreamBuffer = nullptr; - nri::Buffer* m_DecodeBitstreamBuffer = nullptr; - nri::Buffer* m_MetadataBuffer = nullptr; - nri::Buffer* m_ResolvedMetadataBuffer = nullptr; - nri::Buffer* m_ResolvedMetadataReadbackBuffer = nullptr; - nri::VideoPicture* m_EncodePicture = nullptr; - nri::VideoPicture* m_ReconstructedPicture = nullptr; - nri::VideoPicture* m_AV1PReconstructedPicture = nullptr; - nri::VideoPicture* m_DecodePicture = nullptr; - nri::VideoPicture* m_AV1PDecodePicture = nullptr; - nri::CommandAllocator* m_MetadataReadbackCommandAllocator = nullptr; - nri::CommandBuffer* m_MetadataReadbackCommandBuffer = nullptr; - nri::Fence* m_MetadataReadbackFence = nullptr; - std::vector m_QueuedFrames; std::vector m_SwapChainTextures; Nv12BufferLayout m_Nv12Layout = {}; + video_sample::Nv12BufferLayout m_SharedNv12Layout = {}; nri::Format m_SwapChainFormat = nri::Format::UNKNOWN; std::string m_VideoStatus = "Initializing video"; std::string m_PreviewStatus = "Initializing preview"; @@ -346,18 +280,11 @@ class Sample : public SampleBase { uint32_t m_CodedVideoWidth = DEFAULT_VIDEO_WIDTH; uint32_t m_CodedVideoHeight = DEFAULT_VIDEO_HEIGHT; uint32_t m_DecodeBitstreamSizeAlignment = 256; - uint32_t m_DecodeFrameIndex = 0; uint32_t m_QpI = 20; uint32_t m_QpP = 22; uint32_t m_QpB = 24; uint32_t m_AV1BaseQIndex = 20; SampleCodec m_Codec = SampleCodec::H264; - nri::VideoH264SequenceParameterSetDesc m_H264Sps = {}; - nri::VideoH264PictureParameterSetDesc m_H264Pps = {}; - nri::VideoH265VideoParameterSetDesc m_H265Vps = {}; - nri::VideoH265SequenceParameterSetDesc m_H265Sps = {}; - nri::VideoH265PictureParameterSetDesc m_H265Pps = {}; - nri::VideoAV1SequenceDesc m_AV1Sequence = {}; double m_StartTimeSec = 0.0; double m_LastRoundTripTimeSec = -ROUND_TRIP_INTERVAL_SEC; bool m_VideoReady = false; @@ -365,47 +292,16 @@ class Sample : public SampleBase { bool m_DecodePreviewReady = false; bool m_PreviewTexturesShaderReadable = false; bool m_SourcePreviewReady = false; - bool m_MetadataReadbackPending = false; bool m_AV1PFrameVisual = false; - uint32_t m_AV1PFrameStage = 0; - uint64_t m_MetadataReadbackFenceValue = 0; }; Sample::~Sample() { if (NRI.HasCore()) { NRI.DeviceWaitIdle(m_Device); - if (Video.DestroyVideoPicture) { - if (m_AV1PDecodePicture) - Video.DestroyVideoPicture(m_AV1PDecodePicture); - if (m_DecodePicture) - Video.DestroyVideoPicture(m_DecodePicture); - if (m_AV1PReconstructedPicture) - Video.DestroyVideoPicture(m_AV1PReconstructedPicture); - if (m_ReconstructedPicture) - Video.DestroyVideoPicture(m_ReconstructedPicture); - if (m_EncodePicture) - Video.DestroyVideoPicture(m_EncodePicture); - if (m_DecodeParameters) - Video.DestroyVideoSessionParameters(m_DecodeParameters); - if (m_EncodeParameters) - Video.DestroyVideoSessionParameters(m_EncodeParameters); - if (m_DecodeSession) - Video.DestroyVideoSession(m_DecodeSession); - if (m_EncodeSession) - Video.DestroyVideoSession(m_EncodeSession); - } + m_Decoder.reset(); + m_Encoder.reset(); - if (m_MetadataReadbackCommandBuffer) - NRI.DestroyCommandBuffer(m_MetadataReadbackCommandBuffer); - if (m_MetadataReadbackCommandAllocator) - NRI.DestroyCommandAllocator(m_MetadataReadbackCommandAllocator); - if (m_MetadataReadbackFence) - NRI.DestroyFence(m_MetadataReadbackFence); - if (m_ResolvedMetadataReadbackBuffer) - NRI.DestroyBuffer(m_ResolvedMetadataReadbackBuffer); - if (m_ResolvedMetadataBuffer) - NRI.DestroyBuffer(m_ResolvedMetadataBuffer); if (m_GenerateDescriptorPool) NRI.DestroyDescriptorPool(m_GenerateDescriptorPool); if (m_GenerateComputePipeline) @@ -414,14 +310,6 @@ Sample::~Sample() { NRI.DestroyPipelineLayout(m_GeneratePipelineLayout); if (m_UploadBufferView) NRI.DestroyDescriptor(m_UploadBufferView); - if (m_MetadataBuffer) - NRI.DestroyBuffer(m_MetadataBuffer); - if (m_DecodeBitstreamBuffer) - NRI.DestroyBuffer(m_DecodeBitstreamBuffer); - if (m_BitstreamBuffer) - NRI.DestroyBuffer(m_BitstreamBuffer); - if (m_BitstreamHeaderUploadBuffer) - NRI.DestroyBuffer(m_BitstreamHeaderUploadBuffer); if (m_UploadBuffer) NRI.DestroyBuffer(m_UploadBuffer); if (m_SourcePreviewStorage) @@ -436,17 +324,6 @@ Sample::~Sample() { NRI.DestroyTexture(m_SourcePreviewTexture); if (m_DecodePreviewTexture) NRI.DestroyTexture(m_DecodePreviewTexture); - if (m_DecodeTexture) - NRI.DestroyTexture(m_DecodeTexture); - if (m_AV1PDecodeTexture) - NRI.DestroyTexture(m_AV1PDecodeTexture); - if (m_ReconstructedTexture) - NRI.DestroyTexture(m_ReconstructedTexture); - if (m_AV1PReconstructedTexture) - NRI.DestroyTexture(m_AV1PReconstructedTexture); - if (m_EncodeTexture) - NRI.DestroyTexture(m_EncodeTexture); - for (QueuedFrame& queuedFrame : m_QueuedFrames) { NRI.DestroyCommandBuffer(queuedFrame.commandBuffer); NRI.DestroyCommandAllocator(queuedFrame.commandAllocator); @@ -514,6 +391,15 @@ bool Sample::Initialize(nri::GraphicsAPI graphicsAPI, bool) { } m_GraphicsAPI = graphicsAPI; + m_VideoConfig.codec = m_Codec == SampleCodec::H265 ? video_sample::SampleCodec::H265 : (m_Codec == SampleCodec::AV1 ? video_sample::SampleCodec::AV1 : video_sample::SampleCodec::H264); + m_VideoConfig.videoWidth = m_VideoWidth; + m_VideoConfig.videoHeight = m_VideoHeight; + m_VideoConfig.qpI = m_QpI; + m_VideoConfig.qpP = m_QpP; + m_VideoConfig.qpB = m_QpB; + m_VideoConfig.av1BaseQIndex = m_AV1BaseQIndex; + m_VideoConfig.av1PFrameVisual = m_AV1PFrameVisual; + if (!InitializeGraphics(graphicsAPI)) return false; @@ -565,6 +451,7 @@ bool Sample::InitializeGraphics(nri::GraphicsAPI graphicsAPI) { NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::StreamerInterface), (nri::StreamerInterface*)&NRI)); NRI_ABORT_ON_FAILURE(nri::nriGetInterface(*m_Device, NRI_INTERFACE(nri::SwapChainInterface), (nri::SwapChainInterface*)&NRI)); m_Nv12Layout = MakeNv12BufferLayout(NRI.GetDeviceDesc(*m_Device), m_VideoWidth, m_VideoHeight); + m_SharedNv12Layout = video_sample::MakeNv12BufferLayout(NRI.GetDeviceDesc(*m_Device), m_VideoWidth, m_VideoHeight); nri::StreamerDesc streamerDesc = {}; streamerDesc.dynamicBufferMemoryLocation = nri::MemoryLocation::HOST_UPLOAD; @@ -639,13 +526,14 @@ void Sample::InitializeGeneratedFrames(float timeSec, bool uploadEncodeTexture) PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); - if (m_EncodeTexture && uploadEncodeTexture) { + nri::Texture* encodeTexture = m_Encoder ? m_Encoder->GetInputTexture() : nullptr; + if (encodeTexture && uploadEncodeTexture) { if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { m_PreviewStatus = "Failed to generate source pattern via compute"; return; } - if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture, m_VideoWidth, m_VideoHeight)) { + if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *encodeTexture, m_VideoWidth, m_VideoHeight)) { m_PreviewStatus = "Failed to upload NV12 source to video texture"; return; } @@ -919,12 +807,7 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (!m_VideoQueuesRequested) { - m_VideoStatus = "Adapter has no NRI video encode/decode queues"; - return; - } - - if (!deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_ENCODE] || !deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_DECODE]) { + if (!m_VideoQueuesRequested || !deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_ENCODE] || !deviceDesc.adapterDesc.queueNum[(uint32_t)nri::QueueType::VIDEO_DECODE]) { m_VideoStatus = "Adapter has no NRI video encode/decode queues"; return; } @@ -982,1036 +865,75 @@ void Sample::TryInitializeVideo(nri::GraphicsAPI graphicsAPI) { return; } - if (Video.CreateVideoSession(*m_Device, encodeSessionDesc, m_EncodeSession) != nri::Result::SUCCESS) { - m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " encode session"; - return; - } - - if (Video.CreateVideoSession(*m_Device, decodeSessionDesc, m_DecodeSession) != nri::Result::SUCCESS) { - m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " decode session"; - return; - } - - nri::VideoH264SequenceParameterSetDesc sps = {}; - sps.flags = nri::VideoH264SequenceParameterSetBits::DIRECT_8X8_INFERENCE | nri::VideoH264SequenceParameterSetBits::FRAME_MBS_ONLY; - sps.profileIdc = 100; - sps.levelIdc = 42; - sps.chromaFormatIdc = 1; - sps.sequenceParameterSetId = 0; - sps.log2MaxFrameNumMinus4 = 0; - sps.pictureOrderCountType = 0; - sps.log2MaxPictureOrderCountLsbMinus4 = 0; - sps.referenceFrameNum = 1; - sps.pictureWidthInMbsMinus1 = (uint16_t)((m_CodedVideoWidth + 15) / 16 - 1); - sps.pictureHeightInMapUnitsMinus1 = (uint16_t)((m_CodedVideoHeight + 15) / 16 - 1); - - nri::VideoH264PictureParameterSetDesc pps = {}; - pps.flags = nri::VideoH264PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; - pps.sequenceParameterSetId = 0; - pps.pictureParameterSetId = 0; - pps.refIndexL0DefaultActiveMinus1 = 0; - pps.refIndexL1DefaultActiveMinus1 = 0; - m_H264Sps = sps; - m_H264Pps = pps; - - nri::VideoH264SessionParametersDesc h264Parameters = {}; - h264Parameters.sequenceParameterSets = &sps; - h264Parameters.sequenceParameterSetNum = 1; - h264Parameters.pictureParameterSets = &pps; - h264Parameters.pictureParameterSetNum = 1; - h264Parameters.maxSequenceParameterSetNum = 1; - h264Parameters.maxPictureParameterSetNum = 1; - - nri::VideoH265VideoParameterSetDesc vps = {}; - vps.flags = nri::VideoH265VideoParameterSetBits::TEMPORAL_ID_NESTING; - vps.videoParameterSetId = 0; - vps.maxSubLayersMinus1 = 0; - vps.profileTierLevel.flags = nri::VideoH265ProfileTierLevelBits::FRAME_ONLY_CONSTRAINT; - vps.profileTierLevel.generalProfileIdc = 1; - vps.profileTierLevel.generalLevelIdc = 90; - vps.decPicBufMgr.maxDecPicBufferingMinus1[0] = 2; - vps.decPicBufMgr.maxNumReorderPics[0] = 1; - - nri::VideoH265SequenceParameterSetDesc h265Sps = {}; - h265Sps.flags = nri::VideoH265SequenceParameterSetBits::TEMPORAL_ID_NESTING | nri::VideoH265SequenceParameterSetBits::AMP_ENABLED | nri::VideoH265SequenceParameterSetBits::SAMPLE_ADAPTIVE_OFFSET_ENABLED; - h265Sps.videoParameterSetId = vps.videoParameterSetId; - h265Sps.maxSubLayersMinus1 = vps.maxSubLayersMinus1; - h265Sps.sequenceParameterSetId = 0; - h265Sps.chromaFormatIdc = 1; - h265Sps.pictureWidthInLumaSamples = m_CodedVideoWidth; - h265Sps.pictureHeightInLumaSamples = m_CodedVideoHeight; - h265Sps.log2MaxPictureOrderCountLsbMinus4 = 3; - h265Sps.log2MinLumaCodingBlockSizeMinus3 = 0; - h265Sps.log2DiffMaxMinLumaCodingBlockSize = 2; - h265Sps.log2MinLumaTransformBlockSizeMinus2 = 0; - h265Sps.log2DiffMaxMinLumaTransformBlockSize = 3; - h265Sps.maxTransformHierarchyDepthInter = 3; - h265Sps.maxTransformHierarchyDepthIntra = 3; - h265Sps.profileTierLevel = vps.profileTierLevel; - h265Sps.decPicBufMgr = vps.decPicBufMgr; - - nri::VideoH265PictureParameterSetDesc h265Pps = {}; - h265Pps.flags = nri::VideoH265PictureParameterSetBits::CABAC_INIT_PRESENT | nri::VideoH265PictureParameterSetBits::TRANSFORM_SKIP_ENABLED | nri::VideoH265PictureParameterSetBits::CU_QP_DELTA_ENABLED | nri::VideoH265PictureParameterSetBits::SLICE_CHROMA_QP_OFFSETS_PRESENT | nri::VideoH265PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; - h265Pps.pictureParameterSetId = 0; - h265Pps.sequenceParameterSetId = h265Sps.sequenceParameterSetId; - h265Pps.videoParameterSetId = vps.videoParameterSetId; - m_H265Vps = vps; - m_H265Sps = h265Sps; - m_H265Pps = h265Pps; - - nri::VideoH265SessionParametersDesc h265Parameters = {}; - h265Parameters.videoParameterSets = &vps; - h265Parameters.videoParameterSetNum = 1; - h265Parameters.sequenceParameterSets = &h265Sps; - h265Parameters.sequenceParameterSetNum = 1; - h265Parameters.pictureParameterSets = &h265Pps; - h265Parameters.pictureParameterSetNum = 1; - h265Parameters.maxVideoParameterSetNum = 1; - h265Parameters.maxSequenceParameterSetNum = 1; - h265Parameters.maxPictureParameterSetNum = 1; - - Av1SequenceOptions av1SequenceOptions = {}; - if (graphicsAPI == nri::GraphicsAPI::VK) { - av1SequenceOptions.enableCdef = false; - av1SequenceOptions.enableRestoration = false; - av1SequenceOptions.seqForceScreenContentTools = 0; - } - m_AV1Sequence = MakeAV1SequenceDesc(m_CodedVideoWidth, m_CodedVideoHeight, av1SequenceOptions); - nri::VideoAV1SessionParametersDesc av1Parameters = {}; - av1Parameters.sequence = m_AV1Sequence; - - nri::VideoSessionParametersDesc encodeParametersDesc = {}; - encodeParametersDesc.session = m_EncodeSession; - encodeParametersDesc.h264Parameters = m_Codec == SampleCodec::H264 ? &h264Parameters : nullptr; - encodeParametersDesc.h265Parameters = m_Codec == SampleCodec::H265 ? &h265Parameters : nullptr; - encodeParametersDesc.av1Parameters = m_Codec == SampleCodec::AV1 ? &av1Parameters : nullptr; - - nri::VideoSessionParametersDesc decodeParametersDesc = {}; - decodeParametersDesc.session = m_DecodeSession; - decodeParametersDesc.h264Parameters = m_Codec == SampleCodec::H264 ? &h264Parameters : nullptr; - decodeParametersDesc.h265Parameters = m_Codec == SampleCodec::H265 ? &h265Parameters : nullptr; - decodeParametersDesc.av1Parameters = m_Codec == SampleCodec::AV1 ? &av1Parameters : nullptr; - - if (Video.CreateVideoSessionParameters(*m_Device, encodeParametersDesc, m_EncodeParameters) != nri::Result::SUCCESS) { - m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " encode parameters"; - return; - } - - if (Video.CreateVideoSessionParameters(*m_Device, decodeParametersDesc, m_DecodeParameters) != nri::Result::SUCCESS) { - m_VideoStatus = std::string("Failed to create ") + GetCodecName(m_Codec) + " decode parameters"; - return; - } - - nri::TextureDesc encodeTextureDesc = {}; - encodeTextureDesc.type = nri::TextureType::TEXTURE_2D; - encodeTextureDesc.usage = nri::TextureUsageBits::VIDEO_ENCODE; - encodeTextureDesc.format = nri::Format::NV12_UNORM; - encodeTextureDesc.width = (nri::Dim_t)m_CodedVideoWidth; - encodeTextureDesc.height = (nri::Dim_t)m_CodedVideoHeight; - encodeTextureDesc.mipNum = 1; - encodeTextureDesc.layerNum = 1; - encodeTextureDesc.videoCodec = GetNriCodec(m_Codec); - - nri::TextureDesc decodeTextureDesc = encodeTextureDesc; - decodeTextureDesc.usage = nri::TextureUsageBits::VIDEO_DECODE; - - if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeTextureDesc, m_EncodeTexture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create NV12 encode texture"; - return; - } - NRI.SetDebugName(m_EncodeTexture, "VideoEncodeTexture"); - - if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeTextureDesc, m_ReconstructedTexture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create NV12 reconstructed texture"; + m_VideoSize.videoWidth = m_VideoWidth; + m_VideoSize.videoHeight = m_VideoHeight; + m_VideoSize.codedWidth = m_CodedVideoWidth; + m_VideoSize.codedHeight = m_CodedVideoHeight; + m_VideoSize.decodeBitstreamSizeAlignment = m_DecodeBitstreamSizeAlignment; + m_CodecParameters = video_sample::MakeCodecParameters(graphicsAPI, m_CodedVideoWidth, m_CodedVideoHeight); + + video_sample::VideoContext videoContext = {}; + videoContext.nri = &NRI; + videoContext.video = &Video; + videoContext.device = m_Device; + videoContext.graphicsQueue = m_GraphicsQueue; + videoContext.encodeQueue = m_VideoEncodeQueue; + videoContext.decodeQueue = m_VideoDecodeQueue; + videoContext.graphicsAPI = graphicsAPI; + + m_Encoder = std::make_unique(); + if (!m_Encoder->Initialize(videoContext, m_VideoConfig, m_VideoSize, m_CodecParameters)) { + m_VideoStatus = m_Encoder->GetStatus(); return; } - NRI.SetDebugName(m_ReconstructedTexture, "VideoReconstructedTexture"); - - if (m_AV1PFrameVisual) { - if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, encodeTextureDesc, m_AV1PReconstructedTexture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create second NV12 reconstructed texture"; - return; - } - NRI.SetDebugName(m_AV1PReconstructedTexture, "VideoAV1PReconstructedTexture"); - } - if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, decodeTextureDesc, m_DecodeTexture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create NV12 decode texture"; + m_Decoder = std::make_unique(); + if (!m_Decoder->Initialize(videoContext, m_VideoConfig, m_VideoSize, m_CodecParameters)) { + m_VideoStatus = m_Decoder->GetStatus(); return; } - NRI.SetDebugName(m_DecodeTexture, "VideoDecodeTexture"); - - if (m_AV1PFrameVisual) { - if (NRI.CreateCommittedTexture(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, decodeTextureDesc, m_AV1PDecodeTexture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create second NV12 decode texture"; - return; - } - NRI.SetDebugName(m_AV1PDecodeTexture, "VideoAV1PDecodeTexture"); - } - - if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::TextureBarrierDesc textureBarriers[5] = {}; - uint32_t textureBarrierNum = 0; - textureBarriers[textureBarrierNum++].texture = m_EncodeTexture; - textureBarriers[textureBarrierNum++].texture = m_ReconstructedTexture; - textureBarriers[textureBarrierNum++].texture = m_DecodeTexture; - if (m_AV1PReconstructedTexture) - textureBarriers[textureBarrierNum++].texture = m_AV1PReconstructedTexture; - if (m_AV1PDecodeTexture) - textureBarriers[textureBarrierNum++].texture = m_AV1PDecodeTexture; - - for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { - if (!textureBarrier.texture) - continue; - textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::UNDEFINED, nri::StageBits::ALL}; - textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarrier.mipNum = nri::REMAINING; - textureBarrier.layerNum = nri::REMAINING; - textureBarrier.planes = nri::PlaneBits::ALL; - } - - nri::BarrierDesc barrierDesc = {}; - barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = textureBarrierNum; - NRI.CmdBarrier(commandBuffer, barrierDesc); - })) { - m_VideoStatus = "Failed to initialize video texture layouts"; - return; - } - - if (!m_GenerateComputePipeline) { - nri::TextureViewDesc sourceStorageTextureViewDesc = {m_SourcePreviewTexture, nri::TextureView::STORAGE_TEXTURE, nri::Format::RGBA8_UNORM}; - nri::TextureViewDesc decodeStorageTextureViewDesc = {m_DecodePreviewTexture, nri::TextureView::STORAGE_TEXTURE, nri::Format::RGBA8_UNORM}; - - if (!m_SourcePreviewStorage) { - if (NRI.CreateTextureView(sourceStorageTextureViewDesc, m_SourcePreviewStorage) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create source preview storage texture view"; - return; - } - } - - if (!m_DecodePreviewStorage) { - if (NRI.CreateTextureView(decodeStorageTextureViewDesc, m_DecodePreviewStorage) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create decode preview storage texture view"; - return; - } - } - - nri::BufferDesc uploadBufferDesc = {}; - uploadBufferDesc.size = m_Nv12Layout.totalSizeBytes; - uploadBufferDesc.usage = nri::BufferUsageBits::SHADER_RESOURCE_STORAGE; - - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, uploadBufferDesc, m_UploadBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create NV12 upload buffer"; - return; - } - - nri::BufferViewDesc uploadBufferViewDesc = {}; - uploadBufferViewDesc.buffer = m_UploadBuffer; - uploadBufferViewDesc.format = nri::Format::R32_UINT; - uploadBufferViewDesc.type = nri::BufferView::STORAGE_BUFFER; - uploadBufferViewDesc.size = uploadBufferDesc.size; - - if (NRI.CreateBufferView(uploadBufferViewDesc, m_UploadBufferView) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create NV12 compute output buffer view"; - return; - } - - utils::ShaderCodeStorage shaderCodeStorage; - { - nri::DescriptorRangeDesc descriptorRanges[] = { - {0, 1, nri::DescriptorType::STORAGE_BUFFER, nri::StageBits::COMPUTE_SHADER}, - {1, 1, nri::DescriptorType::STORAGE_TEXTURE, nri::StageBits::COMPUTE_SHADER}, - {2, 1, nri::DescriptorType::STORAGE_TEXTURE, nri::StageBits::COMPUTE_SHADER}, - }; - nri::DescriptorSetDesc descriptorSetDescs[] = {{0, descriptorRanges, helper::GetCountOf(descriptorRanges)}}; - - nri::RootConstantDesc rootConstantDesc = {}; - rootConstantDesc.registerIndex = 0; - rootConstantDesc.size = sizeof(PatternConstants); - rootConstantDesc.shaderStages = nri::StageBits::COMPUTE_SHADER; - - nri::PipelineLayoutDesc pipelineLayoutDesc = {}; - pipelineLayoutDesc.rootConstantNum = 1; - pipelineLayoutDesc.rootConstants = &rootConstantDesc; - pipelineLayoutDesc.descriptorSetNum = helper::GetCountOf(descriptorSetDescs); - pipelineLayoutDesc.descriptorSets = descriptorSetDescs; - pipelineLayoutDesc.shaderStages = nri::StageBits::COMPUTE_SHADER; - if (NRI.CreatePipelineLayout(*m_Device, pipelineLayoutDesc, m_GeneratePipelineLayout) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create compute pipeline layout for pattern generation"; - return; - } - - nri::ComputePipelineDesc computePipelineDesc = {}; - computePipelineDesc.pipelineLayout = m_GeneratePipelineLayout; - computePipelineDesc.shader = utils::LoadShader(deviceDesc.graphicsAPI, "VideoEncodePattern.cs", shaderCodeStorage); - if (NRI.CreateComputePipeline(*m_Device, computePipelineDesc, m_GenerateComputePipeline) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create pattern generation compute pipeline"; - return; - } - - nri::DescriptorPoolDesc descriptorPoolDesc = {}; - descriptorPoolDesc.descriptorSetMaxNum = 1; - descriptorPoolDesc.storageBufferMaxNum = 1; - descriptorPoolDesc.storageTextureMaxNum = 2; - descriptorPoolDesc.textureMaxNum = 2; - if (NRI.CreateDescriptorPool(*m_Device, descriptorPoolDesc, m_GenerateDescriptorPool) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create compute descriptor pool for pattern generation"; - return; - } - - if (NRI.AllocateDescriptorSets(*m_GenerateDescriptorPool, *m_GeneratePipelineLayout, 0, &m_GenerateDescriptorSet, 1, 0) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to allocate compute descriptor set for pattern generation"; - return; - } - - nri::UpdateDescriptorRangeDesc updateDescriptorRangeDescs[] = { - {m_GenerateDescriptorSet, 0, 0, &m_UploadBufferView, 1}, - {m_GenerateDescriptorSet, 1, 0, &m_SourcePreviewStorage, 1}, - {m_GenerateDescriptorSet, 2, 0, &m_DecodePreviewStorage, 1}, - }; - NRI.UpdateDescriptorRanges(updateDescriptorRangeDescs, helper::GetCountOf(updateDescriptorRangeDescs)); - } - } - - nri::BufferDesc bitstreamHeaderUploadBufferDesc = {}; - bitstreamHeaderUploadBufferDesc.size = ENCODED_SLICE_OFFSET; - - nri::BufferDesc bitstreamBufferDesc = {}; - bitstreamBufferDesc.size = BITSTREAM_SIZE; - bitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; - - nri::BufferDesc decodeBitstreamBufferDesc = {}; - decodeBitstreamBufferDesc.size = BITSTREAM_SIZE; - decodeBitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_DECODE; - - nri::BufferDesc metadataBufferDesc = {}; - metadataBufferDesc.size = METADATA_SIZE; - metadataBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; - - nri::BufferDesc resolvedMetadataBufferDesc = {}; - resolvedMetadataBufferDesc.size = RESOLVED_METADATA_SIZE; - resolvedMetadataBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; - - nri::BufferDesc resolvedMetadataReadbackBufferDesc = {}; - resolvedMetadataReadbackBufferDesc.size = RESOLVED_METADATA_SIZE; - resolvedMetadataReadbackBufferDesc.usage = nri::BufferUsageBits::NONE; - - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_UPLOAD, 0.0f, bitstreamHeaderUploadBufferDesc, m_BitstreamHeaderUploadBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create bitstream header upload buffer"; - return; - } - - if (CreateEncodeBitstreamBuffer(NRI, *m_Device, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create encode bitstream buffer"; - return; - } - - if (CreateDecodeBitstreamBuffer(NRI, *m_Device, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create decode bitstream buffer"; - return; - } - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, metadataBufferDesc, m_MetadataBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create encode metadata buffer"; - return; - } - - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::DEVICE, 0.0f, resolvedMetadataBufferDesc, m_ResolvedMetadataBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create resolved encode metadata buffer"; - return; - } - - if (NRI.CreateCommittedBuffer(*m_Device, nri::MemoryLocation::HOST_READBACK, 0.0f, resolvedMetadataReadbackBufferDesc, m_ResolvedMetadataReadbackBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create resolved encode metadata readback buffer"; - return; - } - - if (NRI.CreateFence(*m_Device, 0, m_MetadataReadbackFence) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create metadata readback fence"; - return; - } - - if (NRI.CreateCommandAllocator(*m_GraphicsQueue, m_MetadataReadbackCommandAllocator) != nri::Result::SUCCESS || NRI.CreateCommandBuffer(*m_MetadataReadbackCommandAllocator, m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create metadata readback command buffer"; - return; - } - - nri::VideoPictureDesc encodePictureDesc = {}; - encodePictureDesc.texture = m_EncodeTexture; - encodePictureDesc.usage = nri::VideoPictureUsage::ENCODE_INPUT; - encodePictureDesc.width = (nri::Dim_t)m_CodedVideoWidth; - encodePictureDesc.height = (nri::Dim_t)m_CodedVideoHeight; - - nri::VideoPictureDesc decodePictureDesc = {}; - decodePictureDesc.texture = m_DecodeTexture; - decodePictureDesc.usage = nri::VideoPictureUsage::DECODE_OUTPUT; - decodePictureDesc.width = (nri::Dim_t)m_CodedVideoWidth; - decodePictureDesc.height = (nri::Dim_t)m_CodedVideoHeight; - - nri::VideoPictureDesc reconstructedPictureDesc = encodePictureDesc; - reconstructedPictureDesc.texture = m_ReconstructedTexture; - reconstructedPictureDesc.usage = nri::VideoPictureUsage::ENCODE_REFERENCE; - - if (Video.CreateVideoPicture(*m_Device, encodePictureDesc, m_EncodePicture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create encode picture"; - return; - } - - if (Video.CreateVideoPicture(*m_Device, reconstructedPictureDesc, m_ReconstructedPicture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create reconstructed picture"; - return; - } - - if (m_AV1PFrameVisual) { - reconstructedPictureDesc.texture = m_AV1PReconstructedTexture; - if (Video.CreateVideoPicture(*m_Device, reconstructedPictureDesc, m_AV1PReconstructedPicture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create second reconstructed picture"; - return; - } - } - - if (Video.CreateVideoPicture(*m_Device, decodePictureDesc, m_DecodePicture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create decode picture"; - return; - } - - if (m_AV1PFrameVisual) { - decodePictureDesc.texture = m_AV1PDecodeTexture; - if (Video.CreateVideoPicture(*m_Device, decodePictureDesc, m_AV1PDecodePicture) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to create second decode picture"; - return; - } - } m_VideoReady = true; m_VideoStatus = std::string("NRI video queues and ") + GetCodecName(m_Codec) + " encode/decode objects initialized"; } -bool Sample::WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders) { - if (m_Codec == SampleCodec::AV1) { - void* headerPtr = NRI.MapBuffer(*m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); - if (!headerPtr) { - m_VideoStatus = "Failed to map bitstream header upload buffer"; - return false; - } - std::memset(headerPtr, 0, (size_t)ENCODED_SLICE_OFFSET); - NRI.UnmapBuffer(*m_BitstreamHeaderUploadBuffer); - annexBHeaders.clear(); - return true; - } - - nri::VideoAnnexBParameterSetsDesc annexBDesc = {}; - annexBDesc.codec = GetNriCodec(m_Codec); - annexBDesc.h264Sps = &m_H264Sps; - annexBDesc.h264Pps = &m_H264Pps; - annexBDesc.h265Vps = &m_H265Vps; - annexBDesc.h265Sps = &m_H265Sps; - annexBDesc.h265Pps = &m_H265Pps; - - if (Video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS || annexBDesc.writtenSize == 0 || annexBDesc.writtenSize >= ENCODED_SLICE_OFFSET) { - m_VideoStatus = std::string("Failed to query ") + GetCodecName(m_Codec) + " Annex-B parameter-set size"; - return false; - } - - annexBHeaders.resize((size_t)annexBDesc.writtenSize); - annexBDesc.dst = annexBHeaders.data(); - annexBDesc.dstSize = annexBHeaders.size(); - if (Video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS) { - m_VideoStatus = std::string("Failed to build ") + GetCodecName(m_Codec) + " Annex-B parameter sets"; - return false; - } - - void* headerPtr = NRI.MapBuffer(*m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); - if (!headerPtr) { - m_VideoStatus = "Failed to map bitstream header upload buffer"; - return false; - } - std::memset(headerPtr, 0, (size_t)ENCODED_SLICE_OFFSET); - std::memcpy(headerPtr, annexBHeaders.data(), annexBHeaders.size()); - NRI.UnmapBuffer(*m_BitstreamHeaderUploadBuffer); - return true; -} - -bool Sample::WriteAnnexBEndOfStream(std::vector& annexBEndOfStream) { - annexBEndOfStream.clear(); - if (m_Codec == SampleCodec::AV1) - return true; - - nri::VideoAnnexBEndOfStreamDesc annexBDesc = {}; - annexBDesc.codec = GetNriCodec(m_Codec); - if (Video.WriteVideoAnnexBEndOfStream(annexBDesc) != nri::Result::SUCCESS || annexBDesc.writtenSize == 0) { - m_VideoStatus = std::string("Failed to query ") + GetCodecName(m_Codec) + " Annex-B end-of-stream size"; - return false; - } - - annexBEndOfStream.resize((size_t)annexBDesc.writtenSize); - annexBDesc.dst = annexBEndOfStream.data(); - annexBDesc.dstSize = annexBEndOfStream.size(); - if (Video.WriteVideoAnnexBEndOfStream(annexBDesc) != nri::Result::SUCCESS) { - m_VideoStatus = std::string("Failed to build ") + GetCodecName(m_Codec) + " Annex-B end-of-stream marker"; - return false; - } - - return true; -} - bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { if (!CanRunRoundTrip()) { m_VideoStatus = std::string(GetCodecName(m_Codec)) + " round trip is not currently supported in this configuration"; return false; } - const bool av1PFrame = m_AV1PFrameVisual && m_AV1PFrameStage == 1; PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { m_VideoStatus = "Failed to generate NV12 source pattern via compute"; return false; } - if (!CopyNv12BufferToTexture(NRI, *m_GraphicsQueue, m_Nv12Layout, *m_UploadBuffer, *m_EncodeTexture, m_VideoWidth, m_VideoHeight)) { - m_VideoStatus = "Failed to upload NV12 source to video texture"; - return false; - } - std::vector annexBHeaders; - if (!WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) - return false; - - if (!SubmitOneTime(NRI, *m_GraphicsQueue, [&](nri::CommandBuffer& commandBuffer) { - NRI.CmdZeroBuffer(commandBuffer, *m_BitstreamBuffer, 0, BITSTREAM_SIZE); - NRI.CmdCopyBuffer(commandBuffer, *m_BitstreamBuffer, 0, *m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); - })) { - m_VideoStatus = std::string("Failed to upload ") + GetCodecName(m_Codec) + " Annex-B parameter sets"; - return false; - } - - nri::VideoEncodePictureDesc pictureDesc = {}; - pictureDesc.frameType = av1PFrame ? nri::VideoEncodeFrameType::P : nri::VideoEncodeFrameType::IDR; - pictureDesc.frameIndex = av1PFrame ? 1 : 0; - pictureDesc.pictureOrderCount = av1PFrame ? 1 : 0; - pictureDesc.idrPictureId = av1PFrame ? 0 : 1; - - uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((m_CodedVideoWidth + 7) >> 3))}; - uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((m_CodedVideoHeight + 7) >> 3))}; - uint16_t av1WidthInSuperblocksMinus1[] = {(uint16_t)(((m_CodedVideoWidth + 63) / 64) - 1)}; - uint16_t av1HeightInSuperblocksMinus1[] = {(uint16_t)(((m_CodedVideoHeight + 63) / 64) - 1)}; - nri::VideoAV1TileLayoutDesc av1TileLayout = {}; - av1TileLayout.columnNum = 1; - av1TileLayout.rowNum = 1; - av1TileLayout.tileSizeBytesMinus1 = 3; - av1TileLayout.uniformSpacing = 1; - av1TileLayout.miColumnStarts = av1MiColumnStarts; - av1TileLayout.miRowStarts = av1MiRowStarts; - av1TileLayout.widthInSuperblocksMinus1 = av1WidthInSuperblocksMinus1; - av1TileLayout.heightInSuperblocksMinus1 = av1HeightInSuperblocksMinus1; - nri::VideoAV1LoopFilterDesc av1LoopFilter = {}; - av1LoopFilter.refDeltas[0] = 1; - av1LoopFilter.refDeltas[4] = -1; - av1LoopFilter.refDeltas[6] = -1; - av1LoopFilter.refDeltas[7] = -1; - nri::VideoAV1CdefDesc av1Cdef = {}; - nri::VideoAV1LoopRestorationDesc av1LoopRestoration = {}; - nri::VideoAV1GlobalMotionDesc av1GlobalMotion = {}; - for (auto& params : av1GlobalMotion.params) { - params[2] = 1 << 16; - params[5] = 1 << 16; - } - nri::VideoAV1PictureDesc av1PictureDesc = {}; - av1PictureDesc.currentFrameId = av1PFrame ? 1 : 0; - av1PictureDesc.orderHint = av1PFrame ? 1 : 0; - av1PictureDesc.refreshFrameFlags = av1PFrame ? 0x1 : 0xFF; - av1PictureDesc.primaryReferenceName = av1PFrame ? nri::VideoAV1ReferenceName::LAST : nri::VideoAV1ReferenceName::NONE; - nri::VideoAV1PictureBits av1CommonPictureFlags = nri::VideoAV1PictureBits::SHOW_FRAME; - if (m_GraphicsAPI == nri::GraphicsAPI::D3D12) - av1CommonPictureFlags |= nri::VideoAV1PictureBits::SEGMENTATION_ENABLED; - av1PictureDesc.flags = av1PFrame ? av1CommonPictureFlags - : av1CommonPictureFlags | nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE; - av1PictureDesc.renderWidthMinus1 = (uint16_t)((m_GraphicsAPI == nri::GraphicsAPI::VK ? m_CodedVideoWidth : m_VideoWidth) - 1); - av1PictureDesc.renderHeightMinus1 = (uint16_t)((m_GraphicsAPI == nri::GraphicsAPI::VK ? m_CodedVideoHeight : m_VideoHeight) - 1); - av1PictureDesc.baseQIndex = (uint8_t)m_AV1BaseQIndex; - av1PictureDesc.interpolationFilter = 0; - av1PictureDesc.txMode = 2; - av1PictureDesc.cdefDampingMinus3 = 3; - av1PictureDesc.tileLayout = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1TileLayout; - av1PictureDesc.loopFilter = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1LoopFilter; - av1PictureDesc.cdef = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1Cdef; - av1PictureDesc.loopRestoration = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1LoopRestoration; - av1PictureDesc.globalMotion = m_GraphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1GlobalMotion; - nri::VideoReference av1Reference = {m_ReconstructedPicture, 0}; - nri::VideoAV1ReferenceDesc av1References[8] = {}; - if (av1PFrame) { - const nri::VideoAV1ReferenceName av1ReferenceNames[] = { - nri::VideoAV1ReferenceName::LAST, - nri::VideoAV1ReferenceName::LAST2, - nri::VideoAV1ReferenceName::LAST3, - nri::VideoAV1ReferenceName::GOLDEN, - nri::VideoAV1ReferenceName::BWDREF, - nri::VideoAV1ReferenceName::ALTREF2, - nri::VideoAV1ReferenceName::ALTREF, - }; - for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { - av1References[i].name = av1ReferenceNames[i]; - av1References[i].refFrameIndex = 0; - av1References[i].frameType = nri::VideoEncodeFrameType::IDR; - av1References[i].orderHint = 0; - av1References[i].frameId = 0; - av1References[i].slot = 0; - } - av1PictureDesc.references = av1References; - av1PictureDesc.referenceNum = helper::GetCountOf(av1ReferenceNames); - } - - nri::VideoEncodeRateControlDesc rateControlDesc = {}; - rateControlDesc.mode = nri::VideoEncodeRateControlMode::CQP; - rateControlDesc.qpI = (uint8_t)m_QpI; - rateControlDesc.qpP = (uint8_t)m_QpP; - rateControlDesc.qpB = (uint8_t)m_QpB; - rateControlDesc.frameRateNumerator = 30; - rateControlDesc.frameRateDenominator = 1; - - nri::VideoEncodeDesc encodeDesc = {}; - encodeDesc.session = m_EncodeSession; - encodeDesc.parameters = m_EncodeParameters; - encodeDesc.srcPicture = m_EncodePicture; - encodeDesc.dstBitstream.buffer = m_BitstreamBuffer; - encodeDesc.dstBitstream.offset = ENCODED_SLICE_OFFSET; - encodeDesc.dstBitstream.size = BITSTREAM_SIZE - ENCODED_SLICE_OFFSET; - encodeDesc.bitstreamMetadataSize = ENCODED_SLICE_OFFSET; - encodeDesc.pictureDesc = &pictureDesc; - encodeDesc.rateControlDesc = &rateControlDesc; - encodeDesc.reconstructedPicture = m_ReconstructedPicture; - if (av1PFrame) { - encodeDesc.reconstructedPicture = m_AV1PReconstructedPicture; - encodeDesc.references = &av1Reference; - encodeDesc.referenceNum = 1; - encodeDesc.reconstructedSlot = 1; - } - encodeDesc.metadata = m_MetadataBuffer; - encodeDesc.resolvedMetadata = m_GraphicsAPI == nri::GraphicsAPI::VK ? m_ResolvedMetadataReadbackBuffer : m_ResolvedMetadataBuffer; - encodeDesc.av1PictureDesc = m_Codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; - - nri::VideoEncodePictureStates srcPictureStates = {}; - nri::VideoEncodePictureStates reconstructedPictureStates = {}; - if (Video.GetVideoEncodePictureStates(*m_EncodePicture, srcPictureStates) != nri::Result::SUCCESS || Video.GetVideoEncodePictureStates(*(av1PFrame ? m_AV1PReconstructedPicture : m_ReconstructedPicture), reconstructedPictureStates) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to query video encode picture states"; - return false; - } - - if (!SubmitOneTime(NRI, *m_VideoEncodeQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::BufferBarrierDesc bufferBarriers[2] = {}; - bufferBarriers[0].buffer = m_MetadataBuffer; - bufferBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[1].buffer = encodeDesc.resolvedMetadata; - bufferBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - - nri::TextureBarrierDesc textureBarriers[3] = {}; - textureBarriers[0].texture = m_EncodeTexture; - textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[0].after = srcPictureStates.encodeRead; - textureBarriers[0].mipNum = nri::REMAINING; - textureBarriers[0].layerNum = nri::REMAINING; - textureBarriers[0].planes = nri::PlaneBits::ALL; - textureBarriers[1].texture = av1PFrame ? m_AV1PReconstructedTexture : m_ReconstructedTexture; - textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].after = reconstructedPictureStates.encodeWrite; - textureBarriers[1].mipNum = nri::REMAINING; - textureBarriers[1].layerNum = nri::REMAINING; - textureBarriers[1].planes = nri::PlaneBits::ALL; - textureBarriers[2].texture = m_ReconstructedTexture; - textureBarriers[2].before = reconstructedPictureStates.graphicsBefore; - textureBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[2].mipNum = nri::REMAINING; - textureBarriers[2].layerNum = nri::REMAINING; - textureBarriers[2].planes = nri::PlaneBits::ALL; - - nri::BarrierDesc barrierDesc = {}; - barrierDesc.buffers = bufferBarriers; - barrierDesc.bufferNum = helper::GetCountOf(bufferBarriers); - barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; - NRI.CmdBarrier(commandBuffer, barrierDesc); - Video.CmdEncodeVideo(commandBuffer, encodeDesc); - // D3D12 resolves encode metadata inside CmdEncodeVideo and transitions the raw metadata buffer to encode-read before returning. - bufferBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[0].after = {}; - bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - bufferBarriers[1].after = {}; - textureBarriers[0].before = srcPictureStates.encodeRead; - textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].before = reconstructedPictureStates.encodeWrite; - textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[2].after = reconstructedPictureStates.afterEncode; - barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = av1PFrame && reconstructedPictureStates.releaseAfterEncode ? helper::GetCountOf(textureBarriers) : 2; - NRI.CmdBarrier(commandBuffer, barrierDesc); - })) { - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode submission failed"; - return false; - } - - if (m_MetadataReadbackPending) - return true; - - if (m_GraphicsAPI == nri::GraphicsAPI::VK) { - m_MetadataReadbackPending = true; - return false; - } - - NRI.ResetCommandAllocator(*m_MetadataReadbackCommandAllocator); - if (NRI.BeginCommandBuffer(*m_MetadataReadbackCommandBuffer, nullptr) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to begin metadata readback command buffer"; + video_sample::EncodeRequest encodeRequest = {}; + encodeRequest.nv12Buffer = m_UploadBuffer; + encodeRequest.nv12Layout = &m_SharedNv12Layout; + encodeRequest.timeSec = timeSec; + if (!m_Encoder->Encode(encodeRequest)) { + m_VideoStatus = m_Encoder->GetStatus(); return false; } - nri::BufferBarrierDesc metadataBarriers[2] = {}; - metadataBarriers[0].buffer = m_ResolvedMetadataBuffer; - metadataBarriers[0].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; - metadataBarriers[0].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; - metadataBarriers[1].buffer = m_ResolvedMetadataReadbackBuffer; - metadataBarriers[1].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - - nri::BarrierDesc metadataBarrierDesc = {}; - metadataBarrierDesc.buffers = metadataBarriers; - metadataBarrierDesc.bufferNum = helper::GetCountOf(metadataBarriers); - NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); - NRI.CmdCopyBuffer(*m_MetadataReadbackCommandBuffer, *m_ResolvedMetadataReadbackBuffer, 0, *m_ResolvedMetadataBuffer, 0, RESOLVED_METADATA_SIZE); - metadataBarriers[0].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; - metadataBarriers[0].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; - metadataBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - metadataBarriers[1].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; - NRI.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); - - if (NRI.EndCommandBuffer(*m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to end metadata readback command buffer"; - return false; - } - - m_MetadataReadbackFenceValue++; - nri::FenceSubmitDesc signalFence = {}; - signalFence.fence = m_MetadataReadbackFence; - signalFence.value = m_MetadataReadbackFenceValue; - - const nri::CommandBuffer* commandBuffers[] = {m_MetadataReadbackCommandBuffer}; - nri::QueueSubmitDesc submit = {}; - submit.commandBuffers = commandBuffers; - submit.commandBufferNum = helper::GetCountOf(commandBuffers); - submit.signalFences = &signalFence; - submit.signalFenceNum = 1; - if (NRI.QueueSubmit(*m_GraphicsQueue, submit) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to submit async metadata readback"; - return false; - } - - m_MetadataReadbackPending = true; + m_VideoStatus = m_Encoder->GetStatus(); return false; } bool Sample::TryDecodePendingMetadata(float timeSec) { - if (!m_MetadataReadbackPending) { - return false; - } - - const uint64_t completedFence = NRI.GetFenceValue(*m_MetadataReadbackFence); - if (completedFence < m_MetadataReadbackFenceValue) { + video_sample::EncodedFrame encodedFrame = {}; + if (!m_Encoder->Poll(encodedFrame)) return false; - } - - m_MetadataReadbackPending = false; - nri::VideoEncodeFeedback feedback = {}; - const nri::Result feedbackResult = Video.GetVideoEncodeFeedback(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, feedback); - if (feedbackResult != nri::Result::SUCCESS) { - if (feedbackResult == nri::Result::UNSUPPORTED) - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " encode metadata feedback is unsupported"; - else - m_VideoStatus = "Failed to read resolved encode metadata"; - return false; - } - - if (feedback.errorFlags || !feedback.encodedBitstreamWrittenBytes) { - char message[160] = {}; - std::snprintf(message, sizeof(message), "Encoder returned errorFlags=0x%llX bytes=%llu", - (unsigned long long)feedback.errorFlags, (unsigned long long)feedback.encodedBitstreamWrittenBytes); - m_VideoStatus = message; - return false; - } - - nri::VideoAV1EncodeDecodeInfo av1DecodeInfo = {}; - if (m_Codec == SampleCodec::AV1) { - nri::VideoAV1EncodeDecodeInfoDesc av1InfoDesc = {}; - av1InfoDesc.feedback = &feedback; - av1InfoDesc.sequence = &m_AV1Sequence; - nri::VideoAV1ReferenceDesc av1InfoReferences[8] = {}; - if (m_AV1PFrameVisual && m_AV1PFrameStage == 1) { - const nri::VideoAV1ReferenceName av1ReferenceNames[] = { - nri::VideoAV1ReferenceName::LAST, - nri::VideoAV1ReferenceName::LAST2, - nri::VideoAV1ReferenceName::LAST3, - nri::VideoAV1ReferenceName::GOLDEN, - nri::VideoAV1ReferenceName::BWDREF, - nri::VideoAV1ReferenceName::ALTREF2, - nri::VideoAV1ReferenceName::ALTREF, - }; - for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { - av1InfoReferences[i].name = av1ReferenceNames[i]; - av1InfoReferences[i].refFrameIndex = 0; - av1InfoReferences[i].frameType = nri::VideoEncodeFrameType::IDR; - av1InfoReferences[i].orderHint = 0; - av1InfoReferences[i].frameId = 0; - av1InfoReferences[i].slot = 0; - } - av1InfoDesc.references = av1InfoReferences; - av1InfoDesc.referenceNum = helper::GetCountOf(av1ReferenceNames); - } - const uint8_t* encodedPayloadHeader = nullptr; - const uint64_t encodedPayloadOffset = ENCODED_SLICE_OFFSET + feedback.encodedBitstreamOffset; - const uint64_t encodedPayloadSize = feedback.encodedBitstreamWrittenBytes; - if (m_GraphicsAPI == nri::GraphicsAPI::VK && encodedPayloadOffset <= BITSTREAM_SIZE && encodedPayloadSize <= BITSTREAM_SIZE - encodedPayloadOffset) { - encodedPayloadHeader = (const uint8_t*)NRI.MapBuffer(*m_BitstreamBuffer, encodedPayloadOffset, encodedPayloadSize); - av1InfoDesc.encodedPayloadHeader = encodedPayloadHeader; - av1InfoDesc.encodedPayloadHeaderSize = encodedPayloadHeader ? encodedPayloadSize : 0; - } - const nri::Result av1InfoResult = Video.GetVideoEncodeAV1DecodeInfo(*m_EncodeSession, *m_ResolvedMetadataReadbackBuffer, 0, av1InfoDesc, av1DecodeInfo); - if (encodedPayloadHeader) - NRI.UnmapBuffer(*m_BitstreamBuffer); - if (av1InfoResult != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to prepare AV1 decode metadata"; - return false; - } - feedback.encodedBitstreamWrittenBytes = av1DecodeInfo.bitstreamOffset + av1DecodeInfo.bitstreamSize; - } - - const bool decoded = DecodeEncodedBitstream(feedback, m_Codec == SampleCodec::AV1 ? &av1DecodeInfo : nullptr, timeSec); - if (decoded && m_AV1PFrameVisual && m_AV1PFrameStage == 0) { - m_AV1PFrameStage = 1; - return TrySubmitEncodeAndMetadataReadback(timeSec); - } - - return decoded; -} - -bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo, float timeSec) { - std::vector annexBHeaders; - if (!av1DecodeInfo && !WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) - return false; - std::vector annexBEndOfStream; - if (!WriteAnnexBEndOfStream(annexBEndOfStream)) - return false; - if (m_GraphicsAPI == nri::GraphicsAPI::VK) - annexBEndOfStream.clear(); - - const uint64_t encodedPayloadSkip = av1DecodeInfo ? av1DecodeInfo->bitstreamOffset : 0; - const uint64_t encodedPayloadBytes = av1DecodeInfo ? av1DecodeInfo->bitstreamSize : feedback.encodedBitstreamWrittenBytes - encodedPayloadSkip; - const uint64_t decodeSliceOffset = annexBHeaders.size(); - const uint64_t decodeBitstreamSize = AlignUp(decodeSliceOffset + encodedPayloadBytes + annexBEndOfStream.size(), m_DecodeBitstreamSizeAlignment); - const uint64_t encodedSourceOffset = ENCODED_SLICE_OFFSET + feedback.encodedBitstreamOffset + encodedPayloadSkip; - if (feedback.encodedBitstreamOffset > BITSTREAM_SIZE - ENCODED_SLICE_OFFSET || encodedSourceOffset > BITSTREAM_SIZE || encodedPayloadBytes > BITSTREAM_SIZE - encodedSourceOffset || decodeBitstreamSize > BITSTREAM_SIZE) { - m_VideoStatus = std::string("Encoded ") + GetCodecName(m_Codec) + " bitstream exceeded decode buffer size"; - return false; - } - - const uint8_t* encodedPayload = (const uint8_t*)NRI.MapBuffer(*m_BitstreamBuffer, encodedSourceOffset, encodedPayloadBytes); - uint8_t* decodeBitstream = (uint8_t*)NRI.MapBuffer(*m_DecodeBitstreamBuffer, 0, decodeBitstreamSize); - if (!encodedPayload || !decodeBitstream) { - if (encodedPayload) - NRI.UnmapBuffer(*m_BitstreamBuffer); - if (decodeBitstream) - NRI.UnmapBuffer(*m_DecodeBitstreamBuffer); - m_VideoStatus = std::string("Failed to map exact ") + GetCodecName(m_Codec) + " decode bitstream"; - return false; - } - std::memset(decodeBitstream, 0, (size_t)decodeBitstreamSize); - if (!annexBHeaders.empty()) - std::memcpy(decodeBitstream, annexBHeaders.data(), annexBHeaders.size()); - const uint64_t encodedCopyBytes = encodedPayloadBytes; - std::memcpy(decodeBitstream + decodeSliceOffset, encodedPayload, (size_t)encodedCopyBytes); - if (!annexBEndOfStream.empty()) - std::memcpy(decodeBitstream + decodeSliceOffset + encodedCopyBytes, annexBEndOfStream.data(), annexBEndOfStream.size()); - const uint64_t decodeBitstreamRange = AlignUp(decodeSliceOffset + encodedCopyBytes + annexBEndOfStream.size(), m_DecodeBitstreamSizeAlignment); - const uint32_t pictureOffsets[] = {(uint32_t)decodeSliceOffset}; - NRI.UnmapBuffer(*m_BitstreamBuffer); - NRI.UnmapBuffer(*m_DecodeBitstreamBuffer); - - const uint32_t decodeFrameIndex = m_DecodeFrameIndex++; - const uint32_t decodeSlot = m_Codec == SampleCodec::AV1 ? 0 : decodeFrameIndex % 16; - - nri::VideoH264DecodePictureDesc h264DecodePicture = {}; - h264DecodePicture.flags = nri::VideoH264DecodePictureBits::IDR | nri::VideoH264DecodePictureBits::INTRA | nri::VideoH264DecodePictureBits::REFERENCE; - h264DecodePicture.sequenceParameterSetId = m_H264Sps.sequenceParameterSetId; - h264DecodePicture.pictureParameterSetId = m_H264Pps.pictureParameterSetId; - h264DecodePicture.frameNum = (uint16_t)(decodeFrameIndex & 0xF); - h264DecodePicture.idrPictureId = (uint16_t)(1 + (decodeFrameIndex & 0xFFFF)); - h264DecodePicture.topFieldOrderCount = 0; - h264DecodePicture.bottomFieldOrderCount = 0; - h264DecodePicture.sliceOffsets = pictureOffsets; - h264DecodePicture.sliceOffsetNum = helper::GetCountOf(pictureOffsets); - h264DecodePicture.referenceSlot = decodeSlot; - - nri::VideoH265DecodePictureDesc h265DecodePicture = {}; - h265DecodePicture.flags = nri::VideoH265DecodePictureBits::IRAP | nri::VideoH265DecodePictureBits::IDR | nri::VideoH265DecodePictureBits::REFERENCE; - h265DecodePicture.videoParameterSetId = m_H265Vps.videoParameterSetId; - h265DecodePicture.sequenceParameterSetId = m_H265Sps.sequenceParameterSetId; - h265DecodePicture.pictureParameterSetId = m_H265Pps.pictureParameterSetId; - h265DecodePicture.pictureOrderCount = (int32_t)decodeFrameIndex; - h265DecodePicture.sliceSegmentOffsets = pictureOffsets; - h265DecodePicture.sliceSegmentOffsetNum = helper::GetCountOf(pictureOffsets); - - nri::VideoAV1EncodeDecodeInfo av1Info = {}; - if (av1DecodeInfo) { - av1Info = *av1DecodeInfo; - av1Info.picture.tiles = av1Info.tiles; - av1Info.picture.tileLayout = &av1Info.tileLayout; - av1Info.picture.quantization = &av1Info.quantization; - av1Info.picture.loopFilter = &av1Info.loopFilter; - av1Info.picture.cdef = &av1Info.cdef; - av1Info.picture.segmentation = av1DecodeInfo->picture.segmentation ? &av1Info.segmentation : nullptr; - av1Info.picture.loopRestoration = &av1Info.loopRestoration; - av1Info.picture.globalMotion = &av1Info.globalMotion; - av1Info.tileLayout.miColumnStarts = av1Info.miColumnStarts; - av1Info.tileLayout.miRowStarts = av1Info.miRowStarts; - av1Info.tileLayout.widthInSuperblocksMinus1 = av1Info.widthInSuperblocksMinus1; - av1Info.tileLayout.heightInSuperblocksMinus1 = av1Info.heightInSuperblocksMinus1; - if (av1Info.picture.references && av1Info.picture.referenceNum) - av1Info.picture.references = av1Info.references; - } - nri::VideoReference av1DecodeReference = {m_DecodePicture, 0}; - uint8_t av1DecodeOrderHints[8] = {}; - const bool av1PFrame = m_AV1PFrameVisual && m_AV1PFrameStage == 1; - if (av1PFrame) { - av1Info.picture.orderHints = av1DecodeOrderHints; - for (uint32_t i = 0; i < av1Info.picture.referenceNum && i < helper::GetCountOf(av1Info.references); i++) - av1Info.references[i].savedOrderHints = av1DecodeOrderHints; - } - - nri::VideoDecodeDesc decodeDesc = {}; - decodeDesc.session = m_DecodeSession; - decodeDesc.parameters = m_DecodeParameters; - decodeDesc.bitstream.buffer = m_DecodeBitstreamBuffer; - decodeDesc.bitstream.size = decodeBitstreamRange; - decodeDesc.dstPicture = av1PFrame ? m_AV1PDecodePicture : m_DecodePicture; - decodeDesc.references = av1PFrame ? &av1DecodeReference : nullptr; - decodeDesc.referenceNum = av1PFrame ? 1u : 0u; - decodeDesc.dstSlot = av1PFrame ? 1u : decodeSlot; - decodeDesc.h264PictureDesc = m_Codec == SampleCodec::H264 ? &h264DecodePicture : nullptr; - decodeDesc.h265PictureDesc = m_Codec == SampleCodec::H265 ? &h265DecodePicture : nullptr; - decodeDesc.av1PictureDesc = av1DecodeInfo ? &av1Info.picture : nullptr; - - nri::VideoDecodePictureStates decodePictureStates = {}; - if (Video.GetVideoDecodePictureStates(*(av1PFrame ? m_AV1PDecodePicture : m_DecodePicture), decodePictureStates) != nri::Result::SUCCESS) { - m_VideoStatus = "Failed to query video decode picture states"; - return false; - } - if (m_GraphicsAPI == nri::GraphicsAPI::VK && !av1DecodeInfo) { - decodePictureStates.decodeWrite = {nri::AccessBits::VIDEO_DECODE_WRITE, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; - decodePictureStates.graphicsBefore = {nri::AccessBits::VIDEO_DECODE, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; - } - - if (!SubmitOneTime(NRI, *m_VideoDecodeQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::BufferBarrierDesc bufferBarrier = {}; - bufferBarrier.buffer = m_DecodeBitstreamBuffer; - bufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; - bufferBarrier.after = {nri::AccessBits::VIDEO_DECODE_READ, nri::StageBits::VIDEO_DECODE}; - - nri::TextureBarrierDesc textureBarriers[3] = {}; - uint32_t textureBarrierNum = 0; - textureBarriers[textureBarrierNum].texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; - textureBarriers[textureBarrierNum].before = {nri::AccessBits::NONE, m_GraphicsAPI == nri::GraphicsAPI::VK && !av1DecodeInfo ? nri::Layout::UNDEFINED : nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[textureBarrierNum].after = decodePictureStates.decodeWrite; - textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; - textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; - textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; - textureBarrierNum++; - if (av1PFrame) { - textureBarriers[textureBarrierNum].texture = m_DecodeTexture; - textureBarriers[textureBarrierNum].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; - textureBarriers[textureBarrierNum].after = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; - textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; - textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; - textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; - textureBarrierNum++; - } - - nri::BarrierDesc barrierDesc = {}; - barrierDesc.buffers = m_GraphicsAPI == nri::GraphicsAPI::D3D12 ? nullptr : &bufferBarrier; - barrierDesc.bufferNum = m_GraphicsAPI == nri::GraphicsAPI::D3D12 ? 0 : 1; - barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = textureBarrierNum; - NRI.CmdBarrier(commandBuffer, barrierDesc); - Video.CmdDecodeVideo(commandBuffer, decodeDesc); - - if (decodePictureStates.releaseAfterDecode) { - textureBarriers[0].before = decodePictureStates.decodeWrite; - textureBarriers[0].after = decodePictureStates.afterDecode; - if (av1PFrame) { - textureBarriers[1].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; - textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - } - barrierDesc.buffers = nullptr; - barrierDesc.bufferNum = 0; - NRI.CmdBarrier(commandBuffer, barrierDesc); - } - })) { - m_VideoStatus = std::string(GetCodecName(m_Codec)) + " decode submission failed"; - return false; - } - - nri::Queue* readbackQueue = m_GraphicsAPI == nri::GraphicsAPI::VK ? m_VideoDecodeQueue : m_GraphicsQueue; - if (!SubmitOneTime(NRI, *readbackQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::TextureBarrierDesc textureBarrier = {}; - textureBarrier.texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; - textureBarrier.before = decodePictureStates.graphicsBefore; - textureBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; - textureBarrier.mipNum = nri::REMAINING; - textureBarrier.layerNum = nri::REMAINING; - textureBarrier.planes = nri::PlaneBits::ALL; - - nri::BarrierDesc barrierDesc = {}; - barrierDesc.textures = &textureBarrier; - barrierDesc.textureNum = 1; - NRI.CmdBarrier(commandBuffer, barrierDesc); - - nri::BufferBarrierDesc nv12BufferBarrier = {}; - nv12BufferBarrier.buffer = m_UploadBuffer; - nv12BufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; - nv12BufferBarrier.after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - - nri::BarrierDesc copyBarrierDesc = {}; - copyBarrierDesc.buffers = &nv12BufferBarrier; - copyBarrierDesc.bufferNum = 1; - NRI.CmdBarrier(commandBuffer, copyBarrierDesc); - - nri::TextureRegionDesc lumaRegion = {}; - lumaRegion.width = (nri::Dim_t)m_VideoWidth; - lumaRegion.height = (nri::Dim_t)m_VideoHeight; - lumaRegion.depth = 1; - lumaRegion.planes = nri::PlaneBits::PLANE_0; - - nri::TextureDataLayoutDesc lumaLayout = {}; - lumaLayout.rowPitch = m_Nv12Layout.yRowPitchBytes; - lumaLayout.slicePitch = m_Nv12Layout.ySlicePitchBytes; - NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, lumaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), lumaRegion); - - nri::TextureRegionDesc chromaRegion = {}; - chromaRegion.width = (nri::Dim_t)m_VideoWidth; - chromaRegion.height = (nri::Dim_t)m_VideoHeight; - chromaRegion.depth = 1; - chromaRegion.planes = nri::PlaneBits::PLANE_1; - - nri::TextureDataLayoutDesc chromaLayout = {}; - chromaLayout.offset = m_Nv12Layout.uvOffsetBytes; - chromaLayout.rowPitch = m_Nv12Layout.uvRowPitchBytes; - chromaLayout.slicePitch = m_Nv12Layout.uvSlicePitchBytes; - NRI.CmdReadbackTextureToBuffer(commandBuffer, *m_UploadBuffer, chromaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), chromaRegion); - - nv12BufferBarrier.before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; - nv12BufferBarrier.after = {nri::AccessBits::NONE, nri::StageBits::NONE}; - NRI.CmdBarrier(commandBuffer, copyBarrierDesc); - - textureBarrier.before = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; - textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - NRI.CmdBarrier(commandBuffer, barrierDesc); - })) { - m_VideoStatus = "Failed to copy decoded NV12 for preview"; + video_sample::DecodedFrame decodedFrame = {}; + video_sample::DecodeRequest decodeRequest = {}; + decodeRequest.frame = &encodedFrame; + decodeRequest.nv12ReadbackBuffer = m_UploadBuffer; + decodeRequest.nv12Layout = &m_SharedNv12Layout; + if (!m_Decoder->Decode(decodeRequest, decodedFrame)) { + m_VideoStatus = m_Decoder->GetStatus(); return false; } @@ -2022,24 +944,24 @@ bool Sample::DecodeEncodedBitstream(const nri::VideoEncodeFeedback& feedback, co } m_DecodePreviewReady = true; - if (av1PFrame) - m_AV1PFrameStage = 0; + if (m_AV1PFrameVisual && !encodedFrame.isAv1PFrame) + return TrySubmitEncodeAndMetadataReadback(timeSec); char message[128] = {}; - std::snprintf(message, sizeof(message), "%s encode/decode round trip complete, encoded %llu bytes, EOS %llu bytes", GetCodecName(m_Codec), (unsigned long long)feedback.encodedBitstreamWrittenBytes, (unsigned long long)annexBEndOfStream.size()); + std::snprintf(message, sizeof(message), "%s encode/decode round trip complete, encoded %llu bytes", GetCodecName(m_Codec), (unsigned long long)encodedFrame.feedback.encodedBitstreamWrittenBytes); m_VideoStatus = message; return true; } bool Sample::TryRunRoundTrip(float timeSec) { - if (m_MetadataReadbackPending) + if (m_Encoder && m_Encoder->HasPendingFeedback()) return TryDecodePendingMetadata(timeSec); return TrySubmitEncodeAndMetadataReadback(timeSec); } bool Sample::CanRunRoundTrip() const { - return m_VideoReady && m_GraphicsQueue && m_VideoEncodeQueue && m_VideoDecodeQueue && m_UploadBuffer && m_UploadBufferView && m_SourcePreviewStorage && m_DecodePreviewStorage && m_GeneratePipelineLayout && m_GenerateComputePipeline && m_GenerateDescriptorSet; + return m_VideoReady && m_Encoder && m_Encoder->IsReady() && m_Decoder && m_Decoder->IsReady() && m_GraphicsQueue && m_VideoEncodeQueue && m_VideoDecodeQueue && m_UploadBuffer && m_UploadBufferView && m_SourcePreviewStorage && m_DecodePreviewStorage && m_GeneratePipelineLayout && m_GenerateComputePipeline && m_GenerateDescriptorSet; } void Sample::LatencySleep(uint32_t frameIndex) { diff --git a/Source/VideoEncodeDecode/Decoder.cpp b/Source/VideoEncodeDecode/Decoder.cpp new file mode 100644 index 0000000..58a7fb4 --- /dev/null +++ b/Source/VideoEncodeDecode/Decoder.cpp @@ -0,0 +1,474 @@ +// © 2021 NVIDIA Corporation + +#include "Decoder.h" + +#include + +namespace video_sample { + +Decoder::~Decoder() { + if (!m_Context.nri) + return; + + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + if (video.DestroyVideoPicture) { + if (m_AV1PDecodePicture) + video.DestroyVideoPicture(m_AV1PDecodePicture); + if (m_DecodePicture) + video.DestroyVideoPicture(m_DecodePicture); + if (m_Parameters) + video.DestroyVideoSessionParameters(m_Parameters); + if (m_Session) + video.DestroyVideoSession(m_Session); + } + + if (m_DecodeBitstreamBuffer) + nri.DestroyBuffer(m_DecodeBitstreamBuffer); + if (m_DecodeTexture) + nri.DestroyTexture(m_DecodeTexture); + if (m_AV1PDecodeTexture) + nri.DestroyTexture(m_AV1PDecodeTexture); +} + +bool Decoder::Initialize(const VideoContext& context, const VideoConfig& config, const VideoSize& size, const CodecParameters& codecParameters) { + m_Context = context; + m_Config = config; + m_Size = size; + m_CodecParameters = codecParameters; + + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + nri::VideoSessionDesc sessionDesc = {}; + sessionDesc.type = nri::VideoSessionType::DECODE; + sessionDesc.codec = GetNriCodec(m_Config.codec); + sessionDesc.format = nri::Format::NV12_UNORM; + sessionDesc.width = m_Size.codedWidth; + sessionDesc.height = m_Size.codedHeight; + sessionDesc.maxReferenceNum = m_Config.codec == SampleCodec::AV1 ? 1 : 16; + + if (video.CreateVideoSession(*m_Context.device, sessionDesc, m_Session) != nri::Result::SUCCESS) { + m_Status = std::string("Failed to create ") + GetCodecName(m_Config.codec) + " decode session"; + return false; + } + + nri::VideoH264SessionParametersDesc h264Parameters = {}; + h264Parameters.sequenceParameterSets = &m_CodecParameters.h264Sps; + h264Parameters.sequenceParameterSetNum = 1; + h264Parameters.pictureParameterSets = &m_CodecParameters.h264Pps; + h264Parameters.pictureParameterSetNum = 1; + h264Parameters.maxSequenceParameterSetNum = 1; + h264Parameters.maxPictureParameterSetNum = 1; + + nri::VideoH265SessionParametersDesc h265Parameters = {}; + h265Parameters.videoParameterSets = &m_CodecParameters.h265Vps; + h265Parameters.videoParameterSetNum = 1; + h265Parameters.sequenceParameterSets = &m_CodecParameters.h265Sps; + h265Parameters.sequenceParameterSetNum = 1; + h265Parameters.pictureParameterSets = &m_CodecParameters.h265Pps; + h265Parameters.pictureParameterSetNum = 1; + h265Parameters.maxVideoParameterSetNum = 1; + h265Parameters.maxSequenceParameterSetNum = 1; + h265Parameters.maxPictureParameterSetNum = 1; + + nri::VideoAV1SessionParametersDesc av1Parameters = {}; + av1Parameters.sequence = m_CodecParameters.av1Sequence; + + nri::VideoSessionParametersDesc parametersDesc = {}; + parametersDesc.session = m_Session; + parametersDesc.h264Parameters = m_Config.codec == SampleCodec::H264 ? &h264Parameters : nullptr; + parametersDesc.h265Parameters = m_Config.codec == SampleCodec::H265 ? &h265Parameters : nullptr; + parametersDesc.av1Parameters = m_Config.codec == SampleCodec::AV1 ? &av1Parameters : nullptr; + + if (video.CreateVideoSessionParameters(*m_Context.device, parametersDesc, m_Parameters) != nri::Result::SUCCESS) { + m_Status = std::string("Failed to create ") + GetCodecName(m_Config.codec) + " decode parameters"; + return false; + } + + nri::TextureDesc textureDesc = {}; + textureDesc.type = nri::TextureType::TEXTURE_2D; + textureDesc.usage = nri::TextureUsageBits::VIDEO_DECODE; + textureDesc.format = nri::Format::NV12_UNORM; + textureDesc.width = (nri::Dim_t)m_Size.codedWidth; + textureDesc.height = (nri::Dim_t)m_Size.codedHeight; + textureDesc.mipNum = 1; + textureDesc.layerNum = 1; + textureDesc.videoCodec = GetNriCodec(m_Config.codec); + + if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_DecodeTexture) != nri::Result::SUCCESS) { + m_Status = "Failed to create NV12 decode texture"; + return false; + } + nri.SetDebugName(m_DecodeTexture, "VideoDecodeTexture"); + + if (m_Config.av1PFrameVisual) { + if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_AV1PDecodeTexture) != nri::Result::SUCCESS) { + m_Status = "Failed to create second NV12 decode texture"; + return false; + } + nri.SetDebugName(m_AV1PDecodeTexture, "VideoAV1PDecodeTexture"); + } + + if (!SubmitOneTime(nri, *m_Context.graphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::TextureBarrierDesc textureBarriers[2] = {}; + uint32_t textureBarrierNum = 0; + textureBarriers[textureBarrierNum++].texture = m_DecodeTexture; + if (m_AV1PDecodeTexture) + textureBarriers[textureBarrierNum++].texture = m_AV1PDecodeTexture; + + for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { + if (!textureBarrier.texture) + continue; + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::UNDEFINED, nri::StageBits::ALL}; + textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + } + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = textureBarrierNum; + nri.CmdBarrier(commandBuffer, barrierDesc); + })) { + m_Status = "Failed to initialize decode texture layouts"; + return false; + } + + nri::BufferDesc decodeBitstreamBufferDesc = {}; + decodeBitstreamBufferDesc.size = BITSTREAM_SIZE; + decodeBitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_DECODE; + if (CreateDecodeBitstreamBuffer(nri, *m_Context.device, 0.0f, decodeBitstreamBufferDesc, m_DecodeBitstreamBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to create decode bitstream buffer"; + return false; + } + + nri::VideoPictureDesc decodePictureDesc = {}; + decodePictureDesc.texture = m_DecodeTexture; + decodePictureDesc.usage = nri::VideoPictureUsage::DECODE_OUTPUT; + decodePictureDesc.width = (nri::Dim_t)m_Size.codedWidth; + decodePictureDesc.height = (nri::Dim_t)m_Size.codedHeight; + + if (video.CreateVideoPicture(*m_Context.device, decodePictureDesc, m_DecodePicture) != nri::Result::SUCCESS) { + m_Status = "Failed to create decode picture"; + return false; + } + if (m_Config.av1PFrameVisual) { + decodePictureDesc.texture = m_AV1PDecodeTexture; + if (video.CreateVideoPicture(*m_Context.device, decodePictureDesc, m_AV1PDecodePicture) != nri::Result::SUCCESS) { + m_Status = "Failed to create second decode picture"; + return false; + } + } + + m_Ready = true; + m_Status = std::string(GetCodecName(m_Config.codec)) + " decoder initialized"; + return true; +} + +bool Decoder::WriteAnnexBEndOfStream(std::vector& annexBEndOfStream) { + annexBEndOfStream.clear(); + if (m_Config.codec == SampleCodec::AV1) + return true; + + nri::VideoInterface& video = *m_Context.video; + nri::VideoAnnexBEndOfStreamDesc annexBDesc = {}; + annexBDesc.codec = GetNriCodec(m_Config.codec); + if (video.WriteVideoAnnexBEndOfStream(annexBDesc) != nri::Result::SUCCESS || annexBDesc.writtenSize == 0) { + m_Status = std::string("Failed to query ") + GetCodecName(m_Config.codec) + " Annex-B end-of-stream size"; + return false; + } + + annexBEndOfStream.resize((size_t)annexBDesc.writtenSize); + annexBDesc.dst = annexBEndOfStream.data(); + annexBDesc.dstSize = annexBEndOfStream.size(); + if (video.WriteVideoAnnexBEndOfStream(annexBDesc) != nri::Result::SUCCESS) { + m_Status = std::string("Failed to build ") + GetCodecName(m_Config.codec) + " Annex-B end-of-stream marker"; + return false; + } + + return true; +} + +bool Decoder::BuildDecodeBitstream(const EncodedFrame& frame, uint64_t& decodeBitstreamRange, uint32_t& pictureOffset) { + NRIInterface& nri = *m_Context.nri; + + std::vector annexBHeaders; + if (!frame.hasAv1DecodeInfo) { + nri::VideoInterface& video = *m_Context.video; + nri::VideoAnnexBParameterSetsDesc annexBDesc = {}; + annexBDesc.codec = GetNriCodec(m_Config.codec); + annexBDesc.h264Sps = &m_CodecParameters.h264Sps; + annexBDesc.h264Pps = &m_CodecParameters.h264Pps; + annexBDesc.h265Vps = &m_CodecParameters.h265Vps; + annexBDesc.h265Sps = &m_CodecParameters.h265Sps; + annexBDesc.h265Pps = &m_CodecParameters.h265Pps; + if (video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS || annexBDesc.writtenSize == 0 || annexBDesc.writtenSize >= ENCODED_SLICE_OFFSET) { + m_Status = std::string("Failed to query ") + GetCodecName(m_Config.codec) + " Annex-B parameter-set size"; + return false; + } + annexBHeaders.resize((size_t)annexBDesc.writtenSize); + annexBDesc.dst = annexBHeaders.data(); + annexBDesc.dstSize = annexBHeaders.size(); + if (video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS) { + m_Status = std::string("Failed to build ") + GetCodecName(m_Config.codec) + " Annex-B parameter sets"; + return false; + } + } + + std::vector annexBEndOfStream; + if (!WriteAnnexBEndOfStream(annexBEndOfStream)) + return false; + if (m_Context.graphicsAPI == nri::GraphicsAPI::VK) + annexBEndOfStream.clear(); + + const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo = frame.hasAv1DecodeInfo ? &frame.av1DecodeInfo : nullptr; + const uint64_t encodedPayloadSkip = av1DecodeInfo ? av1DecodeInfo->bitstreamOffset : 0; + const uint64_t encodedPayloadBytes = av1DecodeInfo ? av1DecodeInfo->bitstreamSize : frame.feedback.encodedBitstreamWrittenBytes - encodedPayloadSkip; + const uint64_t decodeSliceOffset = annexBHeaders.size(); + pictureOffset = (uint32_t)decodeSliceOffset; + const uint64_t decodeBitstreamSize = AlignUp(decodeSliceOffset + encodedPayloadBytes + annexBEndOfStream.size(), m_Size.decodeBitstreamSizeAlignment); + const uint64_t encodedSourceOffset = ENCODED_SLICE_OFFSET + frame.feedback.encodedBitstreamOffset + encodedPayloadSkip; + if (frame.feedback.encodedBitstreamOffset > BITSTREAM_SIZE - ENCODED_SLICE_OFFSET || encodedSourceOffset > BITSTREAM_SIZE || encodedPayloadBytes > BITSTREAM_SIZE - encodedSourceOffset || decodeBitstreamSize > BITSTREAM_SIZE) { + m_Status = std::string("Encoded ") + GetCodecName(m_Config.codec) + " bitstream exceeded decode buffer size"; + return false; + } + + const uint8_t* encodedPayload = (const uint8_t*)nri.MapBuffer(*frame.bitstreamBuffer, encodedSourceOffset, encodedPayloadBytes); + uint8_t* decodeBitstream = (uint8_t*)nri.MapBuffer(*m_DecodeBitstreamBuffer, 0, decodeBitstreamSize); + if (!encodedPayload || !decodeBitstream) { + if (encodedPayload) + nri.UnmapBuffer(*frame.bitstreamBuffer); + if (decodeBitstream) + nri.UnmapBuffer(*m_DecodeBitstreamBuffer); + m_Status = std::string("Failed to map exact ") + GetCodecName(m_Config.codec) + " decode bitstream"; + return false; + } + std::memset(decodeBitstream, 0, (size_t)decodeBitstreamSize); + if (!annexBHeaders.empty()) + std::memcpy(decodeBitstream, annexBHeaders.data(), annexBHeaders.size()); + std::memcpy(decodeBitstream + decodeSliceOffset, encodedPayload, (size_t)encodedPayloadBytes); + if (!annexBEndOfStream.empty()) + std::memcpy(decodeBitstream + decodeSliceOffset + encodedPayloadBytes, annexBEndOfStream.data(), annexBEndOfStream.size()); + decodeBitstreamRange = AlignUp(decodeSliceOffset + encodedPayloadBytes + annexBEndOfStream.size(), m_Size.decodeBitstreamSizeAlignment); + nri.UnmapBuffer(*frame.bitstreamBuffer); + nri.UnmapBuffer(*m_DecodeBitstreamBuffer); + return true; +} + +bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { + if (!m_Ready || !request.frame || !request.nv12ReadbackBuffer || !request.nv12Layout) { + m_Status = std::string(GetCodecName(m_Config.codec)) + " decoder is not ready"; + return false; + } + + const EncodedFrame& frame = *request.frame; + const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo = frame.hasAv1DecodeInfo ? &frame.av1DecodeInfo : nullptr; + const bool av1PFrame = frame.isAv1PFrame; + + uint64_t decodeBitstreamRange = 0; + uint32_t pictureOffset = 0; + if (!BuildDecodeBitstream(frame, decodeBitstreamRange, pictureOffset)) + return false; + + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + const uint32_t decodeFrameIndex = m_DecodeFrameIndex++; + const uint32_t decodeSlot = m_Config.codec == SampleCodec::AV1 ? 0 : decodeFrameIndex % 16; + const uint32_t pictureOffsets[] = {pictureOffset}; + + nri::VideoH264DecodePictureDesc h264DecodePicture = {}; + h264DecodePicture.flags = nri::VideoH264DecodePictureBits::IDR | nri::VideoH264DecodePictureBits::INTRA | nri::VideoH264DecodePictureBits::REFERENCE; + h264DecodePicture.sequenceParameterSetId = m_CodecParameters.h264Sps.sequenceParameterSetId; + h264DecodePicture.pictureParameterSetId = m_CodecParameters.h264Pps.pictureParameterSetId; + h264DecodePicture.frameNum = (uint16_t)(decodeFrameIndex & 0xF); + h264DecodePicture.idrPictureId = (uint16_t)(1 + (decodeFrameIndex & 0xFFFF)); + h264DecodePicture.topFieldOrderCount = 0; + h264DecodePicture.bottomFieldOrderCount = 0; + h264DecodePicture.sliceOffsets = pictureOffsets; + h264DecodePicture.sliceOffsetNum = helper::GetCountOf(pictureOffsets); + h264DecodePicture.referenceSlot = decodeSlot; + + nri::VideoH265DecodePictureDesc h265DecodePicture = {}; + h265DecodePicture.flags = nri::VideoH265DecodePictureBits::IRAP | nri::VideoH265DecodePictureBits::IDR | nri::VideoH265DecodePictureBits::REFERENCE; + h265DecodePicture.videoParameterSetId = m_CodecParameters.h265Vps.videoParameterSetId; + h265DecodePicture.sequenceParameterSetId = m_CodecParameters.h265Sps.sequenceParameterSetId; + h265DecodePicture.pictureParameterSetId = m_CodecParameters.h265Pps.pictureParameterSetId; + h265DecodePicture.pictureOrderCount = (int32_t)decodeFrameIndex; + h265DecodePicture.sliceSegmentOffsets = pictureOffsets; + h265DecodePicture.sliceSegmentOffsetNum = helper::GetCountOf(pictureOffsets); + + nri::VideoAV1EncodeDecodeInfo av1Info = {}; + if (av1DecodeInfo) { + av1Info = *av1DecodeInfo; + av1Info.picture.tiles = av1Info.tiles; + av1Info.picture.tileLayout = &av1Info.tileLayout; + av1Info.picture.quantization = &av1Info.quantization; + av1Info.picture.loopFilter = &av1Info.loopFilter; + av1Info.picture.cdef = &av1Info.cdef; + av1Info.picture.segmentation = av1DecodeInfo->picture.segmentation ? &av1Info.segmentation : nullptr; + av1Info.picture.loopRestoration = &av1Info.loopRestoration; + av1Info.picture.globalMotion = &av1Info.globalMotion; + av1Info.tileLayout.miColumnStarts = av1Info.miColumnStarts; + av1Info.tileLayout.miRowStarts = av1Info.miRowStarts; + av1Info.tileLayout.widthInSuperblocksMinus1 = av1Info.widthInSuperblocksMinus1; + av1Info.tileLayout.heightInSuperblocksMinus1 = av1Info.heightInSuperblocksMinus1; + if (av1Info.picture.references && av1Info.picture.referenceNum) + av1Info.picture.references = av1Info.references; + } + nri::VideoReference av1DecodeReference = {m_DecodePicture, 0}; + uint8_t av1DecodeOrderHints[8] = {}; + if (av1PFrame) { + av1Info.picture.orderHints = av1DecodeOrderHints; + for (uint32_t i = 0; i < av1Info.picture.referenceNum && i < helper::GetCountOf(av1Info.references); i++) + av1Info.references[i].savedOrderHints = av1DecodeOrderHints; + } + + nri::VideoDecodeDesc decodeDesc = {}; + decodeDesc.session = m_Session; + decodeDesc.parameters = m_Parameters; + decodeDesc.bitstream.buffer = m_DecodeBitstreamBuffer; + decodeDesc.bitstream.size = decodeBitstreamRange; + decodeDesc.dstPicture = av1PFrame ? m_AV1PDecodePicture : m_DecodePicture; + decodeDesc.references = av1PFrame ? &av1DecodeReference : nullptr; + decodeDesc.referenceNum = av1PFrame ? 1u : 0u; + decodeDesc.dstSlot = av1PFrame ? 1u : decodeSlot; + decodeDesc.h264PictureDesc = m_Config.codec == SampleCodec::H264 ? &h264DecodePicture : nullptr; + decodeDesc.h265PictureDesc = m_Config.codec == SampleCodec::H265 ? &h265DecodePicture : nullptr; + decodeDesc.av1PictureDesc = av1DecodeInfo ? &av1Info.picture : nullptr; + + nri::VideoDecodePictureStates decodePictureStates = {}; + if (video.GetVideoDecodePictureStates(*(av1PFrame ? m_AV1PDecodePicture : m_DecodePicture), decodePictureStates) != nri::Result::SUCCESS) { + m_Status = "Failed to query video decode picture states"; + return false; + } + if (m_Context.graphicsAPI == nri::GraphicsAPI::VK && !av1DecodeInfo) { + decodePictureStates.decodeWrite = {nri::AccessBits::VIDEO_DECODE_WRITE, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + decodePictureStates.graphicsBefore = {nri::AccessBits::VIDEO_DECODE, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + } + + if (!SubmitOneTime(nri, *m_Context.decodeQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::BufferBarrierDesc bufferBarrier = {}; + bufferBarrier.buffer = m_DecodeBitstreamBuffer; + bufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + bufferBarrier.after = {nri::AccessBits::VIDEO_DECODE_READ, nri::StageBits::VIDEO_DECODE}; + + nri::TextureBarrierDesc textureBarriers[3] = {}; + uint32_t textureBarrierNum = 0; + textureBarriers[textureBarrierNum].texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; + textureBarriers[textureBarrierNum].before = {nri::AccessBits::NONE, m_Context.graphicsAPI == nri::GraphicsAPI::VK && !av1DecodeInfo ? nri::Layout::UNDEFINED : nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[textureBarrierNum].after = decodePictureStates.decodeWrite; + textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; + textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; + textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; + textureBarrierNum++; + if (av1PFrame) { + textureBarriers[textureBarrierNum].texture = m_DecodeTexture; + textureBarriers[textureBarrierNum].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[textureBarrierNum].after = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; + textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; + textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; + textureBarrierNum++; + } + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = m_Context.graphicsAPI == nri::GraphicsAPI::D3D12 ? nullptr : &bufferBarrier; + barrierDesc.bufferNum = m_Context.graphicsAPI == nri::GraphicsAPI::D3D12 ? 0 : 1; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = textureBarrierNum; + nri.CmdBarrier(commandBuffer, barrierDesc); + video.CmdDecodeVideo(commandBuffer, decodeDesc); + + if (decodePictureStates.releaseAfterDecode) { + textureBarriers[0].before = decodePictureStates.decodeWrite; + textureBarriers[0].after = decodePictureStates.afterDecode; + if (av1PFrame) { + textureBarriers[1].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + } + barrierDesc.buffers = nullptr; + barrierDesc.bufferNum = 0; + nri.CmdBarrier(commandBuffer, barrierDesc); + } + })) { + m_Status = std::string(GetCodecName(m_Config.codec)) + " decode submission failed"; + return false; + } + + nri::Queue* readbackQueue = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_Context.decodeQueue : m_Context.graphicsQueue; + if (!SubmitOneTime(nri, *readbackQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::TextureBarrierDesc textureBarrier = {}; + textureBarrier.texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; + textureBarrier.before = decodePictureStates.graphicsBefore; + textureBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textures = &textureBarrier; + barrierDesc.textureNum = 1; + nri.CmdBarrier(commandBuffer, barrierDesc); + + nri::BufferBarrierDesc nv12BufferBarrier = {}; + nv12BufferBarrier.buffer = request.nv12ReadbackBuffer; + nv12BufferBarrier.before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + nv12BufferBarrier.after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + + nri::BarrierDesc copyBarrierDesc = {}; + copyBarrierDesc.buffers = &nv12BufferBarrier; + copyBarrierDesc.bufferNum = 1; + nri.CmdBarrier(commandBuffer, copyBarrierDesc); + + nri::TextureRegionDesc lumaRegion = {}; + lumaRegion.width = (nri::Dim_t)m_Size.videoWidth; + lumaRegion.height = (nri::Dim_t)m_Size.videoHeight; + lumaRegion.depth = 1; + lumaRegion.planes = nri::PlaneBits::PLANE_0; + + nri::TextureDataLayoutDesc lumaLayout = {}; + lumaLayout.rowPitch = request.nv12Layout->yRowPitchBytes; + lumaLayout.slicePitch = request.nv12Layout->ySlicePitchBytes; + nri.CmdReadbackTextureToBuffer(commandBuffer, *request.nv12ReadbackBuffer, lumaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), lumaRegion); + + nri::TextureRegionDesc chromaRegion = {}; + chromaRegion.width = (nri::Dim_t)m_Size.videoWidth; + chromaRegion.height = (nri::Dim_t)m_Size.videoHeight; + chromaRegion.depth = 1; + chromaRegion.planes = nri::PlaneBits::PLANE_1; + + nri::TextureDataLayoutDesc chromaLayout = {}; + chromaLayout.offset = request.nv12Layout->uvOffsetBytes; + chromaLayout.rowPitch = request.nv12Layout->uvRowPitchBytes; + chromaLayout.slicePitch = request.nv12Layout->uvSlicePitchBytes; + nri.CmdReadbackTextureToBuffer(commandBuffer, *request.nv12ReadbackBuffer, chromaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), chromaRegion); + + nv12BufferBarrier.before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + nv12BufferBarrier.after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + nri.CmdBarrier(commandBuffer, copyBarrierDesc); + + textureBarrier.before = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; + textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + nri.CmdBarrier(commandBuffer, barrierDesc); + })) { + m_Status = "Failed to copy decoded NV12 for preview"; + return false; + } + + decodedFrame = {}; + decodedFrame.texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; + decodedFrame.picture = av1PFrame ? m_AV1PDecodePicture : m_DecodePicture; + decodedFrame.pictureStates = decodePictureStates; + decodedFrame.isAv1PFrame = av1PFrame; + + char message[128] = {}; + std::snprintf(message, sizeof(message), "%s decode complete, encoded %llu bytes", GetCodecName(m_Config.codec), (unsigned long long)frame.feedback.encodedBitstreamWrittenBytes); + m_Status = message; + return true; +} + +} // namespace video_sample diff --git a/Source/VideoEncodeDecode/Decoder.h b/Source/VideoEncodeDecode/Decoder.h new file mode 100644 index 0000000..b3b34b7 --- /dev/null +++ b/Source/VideoEncodeDecode/Decoder.h @@ -0,0 +1,57 @@ +// © 2021 NVIDIA Corporation + +#pragma once + +#include "Shared.h" + +namespace video_sample { + +struct DecodeRequest { + const EncodedFrame* frame = nullptr; + nri::Buffer* nv12ReadbackBuffer = nullptr; + const Nv12BufferLayout* nv12Layout = nullptr; +}; + +class Decoder { +public: + Decoder() = default; + ~Decoder(); + + Decoder(const Decoder&) = delete; + Decoder& operator=(const Decoder&) = delete; + + bool Initialize(const VideoContext& context, const VideoConfig& config, const VideoSize& size, const CodecParameters& codecParameters); + bool Decode(const DecodeRequest& request, DecodedFrame& decodedFrame); + + bool IsReady() const { + return m_Ready; + } + + const std::string& GetStatus() const { + return m_Status; + } + +private: + bool WriteAnnexBEndOfStream(std::vector& annexBEndOfStream); + bool BuildDecodeBitstream(const EncodedFrame& frame, uint64_t& decodeBitstreamRange, uint32_t& pictureOffset); + +private: + VideoContext m_Context = {}; + VideoConfig m_Config = {}; + VideoSize m_Size = {}; + CodecParameters m_CodecParameters = {}; + std::string m_Status = "Initializing decoder"; + + nri::VideoSession* m_Session = nullptr; + nri::VideoSessionParameters* m_Parameters = nullptr; + nri::Texture* m_DecodeTexture = nullptr; + nri::Texture* m_AV1PDecodeTexture = nullptr; + nri::Buffer* m_DecodeBitstreamBuffer = nullptr; + nri::VideoPicture* m_DecodePicture = nullptr; + nri::VideoPicture* m_AV1PDecodePicture = nullptr; + + uint32_t m_DecodeFrameIndex = 0; + bool m_Ready = false; +}; + +} // namespace video_sample diff --git a/Source/VideoEncodeDecode/Encoder.cpp b/Source/VideoEncodeDecode/Encoder.cpp new file mode 100644 index 0000000..66e1753 --- /dev/null +++ b/Source/VideoEncodeDecode/Encoder.cpp @@ -0,0 +1,647 @@ +// © 2021 NVIDIA Corporation + +#include "Encoder.h" + +#include + +namespace video_sample { + +Encoder::~Encoder() { + if (!m_Context.nri) + return; + + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + if (video.DestroyVideoPicture) { + if (m_AV1PReconstructedPicture) + video.DestroyVideoPicture(m_AV1PReconstructedPicture); + if (m_ReconstructedPicture) + video.DestroyVideoPicture(m_ReconstructedPicture); + if (m_EncodePicture) + video.DestroyVideoPicture(m_EncodePicture); + if (m_Parameters) + video.DestroyVideoSessionParameters(m_Parameters); + if (m_Session) + video.DestroyVideoSession(m_Session); + } + + if (m_MetadataReadbackCommandBuffer) + nri.DestroyCommandBuffer(m_MetadataReadbackCommandBuffer); + if (m_MetadataReadbackCommandAllocator) + nri.DestroyCommandAllocator(m_MetadataReadbackCommandAllocator); + if (m_MetadataReadbackFence) + nri.DestroyFence(m_MetadataReadbackFence); + if (m_ResolvedMetadataReadbackBuffer) + nri.DestroyBuffer(m_ResolvedMetadataReadbackBuffer); + if (m_ResolvedMetadataBuffer) + nri.DestroyBuffer(m_ResolvedMetadataBuffer); + if (m_MetadataBuffer) + nri.DestroyBuffer(m_MetadataBuffer); + if (m_BitstreamBuffer) + nri.DestroyBuffer(m_BitstreamBuffer); + if (m_BitstreamHeaderUploadBuffer) + nri.DestroyBuffer(m_BitstreamHeaderUploadBuffer); + if (m_ReconstructedTexture) + nri.DestroyTexture(m_ReconstructedTexture); + if (m_AV1PReconstructedTexture) + nri.DestroyTexture(m_AV1PReconstructedTexture); + if (m_EncodeTexture) + nri.DestroyTexture(m_EncodeTexture); +} + +bool Encoder::Initialize(const VideoContext& context, const VideoConfig& config, const VideoSize& size, const CodecParameters& codecParameters) { + m_Context = context; + m_Config = config; + m_Size = size; + m_CodecParameters = codecParameters; + + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + nri::VideoSessionDesc sessionDesc = {}; + sessionDesc.type = nri::VideoSessionType::ENCODE; + sessionDesc.codec = GetNriCodec(m_Config.codec); + sessionDesc.format = nri::Format::NV12_UNORM; + sessionDesc.width = m_Size.codedWidth; + sessionDesc.height = m_Size.codedHeight; + sessionDesc.maxReferenceNum = 1; + + if (video.CreateVideoSession(*m_Context.device, sessionDesc, m_Session) != nri::Result::SUCCESS) { + m_Status = std::string("Failed to create ") + GetCodecName(m_Config.codec) + " encode session"; + return false; + } + + nri::VideoH264SessionParametersDesc h264Parameters = {}; + h264Parameters.sequenceParameterSets = &m_CodecParameters.h264Sps; + h264Parameters.sequenceParameterSetNum = 1; + h264Parameters.pictureParameterSets = &m_CodecParameters.h264Pps; + h264Parameters.pictureParameterSetNum = 1; + h264Parameters.maxSequenceParameterSetNum = 1; + h264Parameters.maxPictureParameterSetNum = 1; + + nri::VideoH265SessionParametersDesc h265Parameters = {}; + h265Parameters.videoParameterSets = &m_CodecParameters.h265Vps; + h265Parameters.videoParameterSetNum = 1; + h265Parameters.sequenceParameterSets = &m_CodecParameters.h265Sps; + h265Parameters.sequenceParameterSetNum = 1; + h265Parameters.pictureParameterSets = &m_CodecParameters.h265Pps; + h265Parameters.pictureParameterSetNum = 1; + h265Parameters.maxVideoParameterSetNum = 1; + h265Parameters.maxSequenceParameterSetNum = 1; + h265Parameters.maxPictureParameterSetNum = 1; + + nri::VideoAV1SessionParametersDesc av1Parameters = {}; + av1Parameters.sequence = m_CodecParameters.av1Sequence; + + nri::VideoSessionParametersDesc parametersDesc = {}; + parametersDesc.session = m_Session; + parametersDesc.h264Parameters = m_Config.codec == SampleCodec::H264 ? &h264Parameters : nullptr; + parametersDesc.h265Parameters = m_Config.codec == SampleCodec::H265 ? &h265Parameters : nullptr; + parametersDesc.av1Parameters = m_Config.codec == SampleCodec::AV1 ? &av1Parameters : nullptr; + + if (video.CreateVideoSessionParameters(*m_Context.device, parametersDesc, m_Parameters) != nri::Result::SUCCESS) { + m_Status = std::string("Failed to create ") + GetCodecName(m_Config.codec) + " encode parameters"; + return false; + } + + nri::TextureDesc textureDesc = {}; + textureDesc.type = nri::TextureType::TEXTURE_2D; + textureDesc.usage = nri::TextureUsageBits::VIDEO_ENCODE; + textureDesc.format = nri::Format::NV12_UNORM; + textureDesc.width = (nri::Dim_t)m_Size.codedWidth; + textureDesc.height = (nri::Dim_t)m_Size.codedHeight; + textureDesc.mipNum = 1; + textureDesc.layerNum = 1; + textureDesc.videoCodec = GetNriCodec(m_Config.codec); + + if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_EncodeTexture) != nri::Result::SUCCESS) { + m_Status = "Failed to create NV12 encode texture"; + return false; + } + nri.SetDebugName(m_EncodeTexture, "VideoEncodeTexture"); + + if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_ReconstructedTexture) != nri::Result::SUCCESS) { + m_Status = "Failed to create NV12 reconstructed texture"; + return false; + } + nri.SetDebugName(m_ReconstructedTexture, "VideoReconstructedTexture"); + + if (m_Config.av1PFrameVisual) { + if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_AV1PReconstructedTexture) != nri::Result::SUCCESS) { + m_Status = "Failed to create second NV12 reconstructed texture"; + return false; + } + nri.SetDebugName(m_AV1PReconstructedTexture, "VideoAV1PReconstructedTexture"); + } + + if (!SubmitOneTime(nri, *m_Context.graphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::TextureBarrierDesc textureBarriers[3] = {}; + uint32_t textureBarrierNum = 0; + textureBarriers[textureBarrierNum++].texture = m_EncodeTexture; + textureBarriers[textureBarrierNum++].texture = m_ReconstructedTexture; + if (m_AV1PReconstructedTexture) + textureBarriers[textureBarrierNum++].texture = m_AV1PReconstructedTexture; + + for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { + if (!textureBarrier.texture) + continue; + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::UNDEFINED, nri::StageBits::ALL}; + textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + } + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = textureBarrierNum; + nri.CmdBarrier(commandBuffer, barrierDesc); + })) { + m_Status = "Failed to initialize encode texture layouts"; + return false; + } + + nri::BufferDesc bitstreamHeaderUploadBufferDesc = {}; + bitstreamHeaderUploadBufferDesc.size = ENCODED_SLICE_OFFSET; + + nri::BufferDesc bitstreamBufferDesc = {}; + bitstreamBufferDesc.size = BITSTREAM_SIZE; + bitstreamBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; + + nri::BufferDesc metadataBufferDesc = {}; + metadataBufferDesc.size = METADATA_SIZE; + metadataBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; + + nri::BufferDesc resolvedMetadataBufferDesc = {}; + resolvedMetadataBufferDesc.size = RESOLVED_METADATA_SIZE; + resolvedMetadataBufferDesc.usage = nri::BufferUsageBits::VIDEO_ENCODE; + + nri::BufferDesc resolvedMetadataReadbackBufferDesc = {}; + resolvedMetadataReadbackBufferDesc.size = RESOLVED_METADATA_SIZE; + resolvedMetadataReadbackBufferDesc.usage = nri::BufferUsageBits::NONE; + + if (nri.CreateCommittedBuffer(*m_Context.device, nri::MemoryLocation::HOST_UPLOAD, 0.0f, bitstreamHeaderUploadBufferDesc, m_BitstreamHeaderUploadBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to create bitstream header upload buffer"; + return false; + } + if (CreateEncodeBitstreamBuffer(nri, *m_Context.device, 0.0f, bitstreamBufferDesc, m_BitstreamBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to create encode bitstream buffer"; + return false; + } + if (nri.CreateCommittedBuffer(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, metadataBufferDesc, m_MetadataBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to create encode metadata buffer"; + return false; + } + if (nri.CreateCommittedBuffer(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, resolvedMetadataBufferDesc, m_ResolvedMetadataBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to create resolved encode metadata buffer"; + return false; + } + if (nri.CreateCommittedBuffer(*m_Context.device, nri::MemoryLocation::HOST_READBACK, 0.0f, resolvedMetadataReadbackBufferDesc, m_ResolvedMetadataReadbackBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to create resolved encode metadata readback buffer"; + return false; + } + if (nri.CreateFence(*m_Context.device, 0, m_MetadataReadbackFence) != nri::Result::SUCCESS) { + m_Status = "Failed to create metadata readback fence"; + return false; + } + if (nri.CreateCommandAllocator(*m_Context.graphicsQueue, m_MetadataReadbackCommandAllocator) != nri::Result::SUCCESS || nri.CreateCommandBuffer(*m_MetadataReadbackCommandAllocator, m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to create metadata readback command buffer"; + return false; + } + + nri::VideoPictureDesc encodePictureDesc = {}; + encodePictureDesc.texture = m_EncodeTexture; + encodePictureDesc.usage = nri::VideoPictureUsage::ENCODE_INPUT; + encodePictureDesc.width = (nri::Dim_t)m_Size.codedWidth; + encodePictureDesc.height = (nri::Dim_t)m_Size.codedHeight; + + nri::VideoPictureDesc reconstructedPictureDesc = encodePictureDesc; + reconstructedPictureDesc.texture = m_ReconstructedTexture; + reconstructedPictureDesc.usage = nri::VideoPictureUsage::ENCODE_REFERENCE; + + if (video.CreateVideoPicture(*m_Context.device, encodePictureDesc, m_EncodePicture) != nri::Result::SUCCESS) { + m_Status = "Failed to create encode picture"; + return false; + } + if (video.CreateVideoPicture(*m_Context.device, reconstructedPictureDesc, m_ReconstructedPicture) != nri::Result::SUCCESS) { + m_Status = "Failed to create reconstructed picture"; + return false; + } + if (m_Config.av1PFrameVisual) { + reconstructedPictureDesc.texture = m_AV1PReconstructedTexture; + if (video.CreateVideoPicture(*m_Context.device, reconstructedPictureDesc, m_AV1PReconstructedPicture) != nri::Result::SUCCESS) { + m_Status = "Failed to create second reconstructed picture"; + return false; + } + } + + m_Ready = true; + m_Status = std::string(GetCodecName(m_Config.codec)) + " encoder initialized"; + return true; +} + +bool Encoder::IsEncodingAv1PFrame() const { + return m_Config.av1PFrameVisual && m_AV1PFrameStage == 1; +} + +bool Encoder::WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders) { + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + if (m_Config.codec == SampleCodec::AV1) { + void* headerPtr = nri.MapBuffer(*m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); + if (!headerPtr) { + m_Status = "Failed to map bitstream header upload buffer"; + return false; + } + std::memset(headerPtr, 0, (size_t)ENCODED_SLICE_OFFSET); + nri.UnmapBuffer(*m_BitstreamHeaderUploadBuffer); + annexBHeaders.clear(); + return true; + } + + nri::VideoAnnexBParameterSetsDesc annexBDesc = {}; + annexBDesc.codec = GetNriCodec(m_Config.codec); + annexBDesc.h264Sps = &m_CodecParameters.h264Sps; + annexBDesc.h264Pps = &m_CodecParameters.h264Pps; + annexBDesc.h265Vps = &m_CodecParameters.h265Vps; + annexBDesc.h265Sps = &m_CodecParameters.h265Sps; + annexBDesc.h265Pps = &m_CodecParameters.h265Pps; + + if (video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS || annexBDesc.writtenSize == 0 || annexBDesc.writtenSize >= ENCODED_SLICE_OFFSET) { + m_Status = std::string("Failed to query ") + GetCodecName(m_Config.codec) + " Annex-B parameter-set size"; + return false; + } + + annexBHeaders.resize((size_t)annexBDesc.writtenSize); + annexBDesc.dst = annexBHeaders.data(); + annexBDesc.dstSize = annexBHeaders.size(); + if (video.WriteVideoAnnexBParameterSets(annexBDesc) != nri::Result::SUCCESS) { + m_Status = std::string("Failed to build ") + GetCodecName(m_Config.codec) + " Annex-B parameter sets"; + return false; + } + + void* headerPtr = nri.MapBuffer(*m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); + if (!headerPtr) { + m_Status = "Failed to map bitstream header upload buffer"; + return false; + } + std::memset(headerPtr, 0, (size_t)ENCODED_SLICE_OFFSET); + std::memcpy(headerPtr, annexBHeaders.data(), annexBHeaders.size()); + nri.UnmapBuffer(*m_BitstreamHeaderUploadBuffer); + return true; +} + +bool Encoder::Encode(const EncodeRequest& request) { + if (!m_Ready || !request.nv12Buffer || !request.nv12Layout) { + m_Status = std::string(GetCodecName(m_Config.codec)) + " encoder is not ready"; + return false; + } + + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + const bool av1PFrame = IsEncodingAv1PFrame(); + + if (!CopyNv12BufferToTexture(nri, *m_Context.graphicsQueue, *request.nv12Layout, *request.nv12Buffer, *m_EncodeTexture, m_Size.videoWidth, m_Size.videoHeight)) { + m_Status = "Failed to upload NV12 source to video texture"; + return false; + } + + std::vector annexBHeaders; + if (!WriteAnnexBHeadersToUploadBuffer(annexBHeaders)) + return false; + + if (!SubmitOneTime(nri, *m_Context.graphicsQueue, [&](nri::CommandBuffer& commandBuffer) { + nri.CmdZeroBuffer(commandBuffer, *m_BitstreamBuffer, 0, BITSTREAM_SIZE); + nri.CmdCopyBuffer(commandBuffer, *m_BitstreamBuffer, 0, *m_BitstreamHeaderUploadBuffer, 0, ENCODED_SLICE_OFFSET); + })) { + m_Status = std::string("Failed to upload ") + GetCodecName(m_Config.codec) + " Annex-B parameter sets"; + return false; + } + + nri::VideoEncodePictureDesc pictureDesc = {}; + pictureDesc.frameType = av1PFrame ? nri::VideoEncodeFrameType::P : nri::VideoEncodeFrameType::IDR; + pictureDesc.frameIndex = av1PFrame ? 1 : 0; + pictureDesc.pictureOrderCount = av1PFrame ? 1 : 0; + pictureDesc.idrPictureId = av1PFrame ? 0 : 1; + + uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((m_Size.codedWidth + 7) >> 3))}; + uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((m_Size.codedHeight + 7) >> 3))}; + uint16_t av1WidthInSuperblocksMinus1[] = {(uint16_t)(((m_Size.codedWidth + 63) / 64) - 1)}; + uint16_t av1HeightInSuperblocksMinus1[] = {(uint16_t)(((m_Size.codedHeight + 63) / 64) - 1)}; + nri::VideoAV1TileLayoutDesc av1TileLayout = {}; + av1TileLayout.columnNum = 1; + av1TileLayout.rowNum = 1; + av1TileLayout.tileSizeBytesMinus1 = 3; + av1TileLayout.uniformSpacing = 1; + av1TileLayout.miColumnStarts = av1MiColumnStarts; + av1TileLayout.miRowStarts = av1MiRowStarts; + av1TileLayout.widthInSuperblocksMinus1 = av1WidthInSuperblocksMinus1; + av1TileLayout.heightInSuperblocksMinus1 = av1HeightInSuperblocksMinus1; + nri::VideoAV1LoopFilterDesc av1LoopFilter = {}; + av1LoopFilter.refDeltas[0] = 1; + av1LoopFilter.refDeltas[4] = -1; + av1LoopFilter.refDeltas[6] = -1; + av1LoopFilter.refDeltas[7] = -1; + nri::VideoAV1CdefDesc av1Cdef = {}; + nri::VideoAV1LoopRestorationDesc av1LoopRestoration = {}; + nri::VideoAV1GlobalMotionDesc av1GlobalMotion = {}; + for (auto& params : av1GlobalMotion.params) { + params[2] = 1 << 16; + params[5] = 1 << 16; + } + nri::VideoAV1PictureDesc av1PictureDesc = {}; + av1PictureDesc.currentFrameId = av1PFrame ? 1 : 0; + av1PictureDesc.orderHint = av1PFrame ? 1 : 0; + av1PictureDesc.refreshFrameFlags = av1PFrame ? 0x1 : 0xFF; + av1PictureDesc.primaryReferenceName = av1PFrame ? nri::VideoAV1ReferenceName::LAST : nri::VideoAV1ReferenceName::NONE; + nri::VideoAV1PictureBits av1CommonPictureFlags = nri::VideoAV1PictureBits::SHOW_FRAME; + if (m_Context.graphicsAPI == nri::GraphicsAPI::D3D12) + av1CommonPictureFlags |= nri::VideoAV1PictureBits::SEGMENTATION_ENABLED; + av1PictureDesc.flags = av1PFrame ? av1CommonPictureFlags + : av1CommonPictureFlags | nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE; + av1PictureDesc.renderWidthMinus1 = (uint16_t)((m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_Size.codedWidth : m_Size.videoWidth) - 1); + av1PictureDesc.renderHeightMinus1 = (uint16_t)((m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_Size.codedHeight : m_Size.videoHeight) - 1); + av1PictureDesc.baseQIndex = (uint8_t)m_Config.av1BaseQIndex; + av1PictureDesc.interpolationFilter = 0; + av1PictureDesc.txMode = 2; + av1PictureDesc.cdefDampingMinus3 = 3; + av1PictureDesc.tileLayout = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1TileLayout; + av1PictureDesc.loopFilter = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1LoopFilter; + av1PictureDesc.cdef = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1Cdef; + av1PictureDesc.loopRestoration = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1LoopRestoration; + av1PictureDesc.globalMotion = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1GlobalMotion; + nri::VideoReference av1Reference = {m_ReconstructedPicture, 0}; + nri::VideoAV1ReferenceDesc av1References[8] = {}; + if (av1PFrame) { + const nri::VideoAV1ReferenceName av1ReferenceNames[] = { + nri::VideoAV1ReferenceName::LAST, + nri::VideoAV1ReferenceName::LAST2, + nri::VideoAV1ReferenceName::LAST3, + nri::VideoAV1ReferenceName::GOLDEN, + nri::VideoAV1ReferenceName::BWDREF, + nri::VideoAV1ReferenceName::ALTREF2, + nri::VideoAV1ReferenceName::ALTREF, + }; + for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { + av1References[i].name = av1ReferenceNames[i]; + av1References[i].refFrameIndex = 0; + av1References[i].frameType = nri::VideoEncodeFrameType::IDR; + av1References[i].orderHint = 0; + av1References[i].frameId = 0; + av1References[i].slot = 0; + } + av1PictureDesc.references = av1References; + av1PictureDesc.referenceNum = helper::GetCountOf(av1ReferenceNames); + } + + nri::VideoEncodeRateControlDesc rateControlDesc = {}; + rateControlDesc.mode = nri::VideoEncodeRateControlMode::CQP; + rateControlDesc.qpI = (uint8_t)m_Config.qpI; + rateControlDesc.qpP = (uint8_t)m_Config.qpP; + rateControlDesc.qpB = (uint8_t)m_Config.qpB; + rateControlDesc.frameRateNumerator = 30; + rateControlDesc.frameRateDenominator = 1; + + nri::VideoEncodeDesc encodeDesc = {}; + encodeDesc.session = m_Session; + encodeDesc.parameters = m_Parameters; + encodeDesc.srcPicture = m_EncodePicture; + encodeDesc.dstBitstream.buffer = m_BitstreamBuffer; + encodeDesc.dstBitstream.offset = ENCODED_SLICE_OFFSET; + encodeDesc.dstBitstream.size = BITSTREAM_SIZE - ENCODED_SLICE_OFFSET; + encodeDesc.bitstreamMetadataSize = ENCODED_SLICE_OFFSET; + encodeDesc.pictureDesc = &pictureDesc; + encodeDesc.rateControlDesc = &rateControlDesc; + encodeDesc.reconstructedPicture = m_ReconstructedPicture; + if (av1PFrame) { + encodeDesc.reconstructedPicture = m_AV1PReconstructedPicture; + encodeDesc.references = &av1Reference; + encodeDesc.referenceNum = 1; + encodeDesc.reconstructedSlot = 1; + } + encodeDesc.metadata = m_MetadataBuffer; + encodeDesc.resolvedMetadata = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_ResolvedMetadataReadbackBuffer : m_ResolvedMetadataBuffer; + encodeDesc.av1PictureDesc = m_Config.codec == SampleCodec::AV1 ? &av1PictureDesc : nullptr; + + nri::VideoEncodePictureStates srcPictureStates = {}; + nri::VideoEncodePictureStates reconstructedPictureStates = {}; + if (video.GetVideoEncodePictureStates(*m_EncodePicture, srcPictureStates) != nri::Result::SUCCESS || video.GetVideoEncodePictureStates(*(av1PFrame ? m_AV1PReconstructedPicture : m_ReconstructedPicture), reconstructedPictureStates) != nri::Result::SUCCESS) { + m_Status = "Failed to query video encode picture states"; + return false; + } + + if (!SubmitOneTime(nri, *m_Context.encodeQueue, [&](nri::CommandBuffer& commandBuffer) { + nri::BufferBarrierDesc bufferBarriers[2] = {}; + bufferBarriers[0].buffer = m_MetadataBuffer; + bufferBarriers[0].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[1].buffer = encodeDesc.resolvedMetadata; + bufferBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + + nri::TextureBarrierDesc textureBarriers[3] = {}; + textureBarriers[0].texture = m_EncodeTexture; + textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[0].after = srcPictureStates.encodeRead; + textureBarriers[0].mipNum = nri::REMAINING; + textureBarriers[0].layerNum = nri::REMAINING; + textureBarriers[0].planes = nri::PlaneBits::ALL; + textureBarriers[1].texture = av1PFrame ? m_AV1PReconstructedTexture : m_ReconstructedTexture; + textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[1].after = reconstructedPictureStates.encodeWrite; + textureBarriers[1].mipNum = nri::REMAINING; + textureBarriers[1].layerNum = nri::REMAINING; + textureBarriers[1].planes = nri::PlaneBits::ALL; + textureBarriers[2].texture = m_ReconstructedTexture; + textureBarriers[2].before = reconstructedPictureStates.graphicsBefore; + textureBarriers[2].after = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].mipNum = nri::REMAINING; + textureBarriers[2].layerNum = nri::REMAINING; + textureBarriers[2].planes = nri::PlaneBits::ALL; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = bufferBarriers; + barrierDesc.bufferNum = helper::GetCountOf(bufferBarriers); + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; + nri.CmdBarrier(commandBuffer, barrierDesc); + video.CmdEncodeVideo(commandBuffer, encodeDesc); + bufferBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[0].after = {}; + bufferBarriers[1].before = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; + bufferBarriers[1].after = {}; + textureBarriers[0].before = srcPictureStates.encodeRead; + textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[1].before = reconstructedPictureStates.encodeWrite; + textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].after = reconstructedPictureStates.afterEncode; + barrierDesc.textures = textureBarriers; + barrierDesc.textureNum = av1PFrame && reconstructedPictureStates.releaseAfterEncode ? helper::GetCountOf(textureBarriers) : 2; + nri.CmdBarrier(commandBuffer, barrierDesc); + })) { + m_Status = std::string(GetCodecName(m_Config.codec)) + " encode submission failed"; + return false; + } + + if (m_MetadataReadbackPending) + return true; + + if (m_Context.graphicsAPI == nri::GraphicsAPI::VK) { + m_MetadataReadbackPending = true; + return true; + } + + return SubmitMetadataReadback(); +} + +bool Encoder::SubmitMetadataReadback() { + NRIInterface& nri = *m_Context.nri; + + nri.ResetCommandAllocator(*m_MetadataReadbackCommandAllocator); + if (nri.BeginCommandBuffer(*m_MetadataReadbackCommandBuffer, nullptr) != nri::Result::SUCCESS) { + m_Status = "Failed to begin metadata readback command buffer"; + return false; + } + + nri::BufferBarrierDesc metadataBarriers[2] = {}; + metadataBarriers[0].buffer = m_ResolvedMetadataBuffer; + metadataBarriers[0].before = {nri::AccessBits::NONE, nri::StageBits::NONE}; + metadataBarriers[0].after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + metadataBarriers[1].buffer = m_ResolvedMetadataReadbackBuffer; + metadataBarriers[1].after = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + + nri::BarrierDesc metadataBarrierDesc = {}; + metadataBarrierDesc.buffers = metadataBarriers; + metadataBarrierDesc.bufferNum = helper::GetCountOf(metadataBarriers); + nri.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); + nri.CmdCopyBuffer(*m_MetadataReadbackCommandBuffer, *m_ResolvedMetadataReadbackBuffer, 0, *m_ResolvedMetadataBuffer, 0, RESOLVED_METADATA_SIZE); + metadataBarriers[0].before = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + metadataBarriers[0].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + metadataBarriers[1].before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; + metadataBarriers[1].after = {nri::AccessBits::NONE, nri::StageBits::NONE}; + nri.CmdBarrier(*m_MetadataReadbackCommandBuffer, metadataBarrierDesc); + + if (nri.EndCommandBuffer(*m_MetadataReadbackCommandBuffer) != nri::Result::SUCCESS) { + m_Status = "Failed to end metadata readback command buffer"; + return false; + } + + m_MetadataReadbackFenceValue++; + nri::FenceSubmitDesc signalFence = {}; + signalFence.fence = m_MetadataReadbackFence; + signalFence.value = m_MetadataReadbackFenceValue; + + const nri::CommandBuffer* commandBuffers[] = {m_MetadataReadbackCommandBuffer}; + nri::QueueSubmitDesc submit = {}; + submit.commandBuffers = commandBuffers; + submit.commandBufferNum = helper::GetCountOf(commandBuffers); + submit.signalFences = &signalFence; + submit.signalFenceNum = 1; + if (nri.QueueSubmit(*m_Context.graphicsQueue, submit) != nri::Result::SUCCESS) { + m_Status = "Failed to submit async metadata readback"; + return false; + } + + m_MetadataReadbackPending = true; + return true; +} + +bool Encoder::BuildAv1DecodeInfo(nri::VideoEncodeFeedback& feedback, nri::VideoAV1EncodeDecodeInfo& av1DecodeInfo) { + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + nri::VideoAV1EncodeDecodeInfoDesc av1InfoDesc = {}; + av1InfoDesc.feedback = &feedback; + av1InfoDesc.sequence = &m_CodecParameters.av1Sequence; + nri::VideoAV1ReferenceDesc av1InfoReferences[8] = {}; + if (IsEncodingAv1PFrame()) { + const nri::VideoAV1ReferenceName av1ReferenceNames[] = { + nri::VideoAV1ReferenceName::LAST, + nri::VideoAV1ReferenceName::LAST2, + nri::VideoAV1ReferenceName::LAST3, + nri::VideoAV1ReferenceName::GOLDEN, + nri::VideoAV1ReferenceName::BWDREF, + nri::VideoAV1ReferenceName::ALTREF2, + nri::VideoAV1ReferenceName::ALTREF, + }; + for (uint32_t i = 0; i < helper::GetCountOf(av1ReferenceNames); i++) { + av1InfoReferences[i].name = av1ReferenceNames[i]; + av1InfoReferences[i].refFrameIndex = 0; + av1InfoReferences[i].frameType = nri::VideoEncodeFrameType::IDR; + av1InfoReferences[i].orderHint = 0; + av1InfoReferences[i].frameId = 0; + av1InfoReferences[i].slot = 0; + } + av1InfoDesc.references = av1InfoReferences; + av1InfoDesc.referenceNum = helper::GetCountOf(av1ReferenceNames); + } + + const uint8_t* encodedPayloadHeader = nullptr; + const uint64_t encodedPayloadOffset = ENCODED_SLICE_OFFSET + feedback.encodedBitstreamOffset; + const uint64_t encodedPayloadSize = feedback.encodedBitstreamWrittenBytes; + if (m_Context.graphicsAPI == nri::GraphicsAPI::VK && encodedPayloadOffset <= BITSTREAM_SIZE && encodedPayloadSize <= BITSTREAM_SIZE - encodedPayloadOffset) { + encodedPayloadHeader = (const uint8_t*)nri.MapBuffer(*m_BitstreamBuffer, encodedPayloadOffset, encodedPayloadSize); + av1InfoDesc.encodedPayloadHeader = encodedPayloadHeader; + av1InfoDesc.encodedPayloadHeaderSize = encodedPayloadHeader ? encodedPayloadSize : 0; + } + const nri::Result av1InfoResult = video.GetVideoEncodeAV1DecodeInfo(*m_Session, *m_ResolvedMetadataReadbackBuffer, 0, av1InfoDesc, av1DecodeInfo); + if (encodedPayloadHeader) + nri.UnmapBuffer(*m_BitstreamBuffer); + if (av1InfoResult != nri::Result::SUCCESS) { + m_Status = "Failed to prepare AV1 decode metadata"; + return false; + } + feedback.encodedBitstreamWrittenBytes = av1DecodeInfo.bitstreamOffset + av1DecodeInfo.bitstreamSize; + return true; +} + +bool Encoder::Poll(EncodedFrame& frame) { + if (!m_MetadataReadbackPending) + return false; + + NRIInterface& nri = *m_Context.nri; + nri::VideoInterface& video = *m_Context.video; + + const uint64_t completedFence = nri.GetFenceValue(*m_MetadataReadbackFence); + if (completedFence < m_MetadataReadbackFenceValue) + return false; + + m_MetadataReadbackPending = false; + + frame = {}; + frame.bitstreamBuffer = m_BitstreamBuffer; + const nri::Result feedbackResult = video.GetVideoEncodeFeedback(*m_Session, *m_ResolvedMetadataReadbackBuffer, 0, frame.feedback); + if (feedbackResult != nri::Result::SUCCESS) { + if (feedbackResult == nri::Result::UNSUPPORTED) + m_Status = std::string(GetCodecName(m_Config.codec)) + " encode metadata feedback is unsupported"; + else + m_Status = "Failed to read resolved encode metadata"; + return false; + } + + if (frame.feedback.errorFlags || !frame.feedback.encodedBitstreamWrittenBytes) { + char message[160] = {}; + std::snprintf(message, sizeof(message), "Encoder returned errorFlags=0x%llX bytes=%llu", + (unsigned long long)frame.feedback.errorFlags, (unsigned long long)frame.feedback.encodedBitstreamWrittenBytes); + m_Status = message; + return false; + } + + if (m_Config.codec == SampleCodec::AV1) { + frame.hasAv1DecodeInfo = true; + if (!BuildAv1DecodeInfo(frame.feedback, frame.av1DecodeInfo)) + return false; + } + + frame.isAv1PFrame = IsEncodingAv1PFrame(); + if (m_Config.av1PFrameVisual && m_AV1PFrameStage == 0) + m_AV1PFrameStage = 1; + else if (frame.isAv1PFrame) + m_AV1PFrameStage = 0; + + m_Status = std::string(GetCodecName(m_Config.codec)) + " encode complete"; + return true; +} + +} // namespace video_sample diff --git a/Source/VideoEncodeDecode/Encoder.h b/Source/VideoEncodeDecode/Encoder.h new file mode 100644 index 0000000..a183523 --- /dev/null +++ b/Source/VideoEncodeDecode/Encoder.h @@ -0,0 +1,79 @@ +// © 2021 NVIDIA Corporation + +#pragma once + +#include "Shared.h" + +namespace video_sample { + +struct EncodeRequest { + nri::Buffer* nv12Buffer = nullptr; + const Nv12BufferLayout* nv12Layout = nullptr; + float timeSec = 0.0f; +}; + +class Encoder { +public: + Encoder() = default; + ~Encoder(); + + Encoder(const Encoder&) = delete; + Encoder& operator=(const Encoder&) = delete; + + bool Initialize(const VideoContext& context, const VideoConfig& config, const VideoSize& size, const CodecParameters& codecParameters); + bool Encode(const EncodeRequest& request); + bool Poll(EncodedFrame& frame); + + bool IsReady() const { + return m_Ready; + } + + bool HasPendingFeedback() const { + return m_MetadataReadbackPending; + } + + nri::Texture* GetInputTexture() const { + return m_EncodeTexture; + } + + const std::string& GetStatus() const { + return m_Status; + } + +private: + bool WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders); + bool SubmitMetadataReadback(); + bool BuildAv1DecodeInfo(nri::VideoEncodeFeedback& feedback, nri::VideoAV1EncodeDecodeInfo& av1DecodeInfo); + bool IsEncodingAv1PFrame() const; + +private: + VideoContext m_Context = {}; + VideoConfig m_Config = {}; + VideoSize m_Size = {}; + CodecParameters m_CodecParameters = {}; + std::string m_Status = "Initializing encoder"; + + nri::VideoSession* m_Session = nullptr; + nri::VideoSessionParameters* m_Parameters = nullptr; + nri::Texture* m_EncodeTexture = nullptr; + nri::Texture* m_ReconstructedTexture = nullptr; + nri::Texture* m_AV1PReconstructedTexture = nullptr; + nri::Buffer* m_BitstreamHeaderUploadBuffer = nullptr; + nri::Buffer* m_BitstreamBuffer = nullptr; + nri::Buffer* m_MetadataBuffer = nullptr; + nri::Buffer* m_ResolvedMetadataBuffer = nullptr; + nri::Buffer* m_ResolvedMetadataReadbackBuffer = nullptr; + nri::VideoPicture* m_EncodePicture = nullptr; + nri::VideoPicture* m_ReconstructedPicture = nullptr; + nri::VideoPicture* m_AV1PReconstructedPicture = nullptr; + nri::CommandAllocator* m_MetadataReadbackCommandAllocator = nullptr; + nri::CommandBuffer* m_MetadataReadbackCommandBuffer = nullptr; + nri::Fence* m_MetadataReadbackFence = nullptr; + + bool m_Ready = false; + bool m_MetadataReadbackPending = false; + uint32_t m_AV1PFrameStage = 0; + uint64_t m_MetadataReadbackFenceValue = 0; +}; + +} // namespace video_sample diff --git a/Source/VideoEncodeDecode/Shared.h b/Source/VideoEncodeDecode/Shared.h new file mode 100644 index 0000000..0a56ffd --- /dev/null +++ b/Source/VideoEncodeDecode/Shared.h @@ -0,0 +1,372 @@ +// © 2021 NVIDIA Corporation + +#pragma once + +#if defined(_WIN32) +# include +#endif + +#include "NRIFramework.h" + +#include "Extensions/NRIVideo.h" + +#include +#include +#include +#include +#include +#include + +namespace video_sample { + +constexpr uint32_t DEFAULT_VIDEO_WIDTH = 1920; +constexpr uint32_t DEFAULT_VIDEO_HEIGHT = 1080; +constexpr double ROUND_TRIP_INTERVAL_SEC = 1.0 / 60.0; +constexpr uint64_t BITSTREAM_SIZE = 2 * 1024 * 1024; +constexpr uint64_t ENCODED_SLICE_OFFSET = 4096; +constexpr uint64_t METADATA_SIZE = 4 * 1024 * 1024; +constexpr uint64_t RESOLVED_METADATA_SIZE = 4096; + +enum PatternOperation : uint32_t { + OP_GENERATE_PATTERN = 0, + OP_NV12_TO_PREVIEW = 1, +}; + +enum class SampleCodec : uint8_t { + H264, + H265, + AV1, +}; + +struct Av1SequenceOptions { + bool enableCdef = true; + bool enableRestoration = true; + uint8_t seqForceScreenContentTools = 2; +}; + +struct QueuedFrame { + nri::CommandAllocator* commandAllocator = nullptr; + nri::CommandBuffer* commandBuffer = nullptr; +}; + +struct PatternConstants { + uint32_t width = DEFAULT_VIDEO_WIDTH; + uint32_t height = DEFAULT_VIDEO_HEIGHT; + uint32_t yOffsetBytes = 0; + uint32_t yRowPitchBytes = 0; + uint32_t uvRowPitchBytes = 0; + uint32_t uvOffsetBytes = 0; + uint32_t operation = OP_GENERATE_PATTERN; + float time = 0.0f; + uint32_t _padding = 0; + uint32_t _padding1 = 0; +}; + +struct Nv12BufferLayout { + uint32_t yRowPitchBytes = DEFAULT_VIDEO_WIDTH; + uint32_t ySlicePitchBytes = DEFAULT_VIDEO_WIDTH * DEFAULT_VIDEO_HEIGHT; + uint64_t uvOffsetBytes = uint64_t(DEFAULT_VIDEO_WIDTH) * DEFAULT_VIDEO_HEIGHT; + uint32_t uvRowPitchBytes = DEFAULT_VIDEO_WIDTH; + uint32_t uvSlicePitchBytes = DEFAULT_VIDEO_WIDTH * DEFAULT_VIDEO_HEIGHT / 2; + uint64_t totalSizeBytes = uint64_t(DEFAULT_VIDEO_WIDTH) * DEFAULT_VIDEO_HEIGHT * 3 / 2; +}; + +struct VideoConfig { + SampleCodec codec = SampleCodec::H264; + uint32_t videoWidth = DEFAULT_VIDEO_WIDTH; + uint32_t videoHeight = DEFAULT_VIDEO_HEIGHT; + uint32_t qpI = 20; + uint32_t qpP = 22; + uint32_t qpB = 24; + uint32_t av1BaseQIndex = 20; + bool av1PFrameVisual = false; +}; + +struct VideoSize { + uint32_t videoWidth = DEFAULT_VIDEO_WIDTH; + uint32_t videoHeight = DEFAULT_VIDEO_HEIGHT; + uint32_t codedWidth = DEFAULT_VIDEO_WIDTH; + uint32_t codedHeight = DEFAULT_VIDEO_HEIGHT; + uint32_t decodeBitstreamSizeAlignment = 256; +}; + +struct VideoContext { + NRIInterface* nri = nullptr; + nri::VideoInterface* video = nullptr; + nri::Device* device = nullptr; + nri::Queue* graphicsQueue = nullptr; + nri::Queue* encodeQueue = nullptr; + nri::Queue* decodeQueue = nullptr; + nri::GraphicsAPI graphicsAPI = nri::GraphicsAPI::NONE; +}; + +struct CodecParameters { + nri::VideoH264SequenceParameterSetDesc h264Sps = {}; + nri::VideoH264PictureParameterSetDesc h264Pps = {}; + nri::VideoH265VideoParameterSetDesc h265Vps = {}; + nri::VideoH265SequenceParameterSetDesc h265Sps = {}; + nri::VideoH265PictureParameterSetDesc h265Pps = {}; + nri::VideoAV1SequenceDesc av1Sequence = {}; +}; + +struct EncodedFrame { + nri::VideoEncodeFeedback feedback = {}; + nri::Buffer* bitstreamBuffer = nullptr; + nri::VideoAV1EncodeDecodeInfo av1DecodeInfo = {}; + bool hasAv1DecodeInfo = false; + bool isAv1PFrame = false; +}; + +struct DecodedFrame { + nri::Texture* texture = nullptr; + nri::VideoPicture* picture = nullptr; + nri::VideoDecodePictureStates pictureStates = {}; + bool isAv1PFrame = false; +}; + +inline const char* GetCodecName(SampleCodec codec) { + switch (codec) { + case SampleCodec::H265: + return "H.265"; + case SampleCodec::AV1: + return "AV1"; + case SampleCodec::H264: + default: + return "H.264"; + } +} + +inline nri::VideoCodec GetNriCodec(SampleCodec codec) { + switch (codec) { + case SampleCodec::H265: + return nri::VideoCodec::H265; + case SampleCodec::AV1: + return nri::VideoCodec::AV1; + case SampleCodec::H264: + default: + return nri::VideoCodec::H264; + } +} + +inline uint64_t AlignUp(uint64_t value, uint64_t alignment) { + return alignment == 0 ? value : ((value + alignment - 1) / alignment) * alignment; +} + +inline uint8_t GetAv1FrameSizeBitsMinus1(uint32_t value) { + uint32_t bits = 0; + uint32_t maxValue = value ? value - 1 : 0; + do { + bits++; + maxValue >>= 1; + } while (maxValue); + return (uint8_t)(bits - 1); +} + +inline nri::VideoAV1SequenceDesc MakeAV1SequenceDesc(uint32_t width, uint32_t height, const Av1SequenceOptions& options) { + nri::VideoAV1SequenceDesc desc = {}; + desc.flags = nri::VideoAV1SequenceBits::ENABLE_ORDER_HINT | nri::VideoAV1SequenceBits::COLOR_DESCRIPTION_PRESENT; + if (options.enableCdef) + desc.flags |= nri::VideoAV1SequenceBits::ENABLE_CDEF; + if (options.enableRestoration) + desc.flags |= nri::VideoAV1SequenceBits::ENABLE_RESTORATION; + desc.bitDepth = 8; + desc.subsamplingX = 1; + desc.subsamplingY = 1; + desc.maxFrameWidthMinus1 = (uint16_t)(width - 1); + desc.maxFrameHeightMinus1 = (uint16_t)(height - 1); + desc.frameWidthBitsMinus1 = GetAv1FrameSizeBitsMinus1(width); + desc.frameHeightBitsMinus1 = GetAv1FrameSizeBitsMinus1(height); + desc.orderHintBitsMinus1 = 7; + desc.seqForceIntegerMv = 2; + desc.seqForceScreenContentTools = options.seqForceScreenContentTools; + desc.colorPrimaries = 1; + desc.transferCharacteristics = 1; + desc.matrixCoefficients = 1; + desc.chromaSamplePosition = 1; + return desc; +} + +inline Nv12BufferLayout MakeNv12BufferLayout(const nri::DeviceDesc& deviceDesc, uint32_t width, uint32_t height) { + const uint32_t rowAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureRow, 1u); + const uint32_t sliceAlignment = std::max(deviceDesc.memoryAlignment.uploadBufferTextureSlice, 1u); + + Nv12BufferLayout layout = {}; + layout.yRowPitchBytes = (uint32_t)AlignUp(width, rowAlignment); + layout.ySlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.yRowPitchBytes) * height, sliceAlignment); + layout.uvOffsetBytes = layout.ySlicePitchBytes; + layout.uvRowPitchBytes = (uint32_t)AlignUp(width, rowAlignment); + layout.uvSlicePitchBytes = (uint32_t)AlignUp(uint64_t(layout.uvRowPitchBytes) * (height / 2), sliceAlignment); + layout.totalSizeBytes = layout.uvOffsetBytes + layout.uvSlicePitchBytes; + return layout; +} + +inline CodecParameters MakeCodecParameters(nri::GraphicsAPI graphicsAPI, uint32_t codedWidth, uint32_t codedHeight) { + CodecParameters params = {}; + + params.h264Sps.flags = nri::VideoH264SequenceParameterSetBits::DIRECT_8X8_INFERENCE | nri::VideoH264SequenceParameterSetBits::FRAME_MBS_ONLY; + params.h264Sps.profileIdc = 100; + params.h264Sps.levelIdc = 42; + params.h264Sps.chromaFormatIdc = 1; + params.h264Sps.sequenceParameterSetId = 0; + params.h264Sps.log2MaxFrameNumMinus4 = 0; + params.h264Sps.pictureOrderCountType = 0; + params.h264Sps.log2MaxPictureOrderCountLsbMinus4 = 0; + params.h264Sps.referenceFrameNum = 1; + params.h264Sps.pictureWidthInMbsMinus1 = (uint16_t)((codedWidth + 15) / 16 - 1); + params.h264Sps.pictureHeightInMapUnitsMinus1 = (uint16_t)((codedHeight + 15) / 16 - 1); + + params.h264Pps.flags = nri::VideoH264PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; + params.h264Pps.sequenceParameterSetId = 0; + params.h264Pps.pictureParameterSetId = 0; + params.h264Pps.refIndexL0DefaultActiveMinus1 = 0; + params.h264Pps.refIndexL1DefaultActiveMinus1 = 0; + + params.h265Vps.flags = nri::VideoH265VideoParameterSetBits::TEMPORAL_ID_NESTING; + params.h265Vps.videoParameterSetId = 0; + params.h265Vps.maxSubLayersMinus1 = 0; + params.h265Vps.profileTierLevel.flags = nri::VideoH265ProfileTierLevelBits::FRAME_ONLY_CONSTRAINT; + params.h265Vps.profileTierLevel.generalProfileIdc = 1; + params.h265Vps.profileTierLevel.generalLevelIdc = 90; + params.h265Vps.decPicBufMgr.maxDecPicBufferingMinus1[0] = 2; + params.h265Vps.decPicBufMgr.maxNumReorderPics[0] = 1; + + params.h265Sps.flags = nri::VideoH265SequenceParameterSetBits::TEMPORAL_ID_NESTING | nri::VideoH265SequenceParameterSetBits::AMP_ENABLED | nri::VideoH265SequenceParameterSetBits::SAMPLE_ADAPTIVE_OFFSET_ENABLED; + params.h265Sps.videoParameterSetId = params.h265Vps.videoParameterSetId; + params.h265Sps.maxSubLayersMinus1 = params.h265Vps.maxSubLayersMinus1; + params.h265Sps.sequenceParameterSetId = 0; + params.h265Sps.chromaFormatIdc = 1; + params.h265Sps.pictureWidthInLumaSamples = codedWidth; + params.h265Sps.pictureHeightInLumaSamples = codedHeight; + params.h265Sps.log2MaxPictureOrderCountLsbMinus4 = 3; + params.h265Sps.log2MinLumaCodingBlockSizeMinus3 = 0; + params.h265Sps.log2DiffMaxMinLumaCodingBlockSize = 2; + params.h265Sps.log2MinLumaTransformBlockSizeMinus2 = 0; + params.h265Sps.log2DiffMaxMinLumaTransformBlockSize = 3; + params.h265Sps.maxTransformHierarchyDepthInter = 3; + params.h265Sps.maxTransformHierarchyDepthIntra = 3; + params.h265Sps.profileTierLevel = params.h265Vps.profileTierLevel; + params.h265Sps.decPicBufMgr = params.h265Vps.decPicBufMgr; + + params.h265Pps.flags = nri::VideoH265PictureParameterSetBits::CABAC_INIT_PRESENT | nri::VideoH265PictureParameterSetBits::TRANSFORM_SKIP_ENABLED | nri::VideoH265PictureParameterSetBits::CU_QP_DELTA_ENABLED | nri::VideoH265PictureParameterSetBits::SLICE_CHROMA_QP_OFFSETS_PRESENT | nri::VideoH265PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; + params.h265Pps.pictureParameterSetId = 0; + params.h265Pps.sequenceParameterSetId = params.h265Sps.sequenceParameterSetId; + params.h265Pps.videoParameterSetId = params.h265Vps.videoParameterSetId; + + Av1SequenceOptions av1SequenceOptions = {}; + if (graphicsAPI == nri::GraphicsAPI::VK) { + av1SequenceOptions.enableCdef = false; + av1SequenceOptions.enableRestoration = false; + av1SequenceOptions.seqForceScreenContentTools = 0; + } + params.av1Sequence = MakeAV1SequenceDesc(codedWidth, codedHeight, av1SequenceOptions); + return params; +} + +template +bool SubmitOneTime(nri::CoreInterface& core, nri::Queue& queue, Record&& record) { + nri::CommandAllocator* allocator = nullptr; + nri::CommandBuffer* commandBuffer = nullptr; + bool ok = core.CreateCommandAllocator(queue, allocator) == nri::Result::SUCCESS && allocator && core.CreateCommandBuffer(*allocator, commandBuffer) == nri::Result::SUCCESS && commandBuffer && core.BeginCommandBuffer(*commandBuffer, nullptr) == nri::Result::SUCCESS; + if (ok) { + std::forward(record)(*commandBuffer); + ok = core.EndCommandBuffer(*commandBuffer) == nri::Result::SUCCESS; + } + if (ok) { + const nri::CommandBuffer* commandBuffers[] = {commandBuffer}; + nri::QueueSubmitDesc submit = {}; + submit.commandBuffers = commandBuffers; + submit.commandBufferNum = 1; + ok = core.QueueSubmit(queue, submit) == nri::Result::SUCCESS && core.QueueWaitIdle(&queue) == nri::Result::SUCCESS; + } + if (commandBuffer) + core.DestroyCommandBuffer(commandBuffer); + if (allocator) + core.DestroyCommandAllocator(allocator); + return ok; +} + +template +bool SubmitOneTime(nri::CoreInterface& core, nri::Queue& queue, nri::DescriptorPool* descriptorPool, Record&& record) { + nri::CommandAllocator* allocator = nullptr; + nri::CommandBuffer* commandBuffer = nullptr; + bool ok = core.CreateCommandAllocator(queue, allocator) == nri::Result::SUCCESS && allocator && core.CreateCommandBuffer(*allocator, commandBuffer) == nri::Result::SUCCESS && commandBuffer && core.BeginCommandBuffer(*commandBuffer, descriptorPool) == nri::Result::SUCCESS; + if (ok) { + std::forward(record)(*commandBuffer); + ok = core.EndCommandBuffer(*commandBuffer) == nri::Result::SUCCESS; + } + if (ok) { + const nri::CommandBuffer* commandBuffers[] = {commandBuffer}; + nri::QueueSubmitDesc submit = {}; + submit.commandBuffers = commandBuffers; + submit.commandBufferNum = 1; + ok = core.QueueSubmit(queue, submit) == nri::Result::SUCCESS && core.QueueWaitIdle(&queue) == nri::Result::SUCCESS; + } + if (commandBuffer) + core.DestroyCommandBuffer(commandBuffer); + if (allocator) + core.DestroyCommandAllocator(allocator); + return ok; +} + +inline bool CopyNv12BufferToTexture(nri::CoreInterface& core, nri::Queue& queue, const Nv12BufferLayout& layout, nri::Buffer& src, nri::Texture& dst, uint32_t width, uint32_t height) { + return SubmitOneTime(core, queue, [&](nri::CommandBuffer& commandBuffer) { + nri::BufferBarrierDesc bufferBarrier = {}; + bufferBarrier.buffer = &src; + bufferBarrier.before = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + bufferBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::StageBits::COPY}; + + nri::TextureBarrierDesc textureBarrier = {}; + textureBarrier.texture = &dst; + textureBarrier.before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + textureBarrier.after = {nri::AccessBits::COPY_DESTINATION, nri::Layout::COPY_DESTINATION, nri::StageBits::COPY}; + textureBarrier.mipNum = nri::REMAINING; + textureBarrier.layerNum = nri::REMAINING; + textureBarrier.planes = nri::PlaneBits::ALL; + + nri::BarrierDesc barrierDesc = {}; + barrierDesc.buffers = &bufferBarrier; + barrierDesc.bufferNum = 1; + barrierDesc.textures = &textureBarrier; + barrierDesc.textureNum = 1; + core.CmdBarrier(commandBuffer, barrierDesc); + + nri::TextureRegionDesc lumaRegion = {}; + lumaRegion.width = (nri::Dim_t)width; + lumaRegion.height = (nri::Dim_t)height; + lumaRegion.depth = 1; + lumaRegion.planes = nri::PlaneBits::PLANE_0; + + nri::TextureDataLayoutDesc lumaLayout = {}; + lumaLayout.rowPitch = layout.yRowPitchBytes; + lumaLayout.slicePitch = layout.ySlicePitchBytes; + core.CmdUploadBufferToTexture(commandBuffer, dst, lumaRegion, src, lumaLayout); + + nri::TextureRegionDesc chromaRegion = {}; + chromaRegion.width = (nri::Dim_t)width; + chromaRegion.height = (nri::Dim_t)height; + chromaRegion.depth = 1; + chromaRegion.planes = nri::PlaneBits::PLANE_1; + + nri::TextureDataLayoutDesc chromaLayout = {}; + chromaLayout.offset = layout.uvOffsetBytes; + chromaLayout.rowPitch = layout.uvRowPitchBytes; + chromaLayout.slicePitch = layout.uvSlicePitchBytes; + core.CmdUploadBufferToTexture(commandBuffer, dst, chromaRegion, src, chromaLayout); + + textureBarrier.before = {nri::AccessBits::COPY_DESTINATION, nri::Layout::COPY_DESTINATION, nri::StageBits::COPY}; + textureBarrier.after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + bufferBarrier.before = bufferBarrier.after; + bufferBarrier.after = {nri::AccessBits::SHADER_RESOURCE_STORAGE, nri::StageBits::COMPUTE_SHADER}; + core.CmdBarrier(commandBuffer, barrierDesc); + }); +} + +inline nri::Result CreateEncodeBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { + return core.CreateCommittedBuffer(device, nri::MemoryLocation::HOST_READBACK, priority, bufferDesc, buffer); +} + +inline nri::Result CreateDecodeBitstreamBuffer(nri::CoreInterface& core, nri::Device& device, float priority, const nri::BufferDesc& bufferDesc, nri::Buffer*& buffer) { + return core.CreateCommittedBuffer(device, nri::MemoryLocation::HOST_UPLOAD, priority, bufferDesc, buffer); +} + +} // namespace video_sample From 02513a3e2b04bbe84bb28c81330443ad757830b6 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 12:11:21 +0200 Subject: [PATCH 18/28] Enhance video encode decode sample --- Shaders/VideoEncodePattern.cs.hlsl | 143 +++++++++++++++++++++------ Source/VideoEncodeDecode.cpp | 100 ++++++++++++++++--- Source/VideoEncodeDecode/Encoder.cpp | 12 ++- Source/VideoEncodeDecode/Encoder.h | 1 + Source/VideoEncodeDecode/Shared.h | 8 +- 5 files changed, 213 insertions(+), 51 deletions(-) diff --git a/Shaders/VideoEncodePattern.cs.hlsl b/Shaders/VideoEncodePattern.cs.hlsl index bf34cb1..ae465d4 100644 --- a/Shaders/VideoEncodePattern.cs.hlsl +++ b/Shaders/VideoEncodePattern.cs.hlsl @@ -18,30 +18,92 @@ struct PatternRootConstants { uint uvOffsetBytes; uint operation; float time; - uint padding; - uint padding1; + float motionSpeed; + float detailStrength; + float diffStrength; + uint showDifference; + uint frameIndex; }; NRI_ROOT_CONSTANTS(PatternRootConstants, g_Pattern, 0, 0); -float3 MakePatternColor(float2 normalizedPixelPos, float time) { +float Stripe(float v, float frequency, float width) { + const float phase = frac(v * frequency); + return smoothstep(width, 0.0f, min(phase, 1.0f - phase)); +} + +float RectMask(float2 p, float2 minP, float2 maxP, float feather) { + const float2 lo = smoothstep(minP, minP + feather, p); + const float2 hi = 1.0f - smoothstep(maxP - feather, maxP, p); + return lo.x * lo.y * hi.x * hi.y; +} + +float3 ColorWheel(float phase) { + const float a = phase * 6.283185307179586f; + return saturate(0.5f + 0.5f * cos(a + float3(0.0f, 4.1887902047863905f, 2.0943951023931953f))); +} + +float3 MakePatternColor(float2 normalizedPixelPos, float time, uint frameIndex) { const float fx = normalizedPixelPos.x; const float fy = normalizedPixelPos.y; const float cx = fx - 0.5f; const float cy = fy - 0.5f; const float radius = sqrt(cx * cx + cy * cy); const float angle = atan2(cy, cx); - - const float sweep = sin(angle * 3.0f + time * 1.7f) * 0.5f + 0.5f; - const float rings = sin((radius * 16.0f - time * 1.25f) * 6.283185307179586f) * 0.5f + 0.5f; - const float diagonal = sin((fx * 5.0f + fy * 3.0f + time * 0.45f) * 6.283185307179586f) * 0.5f + 0.5f; - const float grid = (((uint)(fx * 16.0f) ^ (uint)(fy * 10.0f)) & 1) ? 0.08f : 0.0f; - - const float r = min(1.0f, 0.10f + 0.78f * sweep + 0.18f * diagonal + grid); - const float g = min(1.0f, 0.14f + 0.72f * rings + 0.20f * fy + grid); - const float b = min(1.0f, 0.18f + 0.52f * diagonal + 0.34f * (1.0f - radius) + grid); - - return float3(r, g, b); + const float t = time * g_Pattern.motionSpeed; + const float detail = g_Pattern.detailStrength; + const float detail01 = saturate(detail); + + // Quiet base image: broad gradients and mild chroma rotation reveal banding + // and color drift without hiding problems under noise. + const float lumaRamp = fx; + const float verticalRamp = fy; + const float radialFalloff = saturate(1.0f - radius * 1.45f); + float3 color = lerp(float3(lumaRamp, lumaRamp, lumaRamp), float3(0.12f + 0.78f * lumaRamp, 0.10f + 0.72f * verticalRamp, 0.18f + 0.62f * radialFalloff), 0.42f); + + // Top-left: clean grayscale and chroma ramps for banding/chroma subsampling. + const float rampPanel = RectMask(normalizedPixelPos, float2(0.035f, 0.055f), float2(0.325f, 0.390f), 0.006f); + const float graySteps = floor(fx * 32.0f) / 31.0f; + const float smoothGray = saturate((fx - 0.035f) / 0.290f); + const float panelSplit = step(0.225f, fy); + const float3 rampColor = lerp(float3(smoothGray, smoothGray, smoothGray), ColorWheel(smoothGray + 0.08f * sin(t)), panelSplit); + color = lerp(color, rampColor, rampPanel); + color = lerp(color, float3(graySteps, graySteps, graySteps), rampPanel * (1.0f - panelSplit) * 0.28f * detail01); + + // Center: moving slanted hard edge plus soft rings. Ringing and block edges + // are easy to see here, but the region remains spatially coherent. + const float edgePanel = RectMask(normalizedPixelPos, float2(0.365f, 0.070f), float2(0.665f, 0.610f), 0.006f); + const float edge = smoothstep(-0.004f, 0.004f, (fx - 0.515f) + (fy - 0.33f) * 0.82f + sin(t * 0.7f) * 0.085f); + const float rings = sin((radius * 20.0f - t * 0.75f) * 6.283185307179586f) * 0.5f + 0.5f; + const float3 edgeColor = lerp(float3(0.04f, 0.06f, 0.08f), float3(0.92f, 0.90f, 0.84f), edge); + color = lerp(color, lerp(edgeColor, float3(rings, rings * 0.65f, 1.0f - rings), 0.22f), edgePanel); + + // Right: bounded detail patterns with no frame-to-frame flicker. This exposes + // luma blur, chroma bleed and block behavior without high-contrast temporal noise. + const float detailPanel = RectMask(normalizedPixelPos, float2(0.700f, 0.070f), float2(0.965f, 0.610f), 0.006f); + const float2 detailUv = saturate((normalizedPixelPos - float2(0.700f, 0.070f)) / float2(0.265f, 0.540f)); + const float barFreq = lerp(10.0f, 28.0f, detail01); + const float lumaBars = 0.5f + 0.38f * sin((detailUv.x * barFreq + 0.12f * sin(t * 0.35f)) * 6.283185307179586f); + const float chromaWave = 0.5f + 0.35f * sin((detailUv.y * 7.0f - t * 0.035f) * 6.283185307179586f); + const float diagonalWedge = smoothstep(0.012f, 0.0f, abs(frac((detailUv.x + detailUv.y) * 8.0f) - 0.5f)); + const float3 softBars = lerp(float3(lumaBars, lumaBars, lumaBars), float3(0.78f, 0.24f + 0.48f * chromaWave, 0.28f + 0.42f * (1.0f - chromaWave)), 0.42f); + const float3 wedgeColor = lerp(softBars, float3(0.18f, 0.22f, 0.26f), diagonalWedge * 0.28f * detail01); + color = lerp(color, wedgeColor, detailPanel); + + // Bottom: moving color wheel and measuring grid. Temporal glitches, repeated + // frames and chroma phase issues are obvious but still structured. + const float motionPanel = RectMask(normalizedPixelPos, float2(0.035f, 0.665f), float2(0.965f, 0.935f), 0.006f); + const float wheel = frac(fx * 1.75f + t * 0.10f); + const float markerX = frac(t * 0.16f); + const float marker = smoothstep(0.018f, 0.0f, abs(fx - markerX)) * smoothstep(0.0f, 0.04f, fy - 0.665f) * smoothstep(0.0f, 0.04f, 0.935f - fy); + const float frameBlink = ((frameIndex / 8u) & 1u) ? 1.0f : 0.0f; + const float grid = max(Stripe(fx, 16.0f, 0.018f), Stripe(fy, 9.0f, 0.020f)); + float3 motionColor = ColorWheel(wheel); + motionColor = lerp(motionColor, float3(0.0f, 0.0f, 0.0f), grid * 0.35f * detail01); + motionColor = lerp(motionColor, lerp(float3(1.0f, 1.0f, 1.0f), float3(0.0f, 0.0f, 0.0f), frameBlink), marker); + color = lerp(color, motionColor, motionPanel); + + return saturate(color); } uint ClampToByte(float v) { @@ -73,7 +135,28 @@ float3 YuvToRgb(uint y, uint u, uint v) { } float4 LoadPatternColor(uint px, uint py) { - return float4(MakePatternColor(float2(px, py) / float2(g_Pattern.width - 1u, g_Pattern.height - 1u), g_Pattern.time), 1.0f); + return float4(MakePatternColor(float2(px, py) / float2(g_Pattern.width - 1u, g_Pattern.height - 1u), g_Pattern.time, g_Pattern.frameIndex), 1.0f); +} + +uint3 LoadPatternRgb8(uint px, uint py) { + return uint3(LoadPatternColor(px, py).rgb * 255.0f); +} + +float3 LoadPatternNv12Rgb(uint px, uint py) { + const uint chromaX = px & ~1u; + const uint chromaY = py & ~1u; + + const uint3 rgb = LoadPatternRgb8(px, py); + const uint3 chromaRgb0 = LoadPatternRgb8(chromaX + 0u, chromaY + 0u); + const uint3 chromaRgb1 = LoadPatternRgb8(chromaX + 1u, chromaY + 0u); + const uint3 chromaRgb2 = LoadPatternRgb8(chromaX + 0u, chromaY + 1u); + const uint3 chromaRgb3 = LoadPatternRgb8(chromaX + 1u, chromaY + 1u); + + const uint yValue = RGBToY(rgb); + const uint uValue = (RGBToU(chromaRgb0) + RGBToU(chromaRgb1) + RGBToU(chromaRgb2) + RGBToU(chromaRgb3) + 2u) >> 2u; + const uint vValue = (RGBToV(chromaRgb0) + RGBToV(chromaRgb1) + RGBToV(chromaRgb2) + RGBToV(chromaRgb3) + 2u) >> 2u; + + return YuvToRgb(yValue, uValue, vValue); } void StorePreview(uint2 pixel, float4 color) { @@ -103,10 +186,10 @@ void main(uint3 dispatchThreadID : SV_DispatchThreadID) StorePreview(uint2(blockX + 2u, y), c2); StorePreview(uint2(blockX + 3u, y), c3); - const uint3 rgb0 = uint3(c0.rgb * 255.0f); - const uint3 rgb1 = uint3(c1.rgb * 255.0f); - const uint3 rgb2 = uint3(c2.rgb * 255.0f); - const uint3 rgb3 = uint3(c3.rgb * 255.0f); + const uint3 rgb0 = LoadPatternRgb8(blockX + 0u, y); + const uint3 rgb1 = LoadPatternRgb8(blockX + 1u, y); + const uint3 rgb2 = LoadPatternRgb8(blockX + 2u, y); + const uint3 rgb3 = LoadPatternRgb8(blockX + 3u, y); const uint y0 = RGBToY(rgb0); const uint y1 = RGBToY(rgb1); @@ -122,15 +205,10 @@ void main(uint3 dispatchThreadID : SV_DispatchThreadID) if (y + 1u >= g_Pattern.height) return; - const float4 c4 = LoadPatternColor(blockX + 0u, y + 1u); - const float4 c5 = LoadPatternColor(blockX + 1u, y + 1u); - const float4 c6 = LoadPatternColor(blockX + 2u, y + 1u); - const float4 c7 = LoadPatternColor(blockX + 3u, y + 1u); - - const uint3 rgb4 = uint3(c4.rgb * 255.0f); - const uint3 rgb5 = uint3(c5.rgb * 255.0f); - const uint3 rgb6 = uint3(c6.rgb * 255.0f); - const uint3 rgb7 = uint3(c7.rgb * 255.0f); + const uint3 rgb4 = LoadPatternRgb8(blockX + 0u, y + 1u); + const uint3 rgb5 = LoadPatternRgb8(blockX + 1u, y + 1u); + const uint3 rgb6 = LoadPatternRgb8(blockX + 2u, y + 1u); + const uint3 rgb7 = LoadPatternRgb8(blockX + 3u, y + 1u); const uint u0 = (RGBToU(rgb0) + RGBToU(rgb1) + RGBToU(rgb4) + RGBToU(rgb5) + 2u) >> 2u; const uint v0 = (RGBToV(rgb0) + RGBToV(rgb1) + RGBToV(rgb4) + RGBToV(rgb5) + 2u) >> 2u; @@ -157,7 +235,14 @@ void main(uint3 dispatchThreadID : SV_DispatchThreadID) const uint uValue = (uvWord >> uvShift) & 255u; const uint vValue = (uvWord >> (uvShift + 8u)) & 255u; - StorePreview(uint2(px, y), float4(YuvToRgb(yValue, uValue, vValue), 1.0f)); + const float3 decodedRgb = YuvToRgb(yValue, uValue, vValue); + if (g_Pattern.showDifference != 0u) { + const float3 sourceRgb = LoadPatternNv12Rgb(px, y); + const float3 diffRgb = saturate(abs(sourceRgb - decodedRgb) * g_Pattern.diffStrength); + StorePreview(uint2(px, y), float4(diffRgb, 1.0f)); + } else { + StorePreview(uint2(px, y), float4(decodedRgb, 1.0f)); + } } } } diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 37337f3..ae8f88f 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -77,8 +77,11 @@ struct PatternConstants { uint32_t uvOffsetBytes = 0; uint32_t operation = OP_GENERATE_PATTERN; float time = 0.0f; - uint32_t _padding = 0; - uint32_t _padding1 = 0; + float motionSpeed = 1.0f; + float detailStrength = 1.0f; + float diffStrength = 6.0f; + uint32_t showDifference = 0; + uint32_t frameIndex = 0; }; struct Nv12BufferLayout { @@ -241,6 +244,7 @@ class Sample : public SampleBase { std::unique_ptr m_Encoder; std::unique_ptr m_Decoder; video_sample::VideoConfig m_VideoConfig = {}; + video_sample::VideoQuality m_VideoQuality = {}; video_sample::VideoSize m_VideoSize = {}; video_sample::CodecParameters m_CodecParameters = {}; nri::GraphicsAPI m_GraphicsAPI = nri::GraphicsAPI::NONE; @@ -284,6 +288,11 @@ class Sample : public SampleBase { uint32_t m_QpP = 22; uint32_t m_QpB = 24; uint32_t m_AV1BaseQIndex = 20; + bool m_Lossless = false; + float m_PatternMotionSpeed = 1.0f; + float m_PatternDetailStrength = 1.0f; + float m_DiffStrength = 6.0f; + PatternConstants m_PendingEncodedPatternConstants = {}; SampleCodec m_Codec = SampleCodec::H264; double m_StartTimeSec = 0.0; double m_LastRoundTripTimeSec = -ROUND_TRIP_INTERVAL_SEC; @@ -293,6 +302,9 @@ class Sample : public SampleBase { bool m_PreviewTexturesShaderReadable = false; bool m_SourcePreviewReady = false; bool m_AV1PFrameVisual = false; + bool m_ShowDifference = false; + bool m_HasPendingEncodedPatternConstants = false; + uint32_t m_PatternFrameIndex = 0; }; Sample::~Sample() { @@ -358,6 +370,7 @@ void Sample::InitCmdLine(cmdline::parser& cmdLine) { cmdLine.add("qpP", 0, "CQP quantizer for P frames", false, m_QpP); cmdLine.add("qpB", 0, "CQP quantizer for B frames", false, m_QpB); cmdLine.add("av1BaseQIndex", 0, "AV1 base quantizer index", false, m_AV1BaseQIndex); + cmdLine.add("lossless", 0, "force zero quantizers for lossless-capable codec modes"); } void Sample::ReadCmdLine(cmdline::parser& cmdLine) { @@ -373,6 +386,7 @@ void Sample::ReadCmdLine(cmdline::parser& cmdLine) { m_QpP = cmdLine.get("qpP"); m_QpB = cmdLine.get("qpB"); m_AV1BaseQIndex = cmdLine.get("av1BaseQIndex"); + m_Lossless = cmdLine.exist("lossless"); m_Codec = m_CodecArg == "H265" ? SampleCodec::H265 : (m_CodecArg == "AV1" ? SampleCodec::AV1 : SampleCodec::H264); m_AV1PFrameVisual = m_Codec == SampleCodec::AV1 && m_AV1FrameArg == "P"; } @@ -383,9 +397,10 @@ bool Sample::Initialize(nri::GraphicsAPI graphicsAPI, bool) { std::fprintf(stderr, "%s\n", m_VideoStatus.c_str()); return false; } + const uint32_t minCodecQp = m_Codec == SampleCodec::AV1 ? 1 : 0; const uint32_t maxCodecQp = m_Codec == SampleCodec::AV1 ? 255 : 51; - if (m_QpI > maxCodecQp || m_QpP > maxCodecQp || m_QpB > maxCodecQp || m_AV1BaseQIndex > 255) { - m_VideoStatus = m_Codec == SampleCodec::AV1 ? "AV1 quantizers must be in the 0..255 range" : "H.264/H.265 QP values must be in the 0..51 range"; + if (m_QpI < minCodecQp || m_QpI > maxCodecQp || m_QpP < minCodecQp || m_QpP > maxCodecQp || m_QpB < minCodecQp || m_QpB > maxCodecQp || m_AV1BaseQIndex < minCodecQp || m_AV1BaseQIndex > 255) { + m_VideoStatus = m_Codec == SampleCodec::AV1 ? "AV1 quantizers must be in the 1..255 range on current hardware encode paths" : "H.264/H.265 QP values must be in the 0..51 range"; std::fprintf(stderr, "%s\n", m_VideoStatus.c_str()); return false; } @@ -394,11 +409,12 @@ bool Sample::Initialize(nri::GraphicsAPI graphicsAPI, bool) { m_VideoConfig.codec = m_Codec == SampleCodec::H265 ? video_sample::SampleCodec::H265 : (m_Codec == SampleCodec::AV1 ? video_sample::SampleCodec::AV1 : video_sample::SampleCodec::H264); m_VideoConfig.videoWidth = m_VideoWidth; m_VideoConfig.videoHeight = m_VideoHeight; - m_VideoConfig.qpI = m_QpI; - m_VideoConfig.qpP = m_QpP; - m_VideoConfig.qpB = m_QpB; - m_VideoConfig.av1BaseQIndex = m_AV1BaseQIndex; m_VideoConfig.av1PFrameVisual = m_AV1PFrameVisual; + m_VideoQuality.qpI = m_QpI; + m_VideoQuality.qpP = m_QpP; + m_VideoQuality.qpB = m_QpB; + m_VideoQuality.av1BaseQIndex = m_AV1BaseQIndex; + m_VideoQuality.lossless = m_Lossless; if (!InitializeGraphics(graphicsAPI)) return false; @@ -517,6 +533,11 @@ PatternConstants Sample::MakePatternConstants(PatternOperation operation, float patternConstants.uvOffsetBytes = (uint32_t)m_Nv12Layout.uvOffsetBytes; patternConstants.operation = operation; patternConstants.time = timeSec; + patternConstants.motionSpeed = m_PatternMotionSpeed; + patternConstants.detailStrength = m_PatternDetailStrength; + patternConstants.diffStrength = m_DiffStrength; + patternConstants.showDifference = m_ShowDifference ? 1 : 0; + patternConstants.frameIndex = m_PatternFrameIndex; return patternConstants; } @@ -903,15 +924,19 @@ bool Sample::TrySubmitEncodeAndMetadataReadback(float timeSec) { return false; } + m_PatternFrameIndex++; PatternConstants patternConstants = MakePatternConstants(OP_GENERATE_PATTERN, timeSec); if (!GeneratePatternWithCompute(patternConstants, m_SourcePreviewStorage, true)) { m_VideoStatus = "Failed to generate NV12 source pattern via compute"; return false; } + m_PendingEncodedPatternConstants = patternConstants; + m_HasPendingEncodedPatternConstants = true; video_sample::EncodeRequest encodeRequest = {}; encodeRequest.nv12Buffer = m_UploadBuffer; encodeRequest.nv12Layout = &m_SharedNv12Layout; + encodeRequest.quality = m_VideoQuality; encodeRequest.timeSec = timeSec; if (!m_Encoder->Encode(encodeRequest)) { m_VideoStatus = m_Encoder->GetStatus(); @@ -937,11 +962,15 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { return false; } - PatternConstants patternConstants = MakePatternConstants(OP_NV12_TO_PREVIEW, timeSec); + PatternConstants patternConstants = m_HasPendingEncodedPatternConstants ? m_PendingEncodedPatternConstants : MakePatternConstants(OP_NV12_TO_PREVIEW, timeSec); + patternConstants.operation = OP_NV12_TO_PREVIEW; + patternConstants.diffStrength = m_DiffStrength; + patternConstants.showDifference = m_ShowDifference ? 1 : 0; if (!GeneratePatternWithCompute(patternConstants, m_DecodePreviewStorage, true)) { m_VideoStatus = "Failed to convert decoded NV12 to preview texture"; return false; } + m_HasPendingEncodedPatternConstants = false; m_DecodePreviewReady = true; if (m_AV1PFrameVisual && !encodedFrame.isAv1PFrame) @@ -988,8 +1017,7 @@ void Sample::PrepareFrame(uint32_t) { const double timeSec = m_Timer.GetTimeStamp() * 0.001 - m_StartTimeSec; const bool canRunRoundTrip = CanRunRoundTrip(); - if (!m_SourcePreviewReady) - InitializeGeneratedFrames((float)timeSec); + InitializeGeneratedFrames((float)timeSec); if (canRunRoundTrip && timeSec - m_LastRoundTripTimeSec >= ROUND_TRIP_INTERVAL_SEC) { if (TryRunRoundTrip((float)timeSec)) @@ -1000,12 +1028,19 @@ void Sample::PrepareFrame(uint32_t) { { ImGui::SetNextWindowPos({20.0f, 20.0f}, ImGuiCond_Once); ImGui::SetNextWindowSize({900.0f, 520.0f}, ImGuiCond_Once); - ImGui::Begin("NRI Video Encode / Decode"); + ImGui::Begin("NRI Video Encode / Decode"); { + const uint32_t minCodecQp = m_Codec == SampleCodec::AV1 ? 1 : 0; + const uint32_t effectiveQpI = m_VideoQuality.lossless ? minCodecQp : std::max(m_VideoQuality.qpI, minCodecQp); + const uint32_t effectiveQpP = m_VideoQuality.lossless ? minCodecQp : std::max(m_VideoQuality.qpP, minCodecQp); + const uint32_t effectiveQpB = m_VideoQuality.lossless ? minCodecQp : std::max(m_VideoQuality.qpB, minCodecQp); ImGui::Text("Codec: %s, format: NV12, size: %ux%u", GetCodecName(m_Codec), m_VideoWidth, m_VideoHeight); - ImGui::Text("CQP: I=%u, P=%u, B=%u%s", m_QpI, m_QpP, m_QpB, m_Codec == SampleCodec::AV1 ? ", AV1 baseQIndex follows below" : ""); - if (m_Codec == SampleCodec::AV1) - ImGui::Text("AV1: frame=%s, baseQIndex=%u", m_AV1FrameArg.c_str(), m_AV1BaseQIndex); + ImGui::Text("Startup-only: codec, video size, AV1 frame mode"); + ImGui::Text("CQP: I=%u, P=%u, B=%u%s", effectiveQpI, effectiveQpP, effectiveQpB, m_Codec == SampleCodec::AV1 ? ", AV1 baseQIndex follows below" : ""); + if (m_Codec == SampleCodec::AV1) { + const uint32_t effectiveAv1BaseQIndex = m_VideoQuality.lossless ? 1 : std::max(m_VideoQuality.av1BaseQIndex, 1u); + ImGui::Text("AV1: frame=%s, baseQIndex=%u", m_AV1FrameArg.c_str(), effectiveAv1BaseQIndex); + } ImGui::TextWrapped("Video: %s", m_VideoStatus.c_str()); ImGui::TextWrapped("Preview: %s", m_PreviewStatus.c_str()); ImGui::Text("Encode queue: %s, decode queue: %s", m_VideoEncodeQueue ? "yes" : "no", m_VideoDecodeQueue ? "yes" : "no"); @@ -1014,6 +1049,37 @@ void Sample::PrepareFrame(uint32_t) { if (m_VideoReady && !m_DecodePreviewReady) ImGui::Text("Decode preview: waiting for first decoded frame"); + ImGui::Separator(); + ImGui::Text("Live controls"); + ImGui::Checkbox("Lossless / zero quantizers", &m_VideoQuality.lossless); + if (m_VideoQuality.lossless) { + if (m_Codec == SampleCodec::AV1) + ImGui::TextWrapped("AV1 hardware encode rejects baseQIndex 0 on current backends, so lossless mode uses near-lossless baseQIndex 1."); + else + ImGui::TextWrapped("%s forces CQP 0%s.", GetCodecName(m_Codec), m_Codec == SampleCodec::H264 ? " with transform bypass advertised in the SPS" : ""); + } + int qpI = (int)m_VideoQuality.qpI; + int qpP = (int)m_VideoQuality.qpP; + int qpB = (int)m_VideoQuality.qpB; + const int qpMin = m_Codec == SampleCodec::AV1 ? 1 : 0; + const int qpMax = m_Codec == SampleCodec::AV1 ? 255 : 51; + if (ImGui::SliderInt("QP I / IDR", &qpI, qpMin, qpMax)) + m_VideoQuality.qpI = (uint32_t)qpI; + if (ImGui::SliderInt("QP P", &qpP, qpMin, qpMax)) + m_VideoQuality.qpP = (uint32_t)qpP; + if (ImGui::SliderInt("QP B", &qpB, qpMin, qpMax)) + m_VideoQuality.qpB = (uint32_t)qpB; + if (m_Codec == SampleCodec::AV1) { + int av1BaseQIndex = (int)m_VideoQuality.av1BaseQIndex; + if (ImGui::SliderInt("AV1 base Q index", &av1BaseQIndex, 1, 255)) + m_VideoQuality.av1BaseQIndex = (uint32_t)av1BaseQIndex; + } + ImGui::SliderFloat("Pattern motion", &m_PatternMotionSpeed, 0.0f, 4.0f, "%.2f"); + ImGui::SliderFloat("Pattern detail", &m_PatternDetailStrength, 0.0f, 2.5f, "%.2f"); + ImGui::Checkbox("Show artifact diff", &m_ShowDifference); + if (m_ShowDifference) + ImGui::SliderFloat("Diff amplification", &m_DiffStrength, 1.0f, 20.0f, "%.1f"); + ImGui::Separator(); if (ImGui::BeginTable("PreviewPanels", 2, ImGuiTableFlags_SizingStretchSame)) { ImGui::TableNextColumn(); @@ -1022,7 +1088,7 @@ void Sample::PrepareFrame(uint32_t) { ImGui::TableNextColumn(); width = std::max(200.0f, ImGui::GetContentRegionAvail().x); - DrawTexturePanel(m_DecodePreviewReady ? "Decoded preview" : "Decoded preview pending", m_DecodePreviewReady ? m_DecodePreviewTextureView : nullptr, {width, width * float(m_VideoHeight) / float(m_VideoWidth)}); + DrawTexturePanel(m_DecodePreviewReady ? (m_ShowDifference ? "Artifact diff" : "Decoded preview") : "Decoded preview pending", m_DecodePreviewReady ? m_DecodePreviewTextureView : nullptr, {width, width * float(m_VideoHeight) / float(m_VideoWidth)}); ImGui::EndTable(); } } @@ -1061,6 +1127,8 @@ void Sample::RenderFrame(uint32_t frameIndex) { nri::AttachmentDesc colorAttachmentDesc = {}; colorAttachmentDesc.descriptor = swapChainTexture.colorAttachment; colorAttachmentDesc.clearValue.color.f = {0.03f, 0.03f, 0.03f, 1.0f}; + colorAttachmentDesc.loadOp = nri::LoadOp::CLEAR; + colorAttachmentDesc.storeOp = nri::StoreOp::STORE; nri::RenderingDesc renderingDesc = {}; renderingDesc.colorNum = 1; diff --git a/Source/VideoEncodeDecode/Encoder.cpp b/Source/VideoEncodeDecode/Encoder.cpp index 66e1753..3697662 100644 --- a/Source/VideoEncodeDecode/Encoder.cpp +++ b/Source/VideoEncodeDecode/Encoder.cpp @@ -2,6 +2,7 @@ #include "Encoder.h" +#include #include namespace video_sample { @@ -363,7 +364,10 @@ bool Encoder::Encode(const EncodeRequest& request) { : av1CommonPictureFlags | nri::VideoAV1PictureBits::ERROR_RESILIENT_MODE; av1PictureDesc.renderWidthMinus1 = (uint16_t)((m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_Size.codedWidth : m_Size.videoWidth) - 1); av1PictureDesc.renderHeightMinus1 = (uint16_t)((m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_Size.codedHeight : m_Size.videoHeight) - 1); - av1PictureDesc.baseQIndex = (uint8_t)m_Config.av1BaseQIndex; + const bool av1LosslessFallback = m_Config.codec == SampleCodec::AV1 && request.quality.lossless; + const uint8_t minQ = m_Config.codec == SampleCodec::AV1 ? 1 : 0; + const uint8_t av1BaseQIndex = (uint8_t)(request.quality.lossless ? 1 : std::max(request.quality.av1BaseQIndex, (uint32_t)minQ)); + av1PictureDesc.baseQIndex = av1BaseQIndex; av1PictureDesc.interpolationFilter = 0; av1PictureDesc.txMode = 2; av1PictureDesc.cdefDampingMinus3 = 3; @@ -398,9 +402,9 @@ bool Encoder::Encode(const EncodeRequest& request) { nri::VideoEncodeRateControlDesc rateControlDesc = {}; rateControlDesc.mode = nri::VideoEncodeRateControlMode::CQP; - rateControlDesc.qpI = (uint8_t)m_Config.qpI; - rateControlDesc.qpP = (uint8_t)m_Config.qpP; - rateControlDesc.qpB = (uint8_t)m_Config.qpB; + rateControlDesc.qpI = (uint8_t)(request.quality.lossless ? (av1LosslessFallback ? 1 : 0) : std::max(request.quality.qpI, (uint32_t)minQ)); + rateControlDesc.qpP = (uint8_t)(request.quality.lossless ? (av1LosslessFallback ? 1 : 0) : std::max(request.quality.qpP, (uint32_t)minQ)); + rateControlDesc.qpB = (uint8_t)(request.quality.lossless ? (av1LosslessFallback ? 1 : 0) : std::max(request.quality.qpB, (uint32_t)minQ)); rateControlDesc.frameRateNumerator = 30; rateControlDesc.frameRateDenominator = 1; diff --git a/Source/VideoEncodeDecode/Encoder.h b/Source/VideoEncodeDecode/Encoder.h index a183523..f48f857 100644 --- a/Source/VideoEncodeDecode/Encoder.h +++ b/Source/VideoEncodeDecode/Encoder.h @@ -9,6 +9,7 @@ namespace video_sample { struct EncodeRequest { nri::Buffer* nv12Buffer = nullptr; const Nv12BufferLayout* nv12Layout = nullptr; + VideoQuality quality = {}; float timeSec = 0.0f; }; diff --git a/Source/VideoEncodeDecode/Shared.h b/Source/VideoEncodeDecode/Shared.h index 0a56ffd..d36fae5 100644 --- a/Source/VideoEncodeDecode/Shared.h +++ b/Source/VideoEncodeDecode/Shared.h @@ -75,11 +75,15 @@ struct VideoConfig { SampleCodec codec = SampleCodec::H264; uint32_t videoWidth = DEFAULT_VIDEO_WIDTH; uint32_t videoHeight = DEFAULT_VIDEO_HEIGHT; + bool av1PFrameVisual = false; +}; + +struct VideoQuality { uint32_t qpI = 20; uint32_t qpP = 22; uint32_t qpB = 24; uint32_t av1BaseQIndex = 20; - bool av1PFrameVisual = false; + bool lossless = false; }; struct VideoSize { @@ -203,7 +207,7 @@ inline Nv12BufferLayout MakeNv12BufferLayout(const nri::DeviceDesc& deviceDesc, inline CodecParameters MakeCodecParameters(nri::GraphicsAPI graphicsAPI, uint32_t codedWidth, uint32_t codedHeight) { CodecParameters params = {}; - params.h264Sps.flags = nri::VideoH264SequenceParameterSetBits::DIRECT_8X8_INFERENCE | nri::VideoH264SequenceParameterSetBits::FRAME_MBS_ONLY; + params.h264Sps.flags = nri::VideoH264SequenceParameterSetBits::DIRECT_8X8_INFERENCE | nri::VideoH264SequenceParameterSetBits::FRAME_MBS_ONLY | nri::VideoH264SequenceParameterSetBits::QPPRIME_Y_ZERO_TRANSFORM_BYPASS; params.h264Sps.profileIdc = 100; params.h264Sps.levelIdc = 42; params.h264Sps.chromaFormatIdc = 1; From bec4947624debeb8cd23d2b8cab14429efdc3328 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 12:12:15 +0200 Subject: [PATCH 19/28] Update NRIFramework video TODO pointer --- External/NRIFramework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/External/NRIFramework b/External/NRIFramework index 3f3ffdb..9a6b82d 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit 3f3ffdb3a3a9ea087582dff6a0981d1979958d2a +Subproject commit 9a6b82d6d428dee171f2a5c4bf35d513475e6f9d From 936723e51c6d6c470b491e9890f09a71ddf6f74e Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 16:19:39 +0200 Subject: [PATCH 20/28] Improve video encode sample controls --- .gitignore | 1 + External/NRIFramework | 2 +- Scripts/SmokeVideoEncodeDecode.ps1 | 276 +++++++++++++++++++++++++++ Source/VideoEncodeDecode.cpp | 14 +- Source/VideoEncodeDecode/Decoder.cpp | 112 ++++++++--- Source/VideoEncodeDecode/Decoder.h | 2 + Source/VideoEncodeDecode/Encoder.cpp | 134 +++++++++++-- Source/VideoEncodeDecode/Encoder.h | 10 + Source/VideoEncodeDecode/Shared.h | 12 ++ 9 files changed, 522 insertions(+), 41 deletions(-) create mode 100644 Scripts/SmokeVideoEncodeDecode.ps1 diff --git a/.gitignore b/.gitignore index e251e72..92bbd24 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ _Build/ _Shaders/ _Data/ _Tmp/ +_Smoke/ pso_cache.bin # can be a symbolic link diff --git a/External/NRIFramework b/External/NRIFramework index 9a6b82d..6294ab4 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit 9a6b82d6d428dee171f2a5c4bf35d513475e6f9d +Subproject commit 6294ab479a156bce707ccc83825e8c5914fadb9f diff --git a/Scripts/SmokeVideoEncodeDecode.ps1 b/Scripts/SmokeVideoEncodeDecode.ps1 new file mode 100644 index 0000000..0a23b0f --- /dev/null +++ b/Scripts/SmokeVideoEncodeDecode.ps1 @@ -0,0 +1,276 @@ +param( + [double] $DurationSec = 6.0, + [int] $WindowWidth = 3840, + [int] $WindowHeight = 2160, + [int] $VideoWidth = 1280, + [int] $VideoHeight = 720, + [string[]] $Apis = @("D3D12"), + [string] $CaseFilter = "", + [int] $ScreenshotDelayMs = 2500, + [switch] $NoBuild, + [switch] $NoScreenshots, + [switch] $IncludeVulkan, + [switch] $IncludeDebug +) + +$ErrorActionPreference = "Stop" + +if ($DurationSec -lt 2.0) { + Write-Warning "DurationSec is below 2 seconds; this is only useful for script debugging, not smoke coverage." +} + +$RepoRoot = Split-Path -Parent $PSScriptRoot +$ExePath = Join-Path $RepoRoot "_Bin\Release\VideoEncodeDecode.exe" +$Timestamp = Get-Date -Format "yyyyMMdd_HHmmss" +$OutDir = Join-Path $RepoRoot "_Smoke\VideoEncodeDecode_$Timestamp" +$LogPath = Join-Path $OutDir "VideoEncodeDecodeSmoke.log" + +if ($IncludeVulkan -and -not ($Apis -contains "VULKAN")) { + $Apis += "VULKAN" +} + +New-Item -ItemType Directory -Force -Path $OutDir | Out-Null + +function Write-Log { + param([string] $Message) + $line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff"), $Message + $line | Tee-Object -FilePath $LogPath -Append +} + +if (-not $NoBuild) { + Write-Log "Building VideoEncodeDecode Release target..." + $buildLog = Join-Path $OutDir "build.tmp.log" + $build = Start-Process -FilePath "cmake" -ArgumentList @("--build", "_Build", "--config", "Release", "--target", "VideoEncodeDecode") -WorkingDirectory $RepoRoot -NoNewWindow -Wait -PassThru -RedirectStandardOutput $buildLog -RedirectStandardError "$buildLog.err" + Get-Content $buildLog, "$buildLog.err" -ErrorAction SilentlyContinue | Add-Content $LogPath + Remove-Item $buildLog, "$buildLog.err" -ErrorAction SilentlyContinue + if ($build.ExitCode -ne 0) { + Write-Log "BUILD FAILED: exit code $($build.ExitCode)" + exit $build.ExitCode + } +} + +if (-not (Test-Path $ExePath)) { + Write-Log "Executable not found: $ExePath" + exit 1 +} + +Add-Type @" +using System; +using System.Runtime.InteropServices; + +public static class Win32WindowCapture { + [StructLayout(LayoutKind.Sequential)] + public struct RECT { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + + [DllImport("user32.dll")] + public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); + + [DllImport("user32.dll")] + public static extern bool SetForegroundWindow(IntPtr hWnd); +} +"@ + +if (-not $NoScreenshots) { + Add-Type -AssemblyName System.Drawing +} + +function Wait-MainWindow { + param( + [System.Diagnostics.Process] $Process, + [int] $TimeoutMs = 8000 + ) + + $deadline = [Environment]::TickCount + $TimeoutMs + while (-not $Process.HasExited -and [Environment]::TickCount -lt $deadline) { + $Process.Refresh() + if ($Process.MainWindowHandle -ne [IntPtr]::Zero) { + return $Process.MainWindowHandle + } + Start-Sleep -Milliseconds 100 + } + + return [IntPtr]::Zero +} + +function Capture-Window { + param( + [IntPtr] $Handle, + [string] $Path + ) + + if ($NoScreenshots -or $Handle -eq [IntPtr]::Zero) { + return $false + } + + Start-Sleep -Milliseconds $ScreenshotDelayMs + + $rect = New-Object Win32WindowCapture+RECT + if (-not [Win32WindowCapture]::GetWindowRect($Handle, [ref] $rect)) { + return $false + } + + $width = $rect.Right - $rect.Left + $height = $rect.Bottom - $rect.Top + if ($width -le 0 -or $height -le 0) { + return $false + } + + $bitmap = New-Object System.Drawing.Bitmap $width, $height + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + try { + $graphics.CopyFromScreen($rect.Left, $rect.Top, 0, 0, $bitmap.Size) + $bitmap.Save($Path, [System.Drawing.Imaging.ImageFormat]::Png) + } finally { + $graphics.Dispose() + $bitmap.Dispose() + } + + return $true +} + +function New-Case { + param( + [string] $Name, + [string] $Api, + [string] $Codec, + [string] $FrameArgName, + [string] $FrameArgValue, + [bool] $Lossless + ) + + $args = @( + "--api=$Api", + "--width=$WindowWidth", + "--height=$WindowHeight", + "--videoWidth=$VideoWidth", + "--videoHeight=$VideoHeight", + "--codec=$Codec", + "--timeLimit=$DurationSec", + "--alwaysActive" + ) + + if ($FrameArgName) { + $args += "--$FrameArgName=$FrameArgValue" + } + if ($Lossless) { + $args += "--lossless" + } + if ($IncludeDebug) { + $args += "--debugAPI" + $args += "--debugNRI" + } + + [pscustomobject]@{ + Name = $Name + Args = $args + } +} + +$cases = New-Object System.Collections.Generic.List[object] +foreach ($api in $Apis) { + foreach ($codec in @("H264", "H265")) { + $h26Frames = @("IDR", "P", "B") + if ($api -eq "VULKAN" -and $codec -eq "H265") { + $h26Frames = @("IDR", "P") + } + foreach ($frame in $h26Frames) { + foreach ($lossless in @($false, $true)) { + $suffix = if ($lossless) { "lossless" } else { "cqp" } + $cases.Add((New-Case -Name "$api-$codec-$frame-$suffix" -Api $api -Codec $codec -FrameArgName "h26Frame" -FrameArgValue $frame -Lossless $lossless)) + } + } + } + + foreach ($frame in @("IDR", "P")) { + foreach ($lossless in @($false, $true)) { + $suffix = if ($lossless) { "near-lossless" } else { "cqp" } + $cases.Add((New-Case -Name "$api-AV1-$frame-$suffix" -Api $api -Codec "AV1" -FrameArgName "av1Frame" -FrameArgValue $frame -Lossless $lossless)) + } + } +} + +Write-Log "VideoEncodeDecode smoke started" +Write-Log "Output directory: $OutDir" +Write-Log "Duration per run: $DurationSec seconds" +Write-Log "Window: ${WindowWidth}x${WindowHeight}, video: ${VideoWidth}x${VideoHeight}" +Write-Log "APIs: $($Apis -join ', ')" +Write-Log "Cases: $($cases.Count)" + +if ($CaseFilter) { + $cases = @($cases | Where-Object { $_.Name -match $CaseFilter }) + Write-Log "Case filter: $CaseFilter" + Write-Log "Filtered cases: $($cases.Count)" + if ($cases.Count -eq 0) { + Write-Log "No cases matched filter" + exit 1 + } +} + +$failures = 0 +$index = 0 +foreach ($case in $cases) { + $index++ + $caseOut = Join-Path $OutDir "$($case.Name).stdout.tmp" + $caseErr = Join-Path $OutDir "$($case.Name).stderr.tmp" + $screenshot = Join-Path $OutDir "$($case.Name).png" + + Write-Log "" + Write-Log "[$index/$($cases.Count)] START $($case.Name)" + Write-Log "Command: `"$ExePath`" $($case.Args -join ' ')" + + $process = Start-Process -FilePath $ExePath -ArgumentList $case.Args -WorkingDirectory $RepoRoot -PassThru -RedirectStandardOutput $caseOut -RedirectStandardError $caseErr + $handle = Wait-MainWindow -Process $process + + if ($handle -ne [IntPtr]::Zero) { + Write-Log "Window handle: $handle" + [Win32WindowCapture]::SetForegroundWindow($handle) | Out-Null + if (Capture-Window -Handle $handle -Path $screenshot) { + Write-Log "Screenshot: $screenshot" + } elseif (-not $NoScreenshots) { + Write-Log "Screenshot capture failed" + } + } else { + Write-Log "Window handle not found before timeout" + } + + $maxWaitSec = [Math]::Ceiling($DurationSec + 12) + $timedOut = $false + if (-not $process.WaitForExit($maxWaitSec * 1000)) { + Write-Log "TIMEOUT: killing process after $maxWaitSec seconds" + $timedOut = $true + $process.Kill() + $process.WaitForExit() + } + $process.Refresh() + + $exitCode = $process.ExitCode + if ($null -eq $exitCode -or "$exitCode" -eq "") { + $exitCode = 0 + } + Write-Log "ExitCode: $exitCode" + Add-Content -Path $LogPath -Value "----- stdout: $($case.Name) -----" + Get-Content $caseOut -ErrorAction SilentlyContinue | Add-Content $LogPath + Add-Content -Path $LogPath -Value "----- stderr: $($case.Name) -----" + Get-Content $caseErr -ErrorAction SilentlyContinue | Add-Content $LogPath + $caseText = (Get-Content $caseOut, $caseErr -ErrorAction SilentlyContinue) -join "`n" + Remove-Item $caseOut, $caseErr -ErrorAction SilentlyContinue + + if ($timedOut -or $exitCode -ne 0 -or $caseText -match "(?m)\b(ERROR|FAILED)\b") { + $failures++ + Write-Log "FAILED $($case.Name)" + } else { + Write-Log "OK $($case.Name)" + } +} + +Write-Log "" +Write-Log "VideoEncodeDecode smoke finished: $($cases.Count - $failures) passed, $failures failed" +Write-Log "Log: $LogPath" +if ($failures -ne 0) { + exit 1 +} diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index ae8f88f..3989766 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -279,6 +279,7 @@ class Sample : public SampleBase { std::string m_PreviewStatus = "Initializing preview"; std::string m_CodecArg = "H264"; std::string m_AV1FrameArg = "IDR"; + std::string m_H26FrameArg = "IDR"; uint32_t m_VideoWidth = DEFAULT_VIDEO_WIDTH; uint32_t m_VideoHeight = DEFAULT_VIDEO_HEIGHT; uint32_t m_CodedVideoWidth = DEFAULT_VIDEO_WIDTH; @@ -294,6 +295,7 @@ class Sample : public SampleBase { float m_DiffStrength = 6.0f; PatternConstants m_PendingEncodedPatternConstants = {}; SampleCodec m_Codec = SampleCodec::H264; + video_sample::VisualFrameMode m_H26FrameMode = video_sample::VisualFrameMode::IDR; double m_StartTimeSec = 0.0; double m_LastRoundTripTimeSec = -ROUND_TRIP_INTERVAL_SEC; bool m_VideoReady = false; @@ -364,6 +366,7 @@ Sample::~Sample() { void Sample::InitCmdLine(cmdline::parser& cmdLine) { cmdLine.add("codec", 0, "video codec: H264, H265, or AV1", false, m_CodecArg, cmdline::oneof("H264", "H265", "AV1")); cmdLine.add("av1Frame", 0, "AV1 visual frame permutation: IDR or P", false, m_AV1FrameArg, cmdline::oneof("IDR", "P")); + cmdLine.add("h26Frame", 0, "H.264/H.265 visual frame permutation: IDR, P, or B", false, m_H26FrameArg, cmdline::oneof("IDR", "P", "B")); cmdLine.add("videoWidth", 0, "NV12 video encode/decode width", false, m_VideoWidth); cmdLine.add("videoHeight", 0, "NV12 video encode/decode height", false, m_VideoHeight); cmdLine.add("qpI", 0, "CQP quantizer for I/IDR frames", false, m_QpI); @@ -376,6 +379,7 @@ void Sample::InitCmdLine(cmdline::parser& cmdLine) { void Sample::ReadCmdLine(cmdline::parser& cmdLine) { m_CodecArg = cmdLine.get("codec"); m_AV1FrameArg = cmdLine.get("av1Frame"); + m_H26FrameArg = cmdLine.get("h26Frame"); m_VideoWidth = cmdLine.get("videoWidth"); m_VideoHeight = cmdLine.get("videoHeight"); if (!cmdLine.exist("videoWidth") && cmdLine.exist("width")) @@ -388,6 +392,7 @@ void Sample::ReadCmdLine(cmdline::parser& cmdLine) { m_AV1BaseQIndex = cmdLine.get("av1BaseQIndex"); m_Lossless = cmdLine.exist("lossless"); m_Codec = m_CodecArg == "H265" ? SampleCodec::H265 : (m_CodecArg == "AV1" ? SampleCodec::AV1 : SampleCodec::H264); + m_H26FrameMode = m_H26FrameArg == "B" ? video_sample::VisualFrameMode::B : (m_H26FrameArg == "P" ? video_sample::VisualFrameMode::P : video_sample::VisualFrameMode::IDR); m_AV1PFrameVisual = m_Codec == SampleCodec::AV1 && m_AV1FrameArg == "P"; } @@ -410,6 +415,7 @@ bool Sample::Initialize(nri::GraphicsAPI graphicsAPI, bool) { m_VideoConfig.videoWidth = m_VideoWidth; m_VideoConfig.videoHeight = m_VideoHeight; m_VideoConfig.av1PFrameVisual = m_AV1PFrameVisual; + m_VideoConfig.h26FrameMode = m_H26FrameMode; m_VideoQuality.qpI = m_QpI; m_VideoQuality.qpP = m_QpP; m_VideoQuality.qpB = m_QpB; @@ -962,6 +968,9 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { return false; } + if (!encodedFrame.isDisplayFrame) + return TrySubmitEncodeAndMetadataReadback(timeSec); + PatternConstants patternConstants = m_HasPendingEncodedPatternConstants ? m_PendingEncodedPatternConstants : MakePatternConstants(OP_NV12_TO_PREVIEW, timeSec); patternConstants.operation = OP_NV12_TO_PREVIEW; patternConstants.diffStrength = m_DiffStrength; @@ -1035,12 +1044,13 @@ void Sample::PrepareFrame(uint32_t) { const uint32_t effectiveQpP = m_VideoQuality.lossless ? minCodecQp : std::max(m_VideoQuality.qpP, minCodecQp); const uint32_t effectiveQpB = m_VideoQuality.lossless ? minCodecQp : std::max(m_VideoQuality.qpB, minCodecQp); ImGui::Text("Codec: %s, format: NV12, size: %ux%u", GetCodecName(m_Codec), m_VideoWidth, m_VideoHeight); - ImGui::Text("Startup-only: codec, video size, AV1 frame mode"); + ImGui::Text("Startup-only: codec, video size, AV1/H.26x frame mode"); ImGui::Text("CQP: I=%u, P=%u, B=%u%s", effectiveQpI, effectiveQpP, effectiveQpB, m_Codec == SampleCodec::AV1 ? ", AV1 baseQIndex follows below" : ""); if (m_Codec == SampleCodec::AV1) { const uint32_t effectiveAv1BaseQIndex = m_VideoQuality.lossless ? 1 : std::max(m_VideoQuality.av1BaseQIndex, 1u); ImGui::Text("AV1: frame=%s, baseQIndex=%u", m_AV1FrameArg.c_str(), effectiveAv1BaseQIndex); - } + } else + ImGui::Text("H.26x frame permutation: %s", m_H26FrameArg.c_str()); ImGui::TextWrapped("Video: %s", m_VideoStatus.c_str()); ImGui::TextWrapped("Preview: %s", m_PreviewStatus.c_str()); ImGui::Text("Encode queue: %s, decode queue: %s", m_VideoEncodeQueue ? "yes" : "no", m_VideoDecodeQueue ? "yes" : "no"); diff --git a/Source/VideoEncodeDecode/Decoder.cpp b/Source/VideoEncodeDecode/Decoder.cpp index 58a7fb4..97e736b 100644 --- a/Source/VideoEncodeDecode/Decoder.cpp +++ b/Source/VideoEncodeDecode/Decoder.cpp @@ -14,6 +14,8 @@ Decoder::~Decoder() { nri::VideoInterface& video = *m_Context.video; if (video.DestroyVideoPicture) { + if (m_BDecodePicture) + video.DestroyVideoPicture(m_BDecodePicture); if (m_AV1PDecodePicture) video.DestroyVideoPicture(m_AV1PDecodePicture); if (m_DecodePicture) @@ -30,6 +32,8 @@ Decoder::~Decoder() { nri.DestroyTexture(m_DecodeTexture); if (m_AV1PDecodeTexture) nri.DestroyTexture(m_AV1PDecodeTexture); + if (m_BDecodeTexture) + nri.DestroyTexture(m_BDecodeTexture); } bool Decoder::Initialize(const VideoContext& context, const VideoConfig& config, const VideoSize& size, const CodecParameters& codecParameters) { @@ -103,20 +107,31 @@ bool Decoder::Initialize(const VideoContext& context, const VideoConfig& config, } nri.SetDebugName(m_DecodeTexture, "VideoDecodeTexture"); - if (m_Config.av1PFrameVisual) { + const bool needsSecondDecodePicture = m_Config.av1PFrameVisual || (m_Config.codec != SampleCodec::AV1 && m_Config.h26FrameMode != VisualFrameMode::IDR); + const bool needsThirdDecodePicture = m_Config.codec != SampleCodec::AV1 && m_Config.h26FrameMode == VisualFrameMode::B; + if (needsSecondDecodePicture) { if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_AV1PDecodeTexture) != nri::Result::SUCCESS) { m_Status = "Failed to create second NV12 decode texture"; return false; } - nri.SetDebugName(m_AV1PDecodeTexture, "VideoAV1PDecodeTexture"); + nri.SetDebugName(m_AV1PDecodeTexture, "VideoSecondDecodeTexture"); + } + if (needsThirdDecodePicture) { + if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_BDecodeTexture) != nri::Result::SUCCESS) { + m_Status = "Failed to create third NV12 decode texture"; + return false; + } + nri.SetDebugName(m_BDecodeTexture, "VideoThirdDecodeTexture"); } if (!SubmitOneTime(nri, *m_Context.graphicsQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::TextureBarrierDesc textureBarriers[2] = {}; + nri::TextureBarrierDesc textureBarriers[3] = {}; uint32_t textureBarrierNum = 0; textureBarriers[textureBarrierNum++].texture = m_DecodeTexture; if (m_AV1PDecodeTexture) textureBarriers[textureBarrierNum++].texture = m_AV1PDecodeTexture; + if (m_BDecodeTexture) + textureBarriers[textureBarrierNum++].texture = m_BDecodeTexture; for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { if (!textureBarrier.texture) @@ -155,13 +170,20 @@ bool Decoder::Initialize(const VideoContext& context, const VideoConfig& config, m_Status = "Failed to create decode picture"; return false; } - if (m_Config.av1PFrameVisual) { + if (needsSecondDecodePicture) { decodePictureDesc.texture = m_AV1PDecodeTexture; if (video.CreateVideoPicture(*m_Context.device, decodePictureDesc, m_AV1PDecodePicture) != nri::Result::SUCCESS) { m_Status = "Failed to create second decode picture"; return false; } } + if (needsThirdDecodePicture) { + decodePictureDesc.texture = m_BDecodeTexture; + if (video.CreateVideoPicture(*m_Context.device, decodePictureDesc, m_BDecodePicture) != nri::Result::SUCCESS) { + m_Status = "Failed to create third decode picture"; + return false; + } + } m_Ready = true; m_Status = std::string(GetCodecName(m_Config.codec)) + " decoder initialized"; @@ -267,6 +289,10 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { const EncodedFrame& frame = *request.frame; const nri::VideoAV1EncodeDecodeInfo* av1DecodeInfo = frame.hasAv1DecodeInfo ? &frame.av1DecodeInfo : nullptr; const bool av1PFrame = frame.isAv1PFrame; + const bool h26Codec = m_Config.codec != SampleCodec::AV1; + const bool h26ReferenceFrame = h26Codec && frame.frameType != nri::VideoEncodeFrameType::B; + nri::VideoPicture* dstPicture = av1PFrame ? m_AV1PDecodePicture : (frame.outputSlot == 2 ? m_BDecodePicture : (frame.outputSlot == 1 ? m_AV1PDecodePicture : m_DecodePicture)); + nri::Texture* dstTexture = av1PFrame ? m_AV1PDecodeTexture : (frame.outputSlot == 2 ? m_BDecodeTexture : (frame.outputSlot == 1 ? m_AV1PDecodeTexture : m_DecodeTexture)); uint64_t decodeBitstreamRange = 0; uint32_t pictureOffset = 0; @@ -276,30 +302,57 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { NRIInterface& nri = *m_Context.nri; nri::VideoInterface& video = *m_Context.video; - const uint32_t decodeFrameIndex = m_DecodeFrameIndex++; - const uint32_t decodeSlot = m_Config.codec == SampleCodec::AV1 ? 0 : decodeFrameIndex % 16; + const uint32_t decodeSlot = m_Config.codec == SampleCodec::AV1 ? 0 : frame.outputSlot; const uint32_t pictureOffsets[] = {pictureOffset}; nri::VideoH264DecodePictureDesc h264DecodePicture = {}; - h264DecodePicture.flags = nri::VideoH264DecodePictureBits::IDR | nri::VideoH264DecodePictureBits::INTRA | nri::VideoH264DecodePictureBits::REFERENCE; + h264DecodePicture.flags = nri::VideoH264DecodePictureBits::NONE; + if (frame.frameType == nri::VideoEncodeFrameType::IDR) + h264DecodePicture.flags |= nri::VideoH264DecodePictureBits::IDR | nri::VideoH264DecodePictureBits::INTRA; + if (h26ReferenceFrame) + h264DecodePicture.flags |= nri::VideoH264DecodePictureBits::REFERENCE; h264DecodePicture.sequenceParameterSetId = m_CodecParameters.h264Sps.sequenceParameterSetId; h264DecodePicture.pictureParameterSetId = m_CodecParameters.h264Pps.pictureParameterSetId; - h264DecodePicture.frameNum = (uint16_t)(decodeFrameIndex & 0xF); - h264DecodePicture.idrPictureId = (uint16_t)(1 + (decodeFrameIndex & 0xFFFF)); - h264DecodePicture.topFieldOrderCount = 0; - h264DecodePicture.bottomFieldOrderCount = 0; + h264DecodePicture.frameNum = (uint16_t)(frame.frameIndex & 0xF); + h264DecodePicture.idrPictureId = frame.frameType == nri::VideoEncodeFrameType::IDR ? 1 : 0; + h264DecodePicture.topFieldOrderCount = frame.pictureOrderCount; + h264DecodePicture.bottomFieldOrderCount = frame.pictureOrderCount; h264DecodePicture.sliceOffsets = pictureOffsets; h264DecodePicture.sliceOffsetNum = helper::GetCountOf(pictureOffsets); h264DecodePicture.referenceSlot = decodeSlot; + nri::VideoReference h26DecodeReferences[2] = {}; + nri::VideoH264DecodeReferenceDesc h264DecodeReferences[2] = {}; + nri::VideoH265ReferenceDesc h265DecodeReferences[2] = {}; + uint32_t h26DecodeReferenceNum = 0; + if (h26Codec && frame.frameType != nri::VideoEncodeFrameType::IDR) { + h26DecodeReferences[h26DecodeReferenceNum] = {m_DecodePicture, 0}; + h264DecodeReferences[h26DecodeReferenceNum] = {nri::VideoH264DecodeReferenceBits::NONE, 0, 0, 0, 0}; + h265DecodeReferences[h26DecodeReferenceNum] = {0, 0, 0, nri::VideoEncodeFrameType::IDR, 0, 0}; + h26DecodeReferenceNum++; + if (frame.frameType == nri::VideoEncodeFrameType::B) { + h26DecodeReferences[h26DecodeReferenceNum] = {m_AV1PDecodePicture, 1}; + h264DecodeReferences[h26DecodeReferenceNum] = {nri::VideoH264DecodeReferenceBits::NONE, 2, 1, 2, 2}; + h265DecodeReferences[h26DecodeReferenceNum] = {1, 2, 0, nri::VideoEncodeFrameType::P, 0, 1}; + h26DecodeReferenceNum++; + } + h264DecodePicture.references = h264DecodeReferences; + h264DecodePicture.referenceNum = h26DecodeReferenceNum; + } nri::VideoH265DecodePictureDesc h265DecodePicture = {}; - h265DecodePicture.flags = nri::VideoH265DecodePictureBits::IRAP | nri::VideoH265DecodePictureBits::IDR | nri::VideoH265DecodePictureBits::REFERENCE; + h265DecodePicture.flags = nri::VideoH265DecodePictureBits::NONE; + if (frame.frameType == nri::VideoEncodeFrameType::IDR) + h265DecodePicture.flags |= nri::VideoH265DecodePictureBits::IRAP | nri::VideoH265DecodePictureBits::IDR; + if (h26ReferenceFrame) + h265DecodePicture.flags |= nri::VideoH265DecodePictureBits::REFERENCE; h265DecodePicture.videoParameterSetId = m_CodecParameters.h265Vps.videoParameterSetId; h265DecodePicture.sequenceParameterSetId = m_CodecParameters.h265Sps.sequenceParameterSetId; h265DecodePicture.pictureParameterSetId = m_CodecParameters.h265Pps.pictureParameterSetId; - h265DecodePicture.pictureOrderCount = (int32_t)decodeFrameIndex; + h265DecodePicture.pictureOrderCount = frame.pictureOrderCount; h265DecodePicture.sliceSegmentOffsets = pictureOffsets; h265DecodePicture.sliceSegmentOffsetNum = helper::GetCountOf(pictureOffsets); + h265DecodePicture.references = h265DecodeReferences; + h265DecodePicture.referenceNum = h26DecodeReferenceNum; nri::VideoAV1EncodeDecodeInfo av1Info = {}; if (av1DecodeInfo) { @@ -332,16 +385,16 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { decodeDesc.parameters = m_Parameters; decodeDesc.bitstream.buffer = m_DecodeBitstreamBuffer; decodeDesc.bitstream.size = decodeBitstreamRange; - decodeDesc.dstPicture = av1PFrame ? m_AV1PDecodePicture : m_DecodePicture; - decodeDesc.references = av1PFrame ? &av1DecodeReference : nullptr; - decodeDesc.referenceNum = av1PFrame ? 1u : 0u; + decodeDesc.dstPicture = dstPicture; + decodeDesc.references = av1PFrame ? &av1DecodeReference : (h26DecodeReferenceNum ? h26DecodeReferences : nullptr); + decodeDesc.referenceNum = av1PFrame ? 1u : h26DecodeReferenceNum; decodeDesc.dstSlot = av1PFrame ? 1u : decodeSlot; decodeDesc.h264PictureDesc = m_Config.codec == SampleCodec::H264 ? &h264DecodePicture : nullptr; decodeDesc.h265PictureDesc = m_Config.codec == SampleCodec::H265 ? &h265DecodePicture : nullptr; decodeDesc.av1PictureDesc = av1DecodeInfo ? &av1Info.picture : nullptr; nri::VideoDecodePictureStates decodePictureStates = {}; - if (video.GetVideoDecodePictureStates(*(av1PFrame ? m_AV1PDecodePicture : m_DecodePicture), decodePictureStates) != nri::Result::SUCCESS) { + if (video.GetVideoDecodePictureStates(*dstPicture, decodePictureStates) != nri::Result::SUCCESS) { m_Status = "Failed to query video decode picture states"; return false; } @@ -358,14 +411,14 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { nri::TextureBarrierDesc textureBarriers[3] = {}; uint32_t textureBarrierNum = 0; - textureBarriers[textureBarrierNum].texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; + textureBarriers[textureBarrierNum].texture = dstTexture; textureBarriers[textureBarrierNum].before = {nri::AccessBits::NONE, m_Context.graphicsAPI == nri::GraphicsAPI::VK && !av1DecodeInfo ? nri::Layout::UNDEFINED : nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[textureBarrierNum].after = decodePictureStates.decodeWrite; textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; textureBarrierNum++; - if (av1PFrame) { + if (av1PFrame || h26DecodeReferenceNum) { textureBarriers[textureBarrierNum].texture = m_DecodeTexture; textureBarriers[textureBarrierNum].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; textureBarriers[textureBarrierNum].after = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; @@ -374,6 +427,15 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; textureBarrierNum++; } + if (h26DecodeReferenceNum == 2) { + textureBarriers[textureBarrierNum].texture = m_AV1PDecodeTexture; + textureBarriers[textureBarrierNum].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[textureBarrierNum].after = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[textureBarrierNum].mipNum = nri::REMAINING; + textureBarriers[textureBarrierNum].layerNum = nri::REMAINING; + textureBarriers[textureBarrierNum].planes = nri::PlaneBits::ALL; + textureBarrierNum++; + } nri::BarrierDesc barrierDesc = {}; barrierDesc.buffers = m_Context.graphicsAPI == nri::GraphicsAPI::D3D12 ? nullptr : &bufferBarrier; @@ -386,10 +448,14 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { if (decodePictureStates.releaseAfterDecode) { textureBarriers[0].before = decodePictureStates.decodeWrite; textureBarriers[0].after = decodePictureStates.afterDecode; - if (av1PFrame) { + if (av1PFrame || h26DecodeReferenceNum) { textureBarriers[1].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; } + if (h26DecodeReferenceNum == 2) { + textureBarriers[2].before = {nri::AccessBits::VIDEO_DECODE_READ, nri::Layout::VIDEO_DECODE_DPB, nri::StageBits::VIDEO_DECODE}; + textureBarriers[2].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + } barrierDesc.buffers = nullptr; barrierDesc.bufferNum = 0; nri.CmdBarrier(commandBuffer, barrierDesc); @@ -402,7 +468,7 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { nri::Queue* readbackQueue = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_Context.decodeQueue : m_Context.graphicsQueue; if (!SubmitOneTime(nri, *readbackQueue, [&](nri::CommandBuffer& commandBuffer) { nri::TextureBarrierDesc textureBarrier = {}; - textureBarrier.texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; + textureBarrier.texture = dstTexture; textureBarrier.before = decodePictureStates.graphicsBefore; textureBarrier.after = {nri::AccessBits::COPY_SOURCE, nri::Layout::COPY_SOURCE, nri::StageBits::COPY}; textureBarrier.mipNum = nri::REMAINING; @@ -460,8 +526,8 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { } decodedFrame = {}; - decodedFrame.texture = av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture; - decodedFrame.picture = av1PFrame ? m_AV1PDecodePicture : m_DecodePicture; + decodedFrame.texture = dstTexture; + decodedFrame.picture = dstPicture; decodedFrame.pictureStates = decodePictureStates; decodedFrame.isAv1PFrame = av1PFrame; diff --git a/Source/VideoEncodeDecode/Decoder.h b/Source/VideoEncodeDecode/Decoder.h index b3b34b7..dacd0b3 100644 --- a/Source/VideoEncodeDecode/Decoder.h +++ b/Source/VideoEncodeDecode/Decoder.h @@ -46,9 +46,11 @@ class Decoder { nri::VideoSessionParameters* m_Parameters = nullptr; nri::Texture* m_DecodeTexture = nullptr; nri::Texture* m_AV1PDecodeTexture = nullptr; + nri::Texture* m_BDecodeTexture = nullptr; nri::Buffer* m_DecodeBitstreamBuffer = nullptr; nri::VideoPicture* m_DecodePicture = nullptr; nri::VideoPicture* m_AV1PDecodePicture = nullptr; + nri::VideoPicture* m_BDecodePicture = nullptr; uint32_t m_DecodeFrameIndex = 0; bool m_Ready = false; diff --git a/Source/VideoEncodeDecode/Encoder.cpp b/Source/VideoEncodeDecode/Encoder.cpp index 3697662..e2625e2 100644 --- a/Source/VideoEncodeDecode/Encoder.cpp +++ b/Source/VideoEncodeDecode/Encoder.cpp @@ -15,6 +15,8 @@ Encoder::~Encoder() { nri::VideoInterface& video = *m_Context.video; if (video.DestroyVideoPicture) { + if (m_BReconstructedPicture) + video.DestroyVideoPicture(m_BReconstructedPicture); if (m_AV1PReconstructedPicture) video.DestroyVideoPicture(m_AV1PReconstructedPicture); if (m_ReconstructedPicture) @@ -47,6 +49,8 @@ Encoder::~Encoder() { nri.DestroyTexture(m_ReconstructedTexture); if (m_AV1PReconstructedTexture) nri.DestroyTexture(m_AV1PReconstructedTexture); + if (m_BReconstructedTexture) + nri.DestroyTexture(m_BReconstructedTexture); if (m_EncodeTexture) nri.DestroyTexture(m_EncodeTexture); } @@ -66,7 +70,7 @@ bool Encoder::Initialize(const VideoContext& context, const VideoConfig& config, sessionDesc.format = nri::Format::NV12_UNORM; sessionDesc.width = m_Size.codedWidth; sessionDesc.height = m_Size.codedHeight; - sessionDesc.maxReferenceNum = 1; + sessionDesc.maxReferenceNum = NeedsThirdReconstructedPicture() ? 2 : 1; if (video.CreateVideoSession(*m_Context.device, sessionDesc, m_Session) != nri::Result::SUCCESS) { m_Status = std::string("Failed to create ") + GetCodecName(m_Config.codec) + " encode session"; @@ -128,21 +132,30 @@ bool Encoder::Initialize(const VideoContext& context, const VideoConfig& config, } nri.SetDebugName(m_ReconstructedTexture, "VideoReconstructedTexture"); - if (m_Config.av1PFrameVisual) { + if (NeedsSecondReconstructedPicture()) { if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_AV1PReconstructedTexture) != nri::Result::SUCCESS) { m_Status = "Failed to create second NV12 reconstructed texture"; return false; } - nri.SetDebugName(m_AV1PReconstructedTexture, "VideoAV1PReconstructedTexture"); + nri.SetDebugName(m_AV1PReconstructedTexture, "VideoSecondReconstructedTexture"); + } + if (NeedsThirdReconstructedPicture()) { + if (nri.CreateCommittedTexture(*m_Context.device, nri::MemoryLocation::DEVICE, 0.0f, textureDesc, m_BReconstructedTexture) != nri::Result::SUCCESS) { + m_Status = "Failed to create third NV12 reconstructed texture"; + return false; + } + nri.SetDebugName(m_BReconstructedTexture, "VideoThirdReconstructedTexture"); } if (!SubmitOneTime(nri, *m_Context.graphicsQueue, [&](nri::CommandBuffer& commandBuffer) { - nri::TextureBarrierDesc textureBarriers[3] = {}; + nri::TextureBarrierDesc textureBarriers[4] = {}; uint32_t textureBarrierNum = 0; textureBarriers[textureBarrierNum++].texture = m_EncodeTexture; textureBarriers[textureBarrierNum++].texture = m_ReconstructedTexture; if (m_AV1PReconstructedTexture) textureBarriers[textureBarrierNum++].texture = m_AV1PReconstructedTexture; + if (m_BReconstructedTexture) + textureBarriers[textureBarrierNum++].texture = m_BReconstructedTexture; for (nri::TextureBarrierDesc& textureBarrier : textureBarriers) { if (!textureBarrier.texture) @@ -229,13 +242,20 @@ bool Encoder::Initialize(const VideoContext& context, const VideoConfig& config, m_Status = "Failed to create reconstructed picture"; return false; } - if (m_Config.av1PFrameVisual) { + if (NeedsSecondReconstructedPicture()) { reconstructedPictureDesc.texture = m_AV1PReconstructedTexture; if (video.CreateVideoPicture(*m_Context.device, reconstructedPictureDesc, m_AV1PReconstructedPicture) != nri::Result::SUCCESS) { m_Status = "Failed to create second reconstructed picture"; return false; } } + if (NeedsThirdReconstructedPicture()) { + reconstructedPictureDesc.texture = m_BReconstructedTexture; + if (video.CreateVideoPicture(*m_Context.device, reconstructedPictureDesc, m_BReconstructedPicture) != nri::Result::SUCCESS) { + m_Status = "Failed to create third reconstructed picture"; + return false; + } + } m_Ready = true; m_Status = std::string(GetCodecName(m_Config.codec)) + " encoder initialized"; @@ -246,6 +266,14 @@ bool Encoder::IsEncodingAv1PFrame() const { return m_Config.av1PFrameVisual && m_AV1PFrameStage == 1; } +bool Encoder::NeedsSecondReconstructedPicture() const { + return m_Config.av1PFrameVisual || (m_Config.codec != SampleCodec::AV1 && m_Config.h26FrameMode != VisualFrameMode::IDR); +} + +bool Encoder::NeedsThirdReconstructedPicture() const { + return m_Config.codec != SampleCodec::AV1 && m_Config.h26FrameMode == VisualFrameMode::B; +} + bool Encoder::WriteAnnexBHeadersToUploadBuffer(std::vector& annexBHeaders) { NRIInterface& nri = *m_Context.nri; nri::VideoInterface& video = *m_Context.video; @@ -303,6 +331,28 @@ bool Encoder::Encode(const EncodeRequest& request) { NRIInterface& nri = *m_Context.nri; nri::VideoInterface& video = *m_Context.video; const bool av1PFrame = IsEncodingAv1PFrame(); + const bool h26Codec = m_Config.codec != SampleCodec::AV1; + + nri::VideoEncodeFrameType frameType = av1PFrame ? nri::VideoEncodeFrameType::P : nri::VideoEncodeFrameType::IDR; + uint32_t frameIndex = av1PFrame ? 1 : 0; + int32_t pictureOrderCount = av1PFrame ? 1 : 0; + uint32_t outputSlot = av1PFrame ? 1 : 0; + bool displayFrame = true; + if (h26Codec) { + if (m_Config.h26FrameMode == VisualFrameMode::P) { + frameType = m_H26FrameStage == 0 ? nri::VideoEncodeFrameType::IDR : nri::VideoEncodeFrameType::P; + frameIndex = m_H26FrameStage == 0 ? 0 : 1; + pictureOrderCount = m_H26FrameStage == 0 ? 0 : 1; + outputSlot = m_H26FrameStage == 0 ? 0 : 1; + displayFrame = m_H26FrameStage == 1; + } else if (m_Config.h26FrameMode == VisualFrameMode::B) { + frameType = m_H26FrameStage == 0 ? nri::VideoEncodeFrameType::IDR : (m_H26FrameStage == 1 ? nri::VideoEncodeFrameType::P : nri::VideoEncodeFrameType::B); + frameIndex = m_H26FrameStage == 0 ? 0 : (m_H26FrameStage == 1 ? 2 : 1); + pictureOrderCount = m_H26FrameStage == 0 ? 0 : (m_H26FrameStage == 1 ? 2 : 1); + outputSlot = m_H26FrameStage == 0 ? 0 : (m_H26FrameStage == 1 ? 1 : 2); + displayFrame = m_H26FrameStage == 2; + } + } if (!CopyNv12BufferToTexture(nri, *m_Context.graphicsQueue, *request.nv12Layout, *request.nv12Buffer, *m_EncodeTexture, m_Size.videoWidth, m_Size.videoHeight)) { m_Status = "Failed to upload NV12 source to video texture"; @@ -322,10 +372,10 @@ bool Encoder::Encode(const EncodeRequest& request) { } nri::VideoEncodePictureDesc pictureDesc = {}; - pictureDesc.frameType = av1PFrame ? nri::VideoEncodeFrameType::P : nri::VideoEncodeFrameType::IDR; - pictureDesc.frameIndex = av1PFrame ? 1 : 0; - pictureDesc.pictureOrderCount = av1PFrame ? 1 : 0; - pictureDesc.idrPictureId = av1PFrame ? 0 : 1; + pictureDesc.frameType = frameType; + pictureDesc.frameIndex = frameIndex; + pictureDesc.pictureOrderCount = pictureOrderCount; + pictureDesc.idrPictureId = frameType == nri::VideoEncodeFrameType::IDR ? 1 : 0; uint16_t av1MiColumnStarts[] = {0, (uint16_t)(2 * ((m_Size.codedWidth + 7) >> 3))}; uint16_t av1MiRowStarts[] = {0, (uint16_t)(2 * ((m_Size.codedHeight + 7) >> 3))}; @@ -377,6 +427,27 @@ bool Encoder::Encode(const EncodeRequest& request) { av1PictureDesc.loopRestoration = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1LoopRestoration; av1PictureDesc.globalMotion = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? nullptr : &av1GlobalMotion; nri::VideoReference av1Reference = {m_ReconstructedPicture, 0}; + nri::VideoReference h26References[2] = {}; + nri::VideoH264ReferenceDesc h264References[2] = {}; + nri::VideoH264PictureDesc h264PictureDesc = {}; + nri::VideoH265ReferenceDesc h265References[2] = {}; + uint32_t h26ReferenceNum = 0; + if (h26Codec && frameType != nri::VideoEncodeFrameType::IDR) { + h26References[h26ReferenceNum] = {m_ReconstructedPicture, 0}; + h264References[h26ReferenceNum] = {nri::VideoEncodeFrameType::IDR, 0, 0, 0, 0, 0, 0, 0, 0}; + h265References[h26ReferenceNum] = {0, 0, 0, nri::VideoEncodeFrameType::IDR, 0, 0}; + h26ReferenceNum++; + if (frameType == nri::VideoEncodeFrameType::B) { + h26References[h26ReferenceNum] = {m_AV1PReconstructedPicture, 1}; + h264References[h26ReferenceNum] = {nri::VideoEncodeFrameType::P, 0, 1, 0, 2, 2, 1, 0, 0}; + h265References[h26ReferenceNum] = {1, 2, 0, nri::VideoEncodeFrameType::P, 0, 1}; + h26ReferenceNum++; + } + h264PictureDesc.sequenceParameterSetId = m_CodecParameters.h264Sps.sequenceParameterSetId; + h264PictureDesc.pictureParameterSetId = m_CodecParameters.h264Pps.pictureParameterSetId; + h264PictureDesc.references = h264References; + h264PictureDesc.referenceNum = h26ReferenceNum; + } nri::VideoAV1ReferenceDesc av1References[8] = {}; if (av1PFrame) { const nri::VideoAV1ReferenceName av1ReferenceNames[] = { @@ -418,12 +489,18 @@ bool Encoder::Encode(const EncodeRequest& request) { encodeDesc.bitstreamMetadataSize = ENCODED_SLICE_OFFSET; encodeDesc.pictureDesc = &pictureDesc; encodeDesc.rateControlDesc = &rateControlDesc; - encodeDesc.reconstructedPicture = m_ReconstructedPicture; + encodeDesc.reconstructedPicture = outputSlot == 2 ? m_BReconstructedPicture : (outputSlot == 1 ? m_AV1PReconstructedPicture : m_ReconstructedPicture); + encodeDesc.reconstructedSlot = outputSlot; if (av1PFrame) { encodeDesc.reconstructedPicture = m_AV1PReconstructedPicture; encodeDesc.references = &av1Reference; encodeDesc.referenceNum = 1; encodeDesc.reconstructedSlot = 1; + } else if (h26ReferenceNum) { + encodeDesc.references = h26References; + encodeDesc.referenceNum = h26ReferenceNum; + encodeDesc.h264PictureDesc = m_Config.codec == SampleCodec::H264 ? &h264PictureDesc : nullptr; + encodeDesc.h265ReferenceDescs = m_Config.codec == SampleCodec::H265 ? h265References : nullptr; } encodeDesc.metadata = m_MetadataBuffer; encodeDesc.resolvedMetadata = m_Context.graphicsAPI == nri::GraphicsAPI::VK ? m_ResolvedMetadataReadbackBuffer : m_ResolvedMetadataBuffer; @@ -431,7 +508,8 @@ bool Encoder::Encode(const EncodeRequest& request) { nri::VideoEncodePictureStates srcPictureStates = {}; nri::VideoEncodePictureStates reconstructedPictureStates = {}; - if (video.GetVideoEncodePictureStates(*m_EncodePicture, srcPictureStates) != nri::Result::SUCCESS || video.GetVideoEncodePictureStates(*(av1PFrame ? m_AV1PReconstructedPicture : m_ReconstructedPicture), reconstructedPictureStates) != nri::Result::SUCCESS) { + nri::VideoPicture* reconstructedPicture = outputSlot == 2 ? m_BReconstructedPicture : (outputSlot == 1 ? m_AV1PReconstructedPicture : m_ReconstructedPicture); + if (video.GetVideoEncodePictureStates(*m_EncodePicture, srcPictureStates) != nri::Result::SUCCESS || video.GetVideoEncodePictureStates(*reconstructedPicture, reconstructedPictureStates) != nri::Result::SUCCESS) { m_Status = "Failed to query video encode picture states"; return false; } @@ -443,14 +521,14 @@ bool Encoder::Encode(const EncodeRequest& request) { bufferBarriers[1].buffer = encodeDesc.resolvedMetadata; bufferBarriers[1].after = {nri::AccessBits::VIDEO_ENCODE_WRITE, nri::StageBits::VIDEO_ENCODE}; - nri::TextureBarrierDesc textureBarriers[3] = {}; + nri::TextureBarrierDesc textureBarriers[4] = {}; textureBarriers[0].texture = m_EncodeTexture; textureBarriers[0].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[0].after = srcPictureStates.encodeRead; textureBarriers[0].mipNum = nri::REMAINING; textureBarriers[0].layerNum = nri::REMAINING; textureBarriers[0].planes = nri::PlaneBits::ALL; - textureBarriers[1].texture = av1PFrame ? m_AV1PReconstructedTexture : m_ReconstructedTexture; + textureBarriers[1].texture = outputSlot == 2 ? m_BReconstructedTexture : (outputSlot == 1 ? m_AV1PReconstructedTexture : m_ReconstructedTexture); textureBarriers[1].before = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[1].after = reconstructedPictureStates.encodeWrite; textureBarriers[1].mipNum = nri::REMAINING; @@ -462,12 +540,19 @@ bool Encoder::Encode(const EncodeRequest& request) { textureBarriers[2].mipNum = nri::REMAINING; textureBarriers[2].layerNum = nri::REMAINING; textureBarriers[2].planes = nri::PlaneBits::ALL; + textureBarriers[3] = textureBarriers[2]; + textureBarriers[3].texture = m_AV1PReconstructedTexture; nri::BarrierDesc barrierDesc = {}; barrierDesc.buffers = bufferBarriers; barrierDesc.bufferNum = helper::GetCountOf(bufferBarriers); barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = av1PFrame ? helper::GetCountOf(textureBarriers) : 2; + uint32_t textureBarrierNum = 2; + if (av1PFrame || (h26Codec && frameType == nri::VideoEncodeFrameType::P)) + textureBarrierNum = 3; + else if (h26Codec && frameType == nri::VideoEncodeFrameType::B) + textureBarrierNum = 4; + barrierDesc.textureNum = textureBarrierNum; nri.CmdBarrier(commandBuffer, barrierDesc); video.CmdEncodeVideo(commandBuffer, encodeDesc); bufferBarriers[0].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::StageBits::VIDEO_ENCODE}; @@ -480,14 +565,22 @@ bool Encoder::Encode(const EncodeRequest& request) { textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; textureBarriers[2].after = reconstructedPictureStates.afterEncode; + textureBarriers[3].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[3].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = av1PFrame && reconstructedPictureStates.releaseAfterEncode ? helper::GetCountOf(textureBarriers) : 2; + barrierDesc.textureNum = textureBarrierNum; nri.CmdBarrier(commandBuffer, barrierDesc); })) { m_Status = std::string(GetCodecName(m_Config.codec)) + " encode submission failed"; return false; } + m_PendingFrameType = frameType; + m_PendingFrameIndex = frameIndex; + m_PendingPictureOrderCount = pictureOrderCount; + m_PendingOutputSlot = outputSlot; + m_PendingDisplayFrame = displayFrame; + if (m_MetadataReadbackPending) return true; @@ -638,11 +731,22 @@ bool Encoder::Poll(EncodedFrame& frame) { return false; } + frame.frameType = m_PendingFrameType; + frame.frameIndex = m_PendingFrameIndex; + frame.pictureOrderCount = m_PendingPictureOrderCount; + frame.outputSlot = m_PendingOutputSlot; + frame.isDisplayFrame = m_PendingDisplayFrame; frame.isAv1PFrame = IsEncodingAv1PFrame(); if (m_Config.av1PFrameVisual && m_AV1PFrameStage == 0) m_AV1PFrameStage = 1; else if (frame.isAv1PFrame) m_AV1PFrameStage = 0; + if (m_Config.codec != SampleCodec::AV1) { + if (m_Config.h26FrameMode == VisualFrameMode::P) + m_H26FrameStage = frame.isDisplayFrame ? 0 : 1; + else if (m_Config.h26FrameMode == VisualFrameMode::B) + m_H26FrameStage = frame.isDisplayFrame ? 0 : std::min(m_H26FrameStage + 1, 2u); + } m_Status = std::string(GetCodecName(m_Config.codec)) + " encode complete"; return true; diff --git a/Source/VideoEncodeDecode/Encoder.h b/Source/VideoEncodeDecode/Encoder.h index f48f857..eb11c6e 100644 --- a/Source/VideoEncodeDecode/Encoder.h +++ b/Source/VideoEncodeDecode/Encoder.h @@ -46,6 +46,8 @@ class Encoder { bool SubmitMetadataReadback(); bool BuildAv1DecodeInfo(nri::VideoEncodeFeedback& feedback, nri::VideoAV1EncodeDecodeInfo& av1DecodeInfo); bool IsEncodingAv1PFrame() const; + bool NeedsSecondReconstructedPicture() const; + bool NeedsThirdReconstructedPicture() const; private: VideoContext m_Context = {}; @@ -59,6 +61,7 @@ class Encoder { nri::Texture* m_EncodeTexture = nullptr; nri::Texture* m_ReconstructedTexture = nullptr; nri::Texture* m_AV1PReconstructedTexture = nullptr; + nri::Texture* m_BReconstructedTexture = nullptr; nri::Buffer* m_BitstreamHeaderUploadBuffer = nullptr; nri::Buffer* m_BitstreamBuffer = nullptr; nri::Buffer* m_MetadataBuffer = nullptr; @@ -67,6 +70,7 @@ class Encoder { nri::VideoPicture* m_EncodePicture = nullptr; nri::VideoPicture* m_ReconstructedPicture = nullptr; nri::VideoPicture* m_AV1PReconstructedPicture = nullptr; + nri::VideoPicture* m_BReconstructedPicture = nullptr; nri::CommandAllocator* m_MetadataReadbackCommandAllocator = nullptr; nri::CommandBuffer* m_MetadataReadbackCommandBuffer = nullptr; nri::Fence* m_MetadataReadbackFence = nullptr; @@ -74,6 +78,12 @@ class Encoder { bool m_Ready = false; bool m_MetadataReadbackPending = false; uint32_t m_AV1PFrameStage = 0; + uint32_t m_H26FrameStage = 0; + nri::VideoEncodeFrameType m_PendingFrameType = nri::VideoEncodeFrameType::IDR; + uint32_t m_PendingFrameIndex = 0; + int32_t m_PendingPictureOrderCount = 0; + uint32_t m_PendingOutputSlot = 0; + bool m_PendingDisplayFrame = true; uint64_t m_MetadataReadbackFenceValue = 0; }; diff --git a/Source/VideoEncodeDecode/Shared.h b/Source/VideoEncodeDecode/Shared.h index d36fae5..66f8e03 100644 --- a/Source/VideoEncodeDecode/Shared.h +++ b/Source/VideoEncodeDecode/Shared.h @@ -38,6 +38,12 @@ enum class SampleCodec : uint8_t { AV1, }; +enum class VisualFrameMode : uint8_t { + IDR, + P, + B, +}; + struct Av1SequenceOptions { bool enableCdef = true; bool enableRestoration = true; @@ -76,6 +82,7 @@ struct VideoConfig { uint32_t videoWidth = DEFAULT_VIDEO_WIDTH; uint32_t videoHeight = DEFAULT_VIDEO_HEIGHT; bool av1PFrameVisual = false; + VisualFrameMode h26FrameMode = VisualFrameMode::IDR; }; struct VideoQuality { @@ -117,8 +124,13 @@ struct EncodedFrame { nri::VideoEncodeFeedback feedback = {}; nri::Buffer* bitstreamBuffer = nullptr; nri::VideoAV1EncodeDecodeInfo av1DecodeInfo = {}; + nri::VideoEncodeFrameType frameType = nri::VideoEncodeFrameType::IDR; + uint32_t frameIndex = 0; + int32_t pictureOrderCount = 0; + uint32_t outputSlot = 0; bool hasAv1DecodeInfo = false; bool isAv1PFrame = false; + bool isDisplayFrame = true; }; struct DecodedFrame { From fc5f044693c5a746a3fc0cde1ff529a5a82130cf Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 17:38:56 +0200 Subject: [PATCH 21/28] Improve video encode smoke coverage --- External/NRIFramework | 2 +- Scripts/SmokeVideoEncodeDecode.bat | 231 ++++++++++++++++++++++ Scripts/SmokeVideoEncodeDecode.ps1 | 276 --------------------------- Source/VideoEncodeDecode/Decoder.cpp | 8 +- Source/VideoEncodeDecode/Encoder.cpp | 18 +- 5 files changed, 247 insertions(+), 288 deletions(-) create mode 100644 Scripts/SmokeVideoEncodeDecode.bat delete mode 100644 Scripts/SmokeVideoEncodeDecode.ps1 diff --git a/External/NRIFramework b/External/NRIFramework index 6294ab4..7e1f008 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit 6294ab479a156bce707ccc83825e8c5914fadb9f +Subproject commit 7e1f008213c7946115b0f4e83460be0f286ff696 diff --git a/Scripts/SmokeVideoEncodeDecode.bat b/Scripts/SmokeVideoEncodeDecode.bat new file mode 100644 index 0000000..77c691f --- /dev/null +++ b/Scripts/SmokeVideoEncodeDecode.bat @@ -0,0 +1,231 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion + +set "DURATION_SEC=6" +set "WINDOW_WIDTH=3840" +set "WINDOW_HEIGHT=2160" +set "VIDEO_WIDTH=1280" +set "VIDEO_HEIGHT=720" +set "APIS=D3D12" +set "CASE_FILTER=" +set "NO_BUILD=0" +set "INCLUDE_DEBUG=0" +set "INCLUDE_VULKAN=0" + +call :ParseArgs %* + +set "SCRIPT_DIR=%~dp0" +for %%I in ("%SCRIPT_DIR%..") do set "REPO_ROOT=%%~fI" +set "EXE_PATH=%REPO_ROOT%\_Bin\Release\VideoEncodeDecode.exe" + +call :MakeTimestamp +set "OUT_DIR=%REPO_ROOT%\_Smoke\VideoEncodeDecode_%TIMESTAMP%" +set "LOG_PATH=%OUT_DIR%\VideoEncodeDecodeSmoke.log" +mkdir "%OUT_DIR%" >nul 2>nul + +if "%NO_BUILD%"=="0" ( + call :WriteLog "Building VideoEncodeDecode Release target..." + cmake --build "%REPO_ROOT%\_Build" --config Release --target VideoEncodeDecode > "%OUT_DIR%\build.tmp.log" 2> "%OUT_DIR%\build.tmp.err" + set "BUILD_EXIT=!ERRORLEVEL!" + type "%OUT_DIR%\build.tmp.log" >> "%LOG_PATH%" 2>nul + type "%OUT_DIR%\build.tmp.err" >> "%LOG_PATH%" 2>nul + del "%OUT_DIR%\build.tmp.log" "%OUT_DIR%\build.tmp.err" >nul 2>nul + if not "!BUILD_EXIT!"=="0" ( + call :WriteLog "BUILD FAILED: exit code !BUILD_EXIT!" + exit /b !BUILD_EXIT! + ) +) + +if not exist "%EXE_PATH%" ( + call :WriteLog "Executable not found: %EXE_PATH%" + exit /b 1 +) + +if "%INCLUDE_VULKAN%"=="1" call :AddApi VULKAN + +set "CASE_COUNT=0" +set "PASS_COUNT=0" +set "FAIL_COUNT=0" + +call :WriteLog "VideoEncodeDecode smoke started" +echo %DATE% %TIME% Output directory: %OUT_DIR% +>> "%LOG_PATH%" echo %DATE% %TIME% Output directory: %OUT_DIR% +call :WriteLog "Duration per run: %DURATION_SEC% seconds" +call :WriteLog "Window: %WINDOW_WIDTH%x%WINDOW_HEIGHT%, video: %VIDEO_WIDTH%x%VIDEO_HEIGHT%" +call :WriteLog "APIs: %APIS%" + +for %%A in (%APIS%) do ( + for %%C in (H264 H265) do ( + for %%F in (IDR P B) do ( + call :MaybeRunCase "%%A-%%C-%%F-cqp" %%A %%C h26Frame %%F cqp 0 + call :MaybeRunCase "%%A-%%C-%%F-lossless" %%A %%C h26Frame %%F lossless 1 + ) + ) + + for %%F in (IDR P) do ( + call :MaybeRunCase "%%A-AV1-%%F-cqp" %%A AV1 av1Frame %%F cqp 0 + call :MaybeRunCase "%%A-AV1-%%F-near-lossless" %%A AV1 av1Frame %%F near-lossless 1 + ) +) + +if not "%CASE_FILTER%"=="" ( + call :WriteLog "Case filter: %CASE_FILTER%" + call :WriteLog "Filtered cases: %CASE_COUNT%" + if "%CASE_COUNT%"=="0" ( + call :WriteLog "No cases matched filter" + exit /b 1 + ) +) + +call :WriteLog "" +call :WriteLog "VideoEncodeDecode smoke finished: %PASS_COUNT% passed, %FAIL_COUNT% failed" +call :WriteLog "Log: %LOG_PATH%" + +if not "%FAIL_COUNT%"=="0" exit /b 1 +exit /b 0 + +::======================================================================================== +:ParseArgs +if "%~1"=="" exit /b 0 +set "ARG=%~1" + +if /I "%ARG%"=="-NoBuild" set "NO_BUILD=1" & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--no-build" set "NO_BUILD=1" & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-IncludeVulkan" set "INCLUDE_VULKAN=1" & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--include-vulkan" set "INCLUDE_VULKAN=1" & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-IncludeDebug" set "INCLUDE_DEBUG=1" & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--include-debug" set "INCLUDE_DEBUG=1" & shift /1 & goto :ParseArgs + +if /I "%ARG%"=="-DurationSec" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--duration" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--durationSec" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-WindowWidth" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--width" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--windowWidth" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-WindowHeight" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--height" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--windowHeight" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-VideoWidth" set "VIDEO_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--videoWidth" set "VIDEO_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-VideoHeight" set "VIDEO_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--videoHeight" set "VIDEO_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-Apis" set "APIS=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--apis" set "APIS=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="-CaseFilter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--filter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%ARG%"=="--caseFilter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs + +call :ParseEquals "%ARG%" +shift /1 +goto :ParseArgs + +::======================================================================================== +:ParseEquals +set "ARG=%~1" +for /f "tokens=1,* delims==" %%K in ("%ARG%") do ( + set "KEY=%%K" + set "VALUE=%%L" +) +if /I "%KEY%"=="--duration" set "DURATION_SEC=%VALUE%" +if /I "%KEY%"=="--durationSec" set "DURATION_SEC=%VALUE%" +if /I "%KEY%"=="-DurationSec" set "DURATION_SEC=%VALUE%" +if /I "%KEY%"=="--width" set "WINDOW_WIDTH=%VALUE%" +if /I "%KEY%"=="--windowWidth" set "WINDOW_WIDTH=%VALUE%" +if /I "%KEY%"=="-WindowWidth" set "WINDOW_WIDTH=%VALUE%" +if /I "%KEY%"=="--height" set "WINDOW_HEIGHT=%VALUE%" +if /I "%KEY%"=="--windowHeight" set "WINDOW_HEIGHT=%VALUE%" +if /I "%KEY%"=="-WindowHeight" set "WINDOW_HEIGHT=%VALUE%" +if /I "%KEY%"=="--videoWidth" set "VIDEO_WIDTH=%VALUE%" +if /I "%KEY%"=="-VideoWidth" set "VIDEO_WIDTH=%VALUE%" +if /I "%KEY%"=="--videoHeight" set "VIDEO_HEIGHT=%VALUE%" +if /I "%KEY%"=="-VideoHeight" set "VIDEO_HEIGHT=%VALUE%" +if /I "%KEY%"=="--apis" set "APIS=%VALUE%" +if /I "%KEY%"=="-Apis" set "APIS=%VALUE%" +if /I "%KEY%"=="--filter" set "CASE_FILTER=%VALUE%" +if /I "%KEY%"=="--caseFilter" set "CASE_FILTER=%VALUE%" +if /I "%KEY%"=="-CaseFilter" set "CASE_FILTER=%VALUE%" +exit /b 0 + +::======================================================================================== +:AddApi +echo %APIS% | findstr /I /C:"%~1" >nul +if errorlevel 1 set "APIS=%APIS% %~1" +exit /b 0 + +::======================================================================================== +:MakeTimestamp +set "TS=%DATE%_%TIME%" +set "TS=%TS:/=%" +set "TS=%TS:.=%" +set "TS=%TS::=%" +set "TS=%TS:,=%" +set "TS=%TS: =0%" +set "TIMESTAMP=%TS%" +exit /b 0 + +::======================================================================================== +:WriteLog +echo %DATE% %TIME% %~1 +>> "%LOG_PATH%" echo %DATE% %TIME% %~1 +exit /b 0 + +::======================================================================================== +:MaybeRunCase +set "CASE_NAME=%~1" +if not "%CASE_FILTER%"=="" ( + echo %CASE_NAME% | findstr /I /R /C:"%CASE_FILTER%" >nul + if errorlevel 1 exit /b 0 +) + +call :RunCase %* +exit /b 0 + +::======================================================================================== +:RunCase +set "CASE_NAME=%~1" +set "API=%~2" +set "CODEC=%~3" +set "FRAME_ARG_NAME=%~4" +set "FRAME_ARG_VALUE=%~5" +set "SUFFIX=%~6" +set "LOSSLESS=%~7" + +set /a CASE_COUNT+=1 +set "CASE_OUT=%OUT_DIR%\%CASE_NAME%.stdout.tmp" +set "CASE_ERR=%OUT_DIR%\%CASE_NAME%.stderr.tmp" + +set "CASE_ARGS=--api=%API% --width=%WINDOW_WIDTH% --height=%WINDOW_HEIGHT% --videoWidth=%VIDEO_WIDTH% --videoHeight=%VIDEO_HEIGHT% --codec=%CODEC% --timeLimit=%DURATION_SEC% --alwaysActive --%FRAME_ARG_NAME%=%FRAME_ARG_VALUE%" +if "%LOSSLESS%"=="1" set "CASE_ARGS=%CASE_ARGS% --lossless" +if "%INCLUDE_DEBUG%"=="1" set "CASE_ARGS=%CASE_ARGS% --debugAPI --debugNRI" + +call :WriteLog "" +call :WriteLog "[%CASE_COUNT%] START %CASE_NAME%" +call :WriteLog "Command: ""%EXE_PATH%"" %CASE_ARGS%" + +pushd "%REPO_ROOT%" >nul +"%EXE_PATH%" %CASE_ARGS% > "%CASE_OUT%" 2> "%CASE_ERR%" +set "EXIT_CODE=%ERRORLEVEL%" +popd >nul + +call :WriteLog "ExitCode: %EXIT_CODE%" +>> "%LOG_PATH%" echo ----- stdout: %CASE_NAME% ----- +type "%CASE_OUT%" >> "%LOG_PATH%" 2>nul +>> "%LOG_PATH%" echo ----- stderr: %CASE_NAME% ----- +type "%CASE_ERR%" >> "%LOG_PATH%" 2>nul + +set "CASE_FAILED=0" +if not "%EXIT_CODE%"=="0" set "CASE_FAILED=1" +findstr /R /C:"\" /C:"\" "%CASE_OUT%" "%CASE_ERR%" >nul 2>nul +if not errorlevel 1 set "CASE_FAILED=1" + +del "%CASE_OUT%" "%CASE_ERR%" >nul 2>nul + +if "%CASE_FAILED%"=="1" ( + set /a FAIL_COUNT+=1 + call :WriteLog "FAILED %CASE_NAME%" +) else ( + set /a PASS_COUNT+=1 + call :WriteLog "OK %CASE_NAME%" +) + +exit /b 0 diff --git a/Scripts/SmokeVideoEncodeDecode.ps1 b/Scripts/SmokeVideoEncodeDecode.ps1 deleted file mode 100644 index 0a23b0f..0000000 --- a/Scripts/SmokeVideoEncodeDecode.ps1 +++ /dev/null @@ -1,276 +0,0 @@ -param( - [double] $DurationSec = 6.0, - [int] $WindowWidth = 3840, - [int] $WindowHeight = 2160, - [int] $VideoWidth = 1280, - [int] $VideoHeight = 720, - [string[]] $Apis = @("D3D12"), - [string] $CaseFilter = "", - [int] $ScreenshotDelayMs = 2500, - [switch] $NoBuild, - [switch] $NoScreenshots, - [switch] $IncludeVulkan, - [switch] $IncludeDebug -) - -$ErrorActionPreference = "Stop" - -if ($DurationSec -lt 2.0) { - Write-Warning "DurationSec is below 2 seconds; this is only useful for script debugging, not smoke coverage." -} - -$RepoRoot = Split-Path -Parent $PSScriptRoot -$ExePath = Join-Path $RepoRoot "_Bin\Release\VideoEncodeDecode.exe" -$Timestamp = Get-Date -Format "yyyyMMdd_HHmmss" -$OutDir = Join-Path $RepoRoot "_Smoke\VideoEncodeDecode_$Timestamp" -$LogPath = Join-Path $OutDir "VideoEncodeDecodeSmoke.log" - -if ($IncludeVulkan -and -not ($Apis -contains "VULKAN")) { - $Apis += "VULKAN" -} - -New-Item -ItemType Directory -Force -Path $OutDir | Out-Null - -function Write-Log { - param([string] $Message) - $line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff"), $Message - $line | Tee-Object -FilePath $LogPath -Append -} - -if (-not $NoBuild) { - Write-Log "Building VideoEncodeDecode Release target..." - $buildLog = Join-Path $OutDir "build.tmp.log" - $build = Start-Process -FilePath "cmake" -ArgumentList @("--build", "_Build", "--config", "Release", "--target", "VideoEncodeDecode") -WorkingDirectory $RepoRoot -NoNewWindow -Wait -PassThru -RedirectStandardOutput $buildLog -RedirectStandardError "$buildLog.err" - Get-Content $buildLog, "$buildLog.err" -ErrorAction SilentlyContinue | Add-Content $LogPath - Remove-Item $buildLog, "$buildLog.err" -ErrorAction SilentlyContinue - if ($build.ExitCode -ne 0) { - Write-Log "BUILD FAILED: exit code $($build.ExitCode)" - exit $build.ExitCode - } -} - -if (-not (Test-Path $ExePath)) { - Write-Log "Executable not found: $ExePath" - exit 1 -} - -Add-Type @" -using System; -using System.Runtime.InteropServices; - -public static class Win32WindowCapture { - [StructLayout(LayoutKind.Sequential)] - public struct RECT { - public int Left; - public int Top; - public int Right; - public int Bottom; - } - - [DllImport("user32.dll")] - public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); - - [DllImport("user32.dll")] - public static extern bool SetForegroundWindow(IntPtr hWnd); -} -"@ - -if (-not $NoScreenshots) { - Add-Type -AssemblyName System.Drawing -} - -function Wait-MainWindow { - param( - [System.Diagnostics.Process] $Process, - [int] $TimeoutMs = 8000 - ) - - $deadline = [Environment]::TickCount + $TimeoutMs - while (-not $Process.HasExited -and [Environment]::TickCount -lt $deadline) { - $Process.Refresh() - if ($Process.MainWindowHandle -ne [IntPtr]::Zero) { - return $Process.MainWindowHandle - } - Start-Sleep -Milliseconds 100 - } - - return [IntPtr]::Zero -} - -function Capture-Window { - param( - [IntPtr] $Handle, - [string] $Path - ) - - if ($NoScreenshots -or $Handle -eq [IntPtr]::Zero) { - return $false - } - - Start-Sleep -Milliseconds $ScreenshotDelayMs - - $rect = New-Object Win32WindowCapture+RECT - if (-not [Win32WindowCapture]::GetWindowRect($Handle, [ref] $rect)) { - return $false - } - - $width = $rect.Right - $rect.Left - $height = $rect.Bottom - $rect.Top - if ($width -le 0 -or $height -le 0) { - return $false - } - - $bitmap = New-Object System.Drawing.Bitmap $width, $height - $graphics = [System.Drawing.Graphics]::FromImage($bitmap) - try { - $graphics.CopyFromScreen($rect.Left, $rect.Top, 0, 0, $bitmap.Size) - $bitmap.Save($Path, [System.Drawing.Imaging.ImageFormat]::Png) - } finally { - $graphics.Dispose() - $bitmap.Dispose() - } - - return $true -} - -function New-Case { - param( - [string] $Name, - [string] $Api, - [string] $Codec, - [string] $FrameArgName, - [string] $FrameArgValue, - [bool] $Lossless - ) - - $args = @( - "--api=$Api", - "--width=$WindowWidth", - "--height=$WindowHeight", - "--videoWidth=$VideoWidth", - "--videoHeight=$VideoHeight", - "--codec=$Codec", - "--timeLimit=$DurationSec", - "--alwaysActive" - ) - - if ($FrameArgName) { - $args += "--$FrameArgName=$FrameArgValue" - } - if ($Lossless) { - $args += "--lossless" - } - if ($IncludeDebug) { - $args += "--debugAPI" - $args += "--debugNRI" - } - - [pscustomobject]@{ - Name = $Name - Args = $args - } -} - -$cases = New-Object System.Collections.Generic.List[object] -foreach ($api in $Apis) { - foreach ($codec in @("H264", "H265")) { - $h26Frames = @("IDR", "P", "B") - if ($api -eq "VULKAN" -and $codec -eq "H265") { - $h26Frames = @("IDR", "P") - } - foreach ($frame in $h26Frames) { - foreach ($lossless in @($false, $true)) { - $suffix = if ($lossless) { "lossless" } else { "cqp" } - $cases.Add((New-Case -Name "$api-$codec-$frame-$suffix" -Api $api -Codec $codec -FrameArgName "h26Frame" -FrameArgValue $frame -Lossless $lossless)) - } - } - } - - foreach ($frame in @("IDR", "P")) { - foreach ($lossless in @($false, $true)) { - $suffix = if ($lossless) { "near-lossless" } else { "cqp" } - $cases.Add((New-Case -Name "$api-AV1-$frame-$suffix" -Api $api -Codec "AV1" -FrameArgName "av1Frame" -FrameArgValue $frame -Lossless $lossless)) - } - } -} - -Write-Log "VideoEncodeDecode smoke started" -Write-Log "Output directory: $OutDir" -Write-Log "Duration per run: $DurationSec seconds" -Write-Log "Window: ${WindowWidth}x${WindowHeight}, video: ${VideoWidth}x${VideoHeight}" -Write-Log "APIs: $($Apis -join ', ')" -Write-Log "Cases: $($cases.Count)" - -if ($CaseFilter) { - $cases = @($cases | Where-Object { $_.Name -match $CaseFilter }) - Write-Log "Case filter: $CaseFilter" - Write-Log "Filtered cases: $($cases.Count)" - if ($cases.Count -eq 0) { - Write-Log "No cases matched filter" - exit 1 - } -} - -$failures = 0 -$index = 0 -foreach ($case in $cases) { - $index++ - $caseOut = Join-Path $OutDir "$($case.Name).stdout.tmp" - $caseErr = Join-Path $OutDir "$($case.Name).stderr.tmp" - $screenshot = Join-Path $OutDir "$($case.Name).png" - - Write-Log "" - Write-Log "[$index/$($cases.Count)] START $($case.Name)" - Write-Log "Command: `"$ExePath`" $($case.Args -join ' ')" - - $process = Start-Process -FilePath $ExePath -ArgumentList $case.Args -WorkingDirectory $RepoRoot -PassThru -RedirectStandardOutput $caseOut -RedirectStandardError $caseErr - $handle = Wait-MainWindow -Process $process - - if ($handle -ne [IntPtr]::Zero) { - Write-Log "Window handle: $handle" - [Win32WindowCapture]::SetForegroundWindow($handle) | Out-Null - if (Capture-Window -Handle $handle -Path $screenshot) { - Write-Log "Screenshot: $screenshot" - } elseif (-not $NoScreenshots) { - Write-Log "Screenshot capture failed" - } - } else { - Write-Log "Window handle not found before timeout" - } - - $maxWaitSec = [Math]::Ceiling($DurationSec + 12) - $timedOut = $false - if (-not $process.WaitForExit($maxWaitSec * 1000)) { - Write-Log "TIMEOUT: killing process after $maxWaitSec seconds" - $timedOut = $true - $process.Kill() - $process.WaitForExit() - } - $process.Refresh() - - $exitCode = $process.ExitCode - if ($null -eq $exitCode -or "$exitCode" -eq "") { - $exitCode = 0 - } - Write-Log "ExitCode: $exitCode" - Add-Content -Path $LogPath -Value "----- stdout: $($case.Name) -----" - Get-Content $caseOut -ErrorAction SilentlyContinue | Add-Content $LogPath - Add-Content -Path $LogPath -Value "----- stderr: $($case.Name) -----" - Get-Content $caseErr -ErrorAction SilentlyContinue | Add-Content $LogPath - $caseText = (Get-Content $caseOut, $caseErr -ErrorAction SilentlyContinue) -join "`n" - Remove-Item $caseOut, $caseErr -ErrorAction SilentlyContinue - - if ($timedOut -or $exitCode -ne 0 -or $caseText -match "(?m)\b(ERROR|FAILED)\b") { - $failures++ - Write-Log "FAILED $($case.Name)" - } else { - Write-Log "OK $($case.Name)" - } -} - -Write-Log "" -Write-Log "VideoEncodeDecode smoke finished: $($cases.Count - $failures) passed, $failures failed" -Write-Log "Log: $LogPath" -if ($failures -ne 0) { - exit 1 -} diff --git a/Source/VideoEncodeDecode/Decoder.cpp b/Source/VideoEncodeDecode/Decoder.cpp index 97e736b..8486327 100644 --- a/Source/VideoEncodeDecode/Decoder.cpp +++ b/Source/VideoEncodeDecode/Decoder.cpp @@ -326,12 +326,12 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { uint32_t h26DecodeReferenceNum = 0; if (h26Codec && frame.frameType != nri::VideoEncodeFrameType::IDR) { h26DecodeReferences[h26DecodeReferenceNum] = {m_DecodePicture, 0}; - h264DecodeReferences[h26DecodeReferenceNum] = {nri::VideoH264DecodeReferenceBits::NONE, 0, 0, 0, 0}; + h264DecodeReferences[h26DecodeReferenceNum] = {nri::VideoH264DecodeReferenceBits::TOP_FIELD | nri::VideoH264DecodeReferenceBits::BOTTOM_FIELD, 0, 0, 0, 0}; h265DecodeReferences[h26DecodeReferenceNum] = {0, 0, 0, nri::VideoEncodeFrameType::IDR, 0, 0}; h26DecodeReferenceNum++; if (frame.frameType == nri::VideoEncodeFrameType::B) { h26DecodeReferences[h26DecodeReferenceNum] = {m_AV1PDecodePicture, 1}; - h264DecodeReferences[h26DecodeReferenceNum] = {nri::VideoH264DecodeReferenceBits::NONE, 2, 1, 2, 2}; + h264DecodeReferences[h26DecodeReferenceNum] = {nri::VideoH264DecodeReferenceBits::TOP_FIELD | nri::VideoH264DecodeReferenceBits::BOTTOM_FIELD, 2, 1, 2, 2}; h265DecodeReferences[h26DecodeReferenceNum] = {1, 2, 0, nri::VideoEncodeFrameType::P, 0, 1}; h26DecodeReferenceNum++; } @@ -499,7 +499,7 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { nri::TextureDataLayoutDesc lumaLayout = {}; lumaLayout.rowPitch = request.nv12Layout->yRowPitchBytes; lumaLayout.slicePitch = request.nv12Layout->ySlicePitchBytes; - nri.CmdReadbackTextureToBuffer(commandBuffer, *request.nv12ReadbackBuffer, lumaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), lumaRegion); + nri.CmdReadbackTextureToBuffer(commandBuffer, *request.nv12ReadbackBuffer, lumaLayout, *dstTexture, lumaRegion); nri::TextureRegionDesc chromaRegion = {}; chromaRegion.width = (nri::Dim_t)m_Size.videoWidth; @@ -511,7 +511,7 @@ bool Decoder::Decode(const DecodeRequest& request, DecodedFrame& decodedFrame) { chromaLayout.offset = request.nv12Layout->uvOffsetBytes; chromaLayout.rowPitch = request.nv12Layout->uvRowPitchBytes; chromaLayout.slicePitch = request.nv12Layout->uvSlicePitchBytes; - nri.CmdReadbackTextureToBuffer(commandBuffer, *request.nv12ReadbackBuffer, chromaLayout, *(av1PFrame ? m_AV1PDecodeTexture : m_DecodeTexture), chromaRegion); + nri.CmdReadbackTextureToBuffer(commandBuffer, *request.nv12ReadbackBuffer, chromaLayout, *dstTexture, chromaRegion); nv12BufferBarrier.before = {nri::AccessBits::COPY_DESTINATION, nri::StageBits::COPY}; nv12BufferBarrier.after = {nri::AccessBits::NONE, nri::StageBits::NONE}; diff --git a/Source/VideoEncodeDecode/Encoder.cpp b/Source/VideoEncodeDecode/Encoder.cpp index e2625e2..b2cd86e 100644 --- a/Source/VideoEncodeDecode/Encoder.cpp +++ b/Source/VideoEncodeDecode/Encoder.cpp @@ -561,14 +561,18 @@ bool Encoder::Encode(const EncodeRequest& request) { bufferBarriers[1].after = {}; textureBarriers[0].before = srcPictureStates.encodeRead; textureBarriers[0].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[1].before = reconstructedPictureStates.encodeWrite; - textureBarriers[1].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; - textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[2].after = reconstructedPictureStates.afterEncode; - textureBarriers[3].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; - textureBarriers[3].after = {nri::AccessBits::NONE, nri::Layout::GENERAL, nri::StageBits::NONE}; + uint32_t postEncodeTextureBarrierNum = 1; + if (reconstructedPictureStates.releaseAfterEncode) { + textureBarriers[1].before = reconstructedPictureStates.encodeWrite; + textureBarriers[1].after = reconstructedPictureStates.afterEncode; + textureBarriers[2].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[2].after = reconstructedPictureStates.afterEncode; + textureBarriers[3].before = {nri::AccessBits::VIDEO_ENCODE_READ, nri::Layout::VIDEO_ENCODE_DPB, nri::StageBits::VIDEO_ENCODE}; + textureBarriers[3].after = reconstructedPictureStates.afterEncode; + postEncodeTextureBarrierNum = textureBarrierNum; + } barrierDesc.textures = textureBarriers; - barrierDesc.textureNum = textureBarrierNum; + barrierDesc.textureNum = postEncodeTextureBarrierNum; nri.CmdBarrier(commandBuffer, barrierDesc); })) { m_Status = std::string(GetCodecName(m_Config.codec)) + " encode submission failed"; From eb6895a0c2860d99a4bbfbd6992ea8c8b58af6db Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 17:44:30 +0200 Subject: [PATCH 22/28] Update NRIFramework video TODO --- External/NRIFramework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/External/NRIFramework b/External/NRIFramework index 7e1f008..4b0810c 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit 7e1f008213c7946115b0f4e83460be0f286ff696 +Subproject commit 4b0810cfc82ce358d8c49f46b66aee3dadb4e4b1 From b87fef301f8c0f4409a9c8b59be66389f789b310 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 17:53:58 +0200 Subject: [PATCH 23/28] Update NRIFramework video validation --- External/NRIFramework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/External/NRIFramework b/External/NRIFramework index 4b0810c..f9153ad 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit 4b0810cfc82ce358d8c49f46b66aee3dadb4e4b1 +Subproject commit f9153ad6fe571cf5826d10eb31f60175e0c8b480 From ecb973821b527e02ebd831ba46285dc8e4df97f9 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 18:21:25 +0200 Subject: [PATCH 24/28] Require video roundtrip in smoke --- External/NRIFramework | 2 +- Scripts/SmokeVideoEncodeDecode.bat | 116 ++++++++++++++--------------- Source/VideoEncodeDecode.cpp | 21 +++++- 3 files changed, 79 insertions(+), 60 deletions(-) diff --git a/External/NRIFramework b/External/NRIFramework index f9153ad..909aa19 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit f9153ad6fe571cf5826d10eb31f60175e0c8b480 +Subproject commit 909aa1936a1e7f235778587b7c46ffc9fdaee6cc diff --git a/Scripts/SmokeVideoEncodeDecode.bat b/Scripts/SmokeVideoEncodeDecode.bat index 77c691f..c3aaf7c 100644 --- a/Scripts/SmokeVideoEncodeDecode.bat +++ b/Scripts/SmokeVideoEncodeDecode.bat @@ -18,7 +18,13 @@ set "SCRIPT_DIR=%~dp0" for %%I in ("%SCRIPT_DIR%..") do set "REPO_ROOT=%%~fI" set "EXE_PATH=%REPO_ROOT%\_Bin\Release\VideoEncodeDecode.exe" -call :MakeTimestamp +set "TS=%DATE%_%TIME%" +set "TS=%TS:/=%" +set "TS=%TS:.=%" +set "TS=%TS::=%" +set "TS=%TS:,=%" +set "TS=%TS: =0%" +set "TIMESTAMP=%TS%" set "OUT_DIR=%REPO_ROOT%\_Smoke\VideoEncodeDecode_%TIMESTAMP%" set "LOG_PATH=%OUT_DIR%\VideoEncodeDecodeSmoke.log" mkdir "%OUT_DIR%" >nul 2>nul @@ -52,7 +58,8 @@ echo %DATE% %TIME% Output directory: %OUT_DIR% >> "%LOG_PATH%" echo %DATE% %TIME% Output directory: %OUT_DIR% call :WriteLog "Duration per run: %DURATION_SEC% seconds" call :WriteLog "Window: %WINDOW_WIDTH%x%WINDOW_HEIGHT%, video: %VIDEO_WIDTH%x%VIDEO_HEIGHT%" -call :WriteLog "APIs: %APIS%" +echo %DATE% %TIME% APIs: %APIS% +>> "%LOG_PATH%" echo %DATE% %TIME% APIs: %APIS% for %%A in (%APIS%) do ( for %%C in (H264 H265) do ( @@ -87,33 +94,35 @@ exit /b 0 ::======================================================================================== :ParseArgs if "%~1"=="" exit /b 0 +if /I "%~1"=="--videoWidth" set "VIDEO_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "%~1"=="--videoHeight" set "VIDEO_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs set "ARG=%~1" -if /I "%ARG%"=="-NoBuild" set "NO_BUILD=1" & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--no-build" set "NO_BUILD=1" & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-IncludeVulkan" set "INCLUDE_VULKAN=1" & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--include-vulkan" set "INCLUDE_VULKAN=1" & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-IncludeDebug" set "INCLUDE_DEBUG=1" & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--include-debug" set "INCLUDE_DEBUG=1" & shift /1 & goto :ParseArgs - -if /I "%ARG%"=="-DurationSec" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--duration" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--durationSec" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-WindowWidth" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--width" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--windowWidth" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-WindowHeight" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--height" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--windowHeight" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-VideoWidth" set "VIDEO_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--videoWidth" set "VIDEO_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-VideoHeight" set "VIDEO_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--videoHeight" set "VIDEO_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-Apis" set "APIS=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--apis" set "APIS=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="-CaseFilter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--filter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs -if /I "%ARG%"=="--caseFilter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-NoBuild" set "NO_BUILD=1" & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--no-build" set "NO_BUILD=1" & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-IncludeVulkan" set "INCLUDE_VULKAN=1" & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--include-vulkan" set "INCLUDE_VULKAN=1" & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-IncludeDebug" set "INCLUDE_DEBUG=1" & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--include-debug" set "INCLUDE_DEBUG=1" & shift /1 & goto :ParseArgs + +if /I "!ARG!"=="-DurationSec" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--duration" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--durationSec" set "DURATION_SEC=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-WindowWidth" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--width" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--windowWidth" set "WINDOW_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-WindowHeight" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--height" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--windowHeight" set "WINDOW_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-VideoWidth" set "VIDEO_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--videoWidth" set "VIDEO_WIDTH=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-VideoHeight" set "VIDEO_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--videoHeight" set "VIDEO_HEIGHT=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-Apis" set "APIS=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--apis" set "APIS=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="-CaseFilter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--filter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs +if /I "!ARG!"=="--caseFilter" set "CASE_FILTER=%~2" & shift /1 & shift /1 & goto :ParseArgs call :ParseEquals "%ARG%" shift /1 @@ -122,28 +131,28 @@ goto :ParseArgs ::======================================================================================== :ParseEquals set "ARG=%~1" -for /f "tokens=1,* delims==" %%K in ("%ARG%") do ( +for /f "tokens=1,* delims==" %%K in ("!ARG!") do ( set "KEY=%%K" set "VALUE=%%L" ) -if /I "%KEY%"=="--duration" set "DURATION_SEC=%VALUE%" -if /I "%KEY%"=="--durationSec" set "DURATION_SEC=%VALUE%" -if /I "%KEY%"=="-DurationSec" set "DURATION_SEC=%VALUE%" -if /I "%KEY%"=="--width" set "WINDOW_WIDTH=%VALUE%" -if /I "%KEY%"=="--windowWidth" set "WINDOW_WIDTH=%VALUE%" -if /I "%KEY%"=="-WindowWidth" set "WINDOW_WIDTH=%VALUE%" -if /I "%KEY%"=="--height" set "WINDOW_HEIGHT=%VALUE%" -if /I "%KEY%"=="--windowHeight" set "WINDOW_HEIGHT=%VALUE%" -if /I "%KEY%"=="-WindowHeight" set "WINDOW_HEIGHT=%VALUE%" -if /I "%KEY%"=="--videoWidth" set "VIDEO_WIDTH=%VALUE%" -if /I "%KEY%"=="-VideoWidth" set "VIDEO_WIDTH=%VALUE%" -if /I "%KEY%"=="--videoHeight" set "VIDEO_HEIGHT=%VALUE%" -if /I "%KEY%"=="-VideoHeight" set "VIDEO_HEIGHT=%VALUE%" -if /I "%KEY%"=="--apis" set "APIS=%VALUE%" -if /I "%KEY%"=="-Apis" set "APIS=%VALUE%" -if /I "%KEY%"=="--filter" set "CASE_FILTER=%VALUE%" -if /I "%KEY%"=="--caseFilter" set "CASE_FILTER=%VALUE%" -if /I "%KEY%"=="-CaseFilter" set "CASE_FILTER=%VALUE%" +if /I "!KEY!"=="--duration" set "DURATION_SEC=!VALUE!" +if /I "!KEY!"=="--durationSec" set "DURATION_SEC=!VALUE!" +if /I "!KEY!"=="-DurationSec" set "DURATION_SEC=!VALUE!" +if /I "!KEY!"=="--width" set "WINDOW_WIDTH=!VALUE!" +if /I "!KEY!"=="--windowWidth" set "WINDOW_WIDTH=!VALUE!" +if /I "!KEY!"=="-WindowWidth" set "WINDOW_WIDTH=!VALUE!" +if /I "!KEY!"=="--height" set "WINDOW_HEIGHT=!VALUE!" +if /I "!KEY!"=="--windowHeight" set "WINDOW_HEIGHT=!VALUE!" +if /I "!KEY!"=="-WindowHeight" set "WINDOW_HEIGHT=!VALUE!" +if /I "!KEY!"=="--videoWidth" set "VIDEO_WIDTH=!VALUE!" +if /I "!KEY!"=="-VideoWidth" set "VIDEO_WIDTH=!VALUE!" +if /I "!KEY!"=="--videoHeight" set "VIDEO_HEIGHT=!VALUE!" +if /I "!KEY!"=="-VideoHeight" set "VIDEO_HEIGHT=!VALUE!" +if /I "!KEY!"=="--apis" set "APIS=!VALUE!" +if /I "!KEY!"=="-Apis" set "APIS=!VALUE!" +if /I "!KEY!"=="--filter" set "CASE_FILTER=!VALUE!" +if /I "!KEY!"=="--caseFilter" set "CASE_FILTER=!VALUE!" +if /I "!KEY!"=="-CaseFilter" set "CASE_FILTER=!VALUE!" exit /b 0 ::======================================================================================== @@ -152,17 +161,6 @@ echo %APIS% | findstr /I /C:"%~1" >nul if errorlevel 1 set "APIS=%APIS% %~1" exit /b 0 -::======================================================================================== -:MakeTimestamp -set "TS=%DATE%_%TIME%" -set "TS=%TS:/=%" -set "TS=%TS:.=%" -set "TS=%TS::=%" -set "TS=%TS:,=%" -set "TS=%TS: =0%" -set "TIMESTAMP=%TS%" -exit /b 0 - ::======================================================================================== :WriteLog echo %DATE% %TIME% %~1 @@ -194,7 +192,7 @@ set /a CASE_COUNT+=1 set "CASE_OUT=%OUT_DIR%\%CASE_NAME%.stdout.tmp" set "CASE_ERR=%OUT_DIR%\%CASE_NAME%.stderr.tmp" -set "CASE_ARGS=--api=%API% --width=%WINDOW_WIDTH% --height=%WINDOW_HEIGHT% --videoWidth=%VIDEO_WIDTH% --videoHeight=%VIDEO_HEIGHT% --codec=%CODEC% --timeLimit=%DURATION_SEC% --alwaysActive --%FRAME_ARG_NAME%=%FRAME_ARG_VALUE%" +set "CASE_ARGS=--api=%API% --width=%WINDOW_WIDTH% --height=%WINDOW_HEIGHT% --videoWidth=%VIDEO_WIDTH% --videoHeight=%VIDEO_HEIGHT% --codec=%CODEC% --timeLimit=%DURATION_SEC% --alwaysActive --requireVideoRoundTrip --%FRAME_ARG_NAME%=%FRAME_ARG_VALUE%" if "%LOSSLESS%"=="1" set "CASE_ARGS=%CASE_ARGS% --lossless" if "%INCLUDE_DEBUG%"=="1" set "CASE_ARGS=%CASE_ARGS% --debugAPI --debugNRI" @@ -217,6 +215,8 @@ set "CASE_FAILED=0" if not "%EXIT_CODE%"=="0" set "CASE_FAILED=1" findstr /R /C:"\" /C:"\" "%CASE_OUT%" "%CASE_ERR%" >nul 2>nul if not errorlevel 1 set "CASE_FAILED=1" +findstr /C:"VIDEO_ROUND_TRIP_OK" "%CASE_OUT%" >nul 2>nul +if errorlevel 1 set "CASE_FAILED=1" del "%CASE_OUT%" "%CASE_ERR%" >nul 2>nul diff --git a/Source/VideoEncodeDecode.cpp b/Source/VideoEncodeDecode.cpp index 3989766..3409c9d 100644 --- a/Source/VideoEncodeDecode.cpp +++ b/Source/VideoEncodeDecode.cpp @@ -224,6 +224,7 @@ class Sample : public SampleBase { void LatencySleep(uint32_t frameIndex) override; void PrepareFrame(uint32_t frameIndex) override; void RenderFrame(uint32_t frameIndex) override; + bool AppShouldClose() override; private: bool InitializeGraphics(nri::GraphicsAPI graphicsAPI); @@ -306,6 +307,9 @@ class Sample : public SampleBase { bool m_AV1PFrameVisual = false; bool m_ShowDifference = false; bool m_HasPendingEncodedPatternConstants = false; + bool m_RequireVideoRoundTrip = false; + bool m_SmokeRoundTripComplete = false; + bool m_SmokeRoundTripFailed = false; uint32_t m_PatternFrameIndex = 0; }; @@ -374,6 +378,7 @@ void Sample::InitCmdLine(cmdline::parser& cmdLine) { cmdLine.add("qpB", 0, "CQP quantizer for B frames", false, m_QpB); cmdLine.add("av1BaseQIndex", 0, "AV1 base quantizer index", false, m_AV1BaseQIndex); cmdLine.add("lossless", 0, "force zero quantizers for lossless-capable codec modes"); + cmdLine.add("requireVideoRoundTrip", 0, "exit smoke runs only after a real encode/decode round trip"); } void Sample::ReadCmdLine(cmdline::parser& cmdLine) { @@ -391,6 +396,7 @@ void Sample::ReadCmdLine(cmdline::parser& cmdLine) { m_QpB = cmdLine.get("qpB"); m_AV1BaseQIndex = cmdLine.get("av1BaseQIndex"); m_Lossless = cmdLine.exist("lossless"); + m_RequireVideoRoundTrip = cmdLine.exist("requireVideoRoundTrip"); m_Codec = m_CodecArg == "H265" ? SampleCodec::H265 : (m_CodecArg == "AV1" ? SampleCodec::AV1 : SampleCodec::H264); m_H26FrameMode = m_H26FrameArg == "B" ? video_sample::VisualFrameMode::B : (m_H26FrameArg == "P" ? video_sample::VisualFrameMode::P : video_sample::VisualFrameMode::IDR); m_AV1PFrameVisual = m_Codec == SampleCodec::AV1 && m_AV1FrameArg == "P"; @@ -988,6 +994,10 @@ bool Sample::TryDecodePendingMetadata(float timeSec) { char message[128] = {}; std::snprintf(message, sizeof(message), "%s encode/decode round trip complete, encoded %llu bytes", GetCodecName(m_Codec), (unsigned long long)encodedFrame.feedback.encodedBitstreamWrittenBytes); m_VideoStatus = message; + if (!m_SmokeRoundTripComplete) { + std::printf("VIDEO_ROUND_TRIP_OK: %s\n", message); + m_SmokeRoundTripComplete = true; + } return true; } @@ -1028,6 +1038,11 @@ void Sample::PrepareFrame(uint32_t) { InitializeGeneratedFrames((float)timeSec); + if (m_RequireVideoRoundTrip && !m_SmokeRoundTripComplete && !canRunRoundTrip && !m_SmokeRoundTripFailed) { + std::fprintf(stderr, "VIDEO_ROUND_TRIP_FAILED: %s\n", m_VideoStatus.c_str()); + m_SmokeRoundTripFailed = true; + } + if (canRunRoundTrip && timeSec - m_LastRoundTripTimeSec >= ROUND_TRIP_INTERVAL_SEC) { if (TryRunRoundTrip((float)timeSec)) m_LastRoundTripTimeSec = timeSec; @@ -1037,7 +1052,7 @@ void Sample::PrepareFrame(uint32_t) { { ImGui::SetNextWindowPos({20.0f, 20.0f}, ImGuiCond_Once); ImGui::SetNextWindowSize({900.0f, 520.0f}, ImGuiCond_Once); - ImGui::Begin("NRI Video Encode / Decode"); + ImGui::Begin("NRI Video Encode / Decode"); { const uint32_t minCodecQp = m_Codec == SampleCodec::AV1 ? 1 : 0; const uint32_t effectiveQpI = m_VideoQuality.lossless ? minCodecQp : std::max(m_VideoQuality.qpI, minCodecQp); @@ -1108,6 +1123,10 @@ void Sample::PrepareFrame(uint32_t) { ImGui::Render(); } +bool Sample::AppShouldClose() { + return m_RequireVideoRoundTrip && (m_SmokeRoundTripComplete || m_SmokeRoundTripFailed); +} + void Sample::RenderFrame(uint32_t frameIndex) { uint32_t queuedFrameIndex = frameIndex % (uint32_t)m_QueuedFrames.size(); const QueuedFrame& queuedFrame = m_QueuedFrames[queuedFrameIndex]; From 087ffc06cb204a6c6afef5cd10c3d255efa32e18 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 19:29:28 +0200 Subject: [PATCH 25/28] Update NRIFramework Vulkan video cleanup --- External/NRIFramework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/External/NRIFramework b/External/NRIFramework index 909aa19..aa02b38 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit 909aa1936a1e7f235778587b7c46ffc9fdaee6cc +Subproject commit aa02b383ed2be8b8f4a75b678c56b1e9375b5d00 From 50dbccf55ed87a0d2339431de34f23a5a6134c6e Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 20:34:22 +0200 Subject: [PATCH 26/28] Fix video encode smoke issues --- Source/VideoEncodeDecode/Shared.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/VideoEncodeDecode/Shared.h b/Source/VideoEncodeDecode/Shared.h index 66f8e03..9c85a5c 100644 --- a/Source/VideoEncodeDecode/Shared.h +++ b/Source/VideoEncodeDecode/Shared.h @@ -263,7 +263,7 @@ inline CodecParameters MakeCodecParameters(nri::GraphicsAPI graphicsAPI, uint32_ params.h265Sps.profileTierLevel = params.h265Vps.profileTierLevel; params.h265Sps.decPicBufMgr = params.h265Vps.decPicBufMgr; - params.h265Pps.flags = nri::VideoH265PictureParameterSetBits::CABAC_INIT_PRESENT | nri::VideoH265PictureParameterSetBits::TRANSFORM_SKIP_ENABLED | nri::VideoH265PictureParameterSetBits::CU_QP_DELTA_ENABLED | nri::VideoH265PictureParameterSetBits::SLICE_CHROMA_QP_OFFSETS_PRESENT | nri::VideoH265PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT; + params.h265Pps.flags = nri::VideoH265PictureParameterSetBits::CABAC_INIT_PRESENT | nri::VideoH265PictureParameterSetBits::TRANSFORM_SKIP_ENABLED | nri::VideoH265PictureParameterSetBits::CU_QP_DELTA_ENABLED | nri::VideoH265PictureParameterSetBits::SLICE_CHROMA_QP_OFFSETS_PRESENT | nri::VideoH265PictureParameterSetBits::DEBLOCKING_FILTER_CONTROL_PRESENT | nri::VideoH265PictureParameterSetBits::LISTS_MODIFICATION_PRESENT; params.h265Pps.pictureParameterSetId = 0; params.h265Pps.sequenceParameterSetId = params.h265Sps.sequenceParameterSetId; params.h265Pps.videoParameterSetId = params.h265Vps.videoParameterSetId; From dfecdb17a0bd1ccf7df99a3539c95ce28b9a8090 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Sun, 21 Jun 2026 20:35:00 +0200 Subject: [PATCH 27/28] Update NRIFramework video fixes --- External/NRIFramework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/External/NRIFramework b/External/NRIFramework index aa02b38..a766552 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit aa02b383ed2be8b8f4a75b678c56b1e9375b5d00 +Subproject commit a7665523c161abe8926a93f2479506c97fed822a From 097b51b581fd92006fff40e831b5b73d8526d551 Mon Sep 17 00:00:00 2001 From: Bjorn Schobben Date: Tue, 23 Jun 2026 16:26:00 +0200 Subject: [PATCH 28/28] Update NRIFramework video queues --- External/NRIFramework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/External/NRIFramework b/External/NRIFramework index a766552..ccbe7eb 160000 --- a/External/NRIFramework +++ b/External/NRIFramework @@ -1 +1 @@ -Subproject commit a7665523c161abe8926a93f2479506c97fed822a +Subproject commit ccbe7eb6e81d65ae56928595dc86747b1697cba4