Fix vulkan threading warnings

This is a temporary fix. See #175 for more details. In short it creates a separate vulkan queue for asset transfers and all the background thread code uses it. No locks are used. This is fine since the code currently limits itself to having one background thread for loading/importing maps running at a given moment. We'll want the ability to have multiple threads loading maps at the same time for user convenience and for SP map loading. So in the near future a better approach should be implemented. For now this works so we can get the first version of the C# rewrite up and running quickly.

Author: Moneyl

Nanoforge/Gui/ViewModels/Documents/RendererTestDocumentViewModel.cs

@@ -169,7 +169,7 @@ private MeshInstanceData LoadRfgStaticMesh(string cpuFilePath)
     private Mesh LoadRfgStaticMeshFromPackfile(RenderContext context, string cpuFilePath)
     {
         MeshInstanceData meshData = LoadRfgStaticMesh(cpuFilePath);
-        Mesh mesh = new Mesh(context, meshData.Vertices, meshData.Indices, meshData.Config.NumVertices, meshData.Config.NumIndices, (uint)meshData.Config.IndexSize);
+        Mesh mesh = new Mesh(context, meshData.Vertices, meshData.Indices, meshData.Config.NumVertices, meshData.Config.NumIndices, (uint)meshData.Config.IndexSize, context.TransferCommandPool, context.TransferQueue);
         return mesh;
     }
 
@@ -227,7 +227,7 @@ private Texture2D LoadTextureFromPackfile(RenderContext context, string cpuFileP
             ImageTiling.Optimal, ImageUsageFlags.TransferSrcBit | ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit,
             MemoryPropertyFlags.DeviceLocalBit,
             ImageAspectFlags.ColorBit);
-        texture.SetPixels(pixels, generateMipMaps: false);
+        texture.SetPixels(pixels, context.TransferCommandPool, context.TransferQueue, generateMipMaps: false);
         texture.CreateTextureSampler();
         texture.CreateImageView();
 

Nanoforge/Render/Misc/QueueFamilyIndices.cs

@@ -3,9 +3,12 @@ namespace Nanoforge.Render.Misc;
 internal struct QueueFamilyIndices
 {
     public uint? GraphicsFamily { get; set; }
+    public uint? GraphicsFamilyQueueCount { get; set; }
+    public uint? TransferFamily { get; set; }
+    public uint? TransferFamilyQueueCount { get; set; }
 
     public bool IsComplete()
     {
-        return GraphicsFamily.HasValue;
+        return GraphicsFamily.HasValue && TransferFamily.HasValue;
     }
 }

Nanoforge/Render/RenderContext.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Threading;
 using Nanoforge.Render.Misc;
 using Nanoforge.Render.Resources;
 using Serilog;
@@ -23,7 +25,9 @@ public unsafe class RenderContext : IDisposable
     public Instance Instance;
     public DebugUtilsMessengerEXT DebugUtilsMessenger;
     public CommandPool CommandPool;
+    public CommandPool TransferCommandPool;
     public Queue GraphicsQueue;
+    public Queue TransferQueue;
     private ExtDebugUtils? _debugUtils;
 
     public VkBuffer StagingBuffer = null!;
@@ -47,8 +51,6 @@ public unsafe class RenderContext : IDisposable
 
     };
 
-    //TODO: Will have to change this to not use IWindow when porting this to Avalonia
-    //TODO: Should some of this init code be stuck in a bootstrap class or something?
     public RenderContext()
     {
 #region Create Instance
@@ -148,25 +150,53 @@ public RenderContext()
 #endregion
 
 #region Create logical device
-        var indices = FindQueueFamilies(PhysicalDevice);
+        QueueFamilyIndices queueFamilyIndices = FindQueueFamilies(PhysicalDevice);
+        Debug.Assert(queueFamilyIndices.GraphicsFamily != null, "queueFamilyIndices.GraphicsFamily != null");
+        Debug.Assert(queueFamilyIndices.TransferFamily != null, "queueFamilyIndices.TransferFamily != null");
+        Log.Information($"Selected vulkan queues. Graphics queue: {queueFamilyIndices.GraphicsFamily.Value}, Transfer queue: {queueFamilyIndices.TransferFamily.Value}");
 
-        var uniqueQueueFamilies = new[] { indices.GraphicsFamily!.Value };
+        uint[] uniqueQueueFamilies = [queueFamilyIndices.GraphicsFamily!.Value, queueFamilyIndices.TransferFamily!.Value];
         uniqueQueueFamilies = uniqueQueueFamilies.Distinct().ToArray();
 
         using var mem = GlobalMemory.Allocate(uniqueQueueFamilies.Length * sizeof(DeviceQueueCreateInfo));
         var queueCreateInfos = (DeviceQueueCreateInfo*)Unsafe.AsPointer(ref mem.GetPinnableReference());
 
         float queuePriority = 1.0f;
-        for (int i = 0; i < uniqueQueueFamilies.Length; i++)
-        {
-            queueCreateInfos[i] = new()
+        uint graphicsQueueIndex;
+        uint trafficsQueueIndex;
+        if (uniqueQueueFamilies.Length == 1)
+        {
+            //The graphics and transfer queue are in the same family, so we want two of this queue type with each queue having a different index.
+            graphicsQueueIndex = 0;
+            trafficsQueueIndex = 1;
+            queueCreateInfos[0] = new()
             {
                 SType = StructureType.DeviceQueueCreateInfo,
-                QueueFamilyIndex = uniqueQueueFamilies[i],
-                QueueCount = 1,
+                QueueFamilyIndex = uniqueQueueFamilies[0],
+                QueueCount = 2,
                 PQueuePriorities = &queuePriority
             };
         }
+        else if (uniqueQueueFamilies.Length >= 2)
+        {
+            //The graphics and transfer queue are in different families. So we want 1 of each with each being at index 0.
+            graphicsQueueIndex = 0;
+            trafficsQueueIndex = 0;
+            for (int i = 0; i < uniqueQueueFamilies.Length; i++)
+            {
+                queueCreateInfos[i] = new()
+                {
+                    SType = StructureType.DeviceQueueCreateInfo,
+                    QueueFamilyIndex = uniqueQueueFamilies[i],
+                    QueueCount = 1,
+                    PQueuePriorities = &queuePriority
+                };
+            }
+        }
+        else
+        {
+            throw new Exception($"Unexpected unique vulkan queue family count. Expected either 1 or 2. Found {uniqueQueueFamilies.Length}.");
+        }
 
         PhysicalDeviceFeatures deviceFeatures = new()
         {
@@ -200,7 +230,8 @@ public RenderContext()
             throw new Exception("failed to create logical device!");
         }
 
-        Vk.GetDeviceQueue(Device, indices.GraphicsFamily!.Value, 0, out GraphicsQueue);
+        Vk.GetDeviceQueue(Device, queueFamilyIndices.GraphicsFamily!.Value, graphicsQueueIndex, out GraphicsQueue);
+        Vk.GetDeviceQueue(Device, queueFamilyIndices.TransferFamily!.Value, trafficsQueueIndex, out TransferQueue);
 
         if (EnableValidationLayers)
         {
@@ -211,7 +242,6 @@ public RenderContext()
 #endregion
 
 #region Create main command pool
-        var queueFamilyIndices = FindQueueFamilies(PhysicalDevice);
 
         CommandPoolCreateInfo poolInfo = new()
         {
@@ -222,7 +252,21 @@ public RenderContext()
 
         if (Vk.CreateCommandPool(Device, in poolInfo, null, out CommandPool) != Result.Success)
         {
-            throw new Exception("failed to create command pool!");
+            throw new Exception("Failed to create primary command pool!");
+        }
+#endregion
+
+#region Create command pool for background thread data transfers
+        CommandPoolCreateInfo transferPoolInfo = new()
+        {
+            SType = StructureType.CommandPoolCreateInfo,
+            QueueFamilyIndex = queueFamilyIndices.TransferFamily!.Value,
+            Flags = CommandPoolCreateFlags.ResetCommandBufferBit
+        };
+
+        if (Vk.CreateCommandPool(Device, in transferPoolInfo, null, out TransferCommandPool) != Result.Success)
+        {
+            throw new Exception("Failed to create transfer command pool!");
         }
 #endregion
 
@@ -272,21 +316,62 @@ private QueueFamilyIndices FindQueueFamilies(PhysicalDevice device)
             Vk.GetPhysicalDeviceQueueFamilyProperties(device, ref queueFamilityCount, queueFamiliesPtr);
         }
 
-
-        uint i = 0;
-        foreach (var queueFamily in queueFamilies)
+        //First find any graphics capable queue
+        for (uint i = 0; i < queueFamilies.Length; i++)
         {
+            QueueFamilyProperties queueFamily = queueFamilies[i];
             if (queueFamily.QueueFlags.HasFlag(QueueFlags.GraphicsBit))
             {
                 indices.GraphicsFamily = i;
+                indices.GraphicsFamilyQueueCount = queueFamily.QueueCount;
+                break;
             }
+        }
 
-            if (indices.IsComplete())
+        //TODO: Figure out how to do this. The transfer only queue on my device apparently couldn't support the pipeline barrier in the Texture2D layout transition code
+        //Next try to find a queue that only has transfer and not graphics
+        // for (uint i = 0; i < queueFamilies.Length; i++)
+        // {
+        //     QueueFamilyProperties queueFamily = queueFamilies[i];
+        //     if (queueFamily.QueueFlags.HasFlag(QueueFlags.TransferBit) && !queueFamily.QueueFlags.HasFlag(QueueFlags.GraphicsBit))
+        //     {
+        //         indices.TransferFamily = i;
+        //         indices.TransferFamilyQueueCount = queueFamily.QueueCount;
+        //         break;
+        //     }
+        // }
+        
+        //If we failed to find a transfer only queue then just use whatever queue that's available for transfer
+        if (!indices.TransferFamily.HasValue)
+        {
+            for (uint i = 0; i < queueFamilies.Length; i++)
             {
-                break;
+                QueueFamilyProperties queueFamily = queueFamilies[i];
+                if (queueFamily.QueueFlags.HasFlag(QueueFlags.TransferBit))
+                {
+                    indices.TransferFamily = i;
+                    indices.TransferFamilyQueueCount = queueFamily.QueueCount;
+                    break;
+                }
             }
-
-            i++;
+        }
+        
+        //Make sure there's enough queues available if the graphics and transfer queue are in the same family
+        if (indices is { GraphicsFamily: not null, TransferFamily: not null } && indices.GraphicsFamily.Value == indices.TransferFamily.Value)
+        {
+            if (indices.GraphicsFamilyQueueCount < 2)
+            {
+                string err = $"Graphics and transfer queue families are the same and there isn't enough queues available. Require 2, only {indices.GraphicsFamilyQueueCount} are available.";
+                Log.Error(err);
+                throw new Exception(err);
+            }
+        }
+        
+        if (!indices.IsComplete())
+        {
+            string err = "Failed to find valid vulkan queue families for graphics and transfer.";
+            Log.Error(err);
+            throw new Exception(err);
         }
 
         return indices;
@@ -354,13 +439,13 @@ public uint FindMemoryType(uint typeFilter, MemoryPropertyFlags properties)
         throw new Exception("failed to find suitable memory type!");
     }
 
-    public CommandBuffer BeginSingleTimeCommands()
+    public CommandBuffer BeginSingleTimeCommands(CommandPool pool)
     {
         CommandBufferAllocateInfo allocateInfo = new()
         {
             SType = StructureType.CommandBufferAllocateInfo,
             Level = CommandBufferLevel.Primary,
-            CommandPool = CommandPool,
+            CommandPool = pool,
             CommandBufferCount = 1,
         };
 
@@ -377,7 +462,7 @@ public CommandBuffer BeginSingleTimeCommands()
         return commandBuffer;
     }
 
-    public void EndSingleTimeCommands(CommandBuffer commandBuffer)
+    public void EndSingleTimeCommands(CommandBuffer commandBuffer, CommandPool pool, Queue queue)
     {
         Vk.EndCommandBuffer(commandBuffer);
 
@@ -388,23 +473,23 @@ public void EndSingleTimeCommands(CommandBuffer commandBuffer)
             PCommandBuffers = &commandBuffer,
         };
 
-        Vk.QueueSubmit(GraphicsQueue, 1, in submitInfo, default);
-        Vk.QueueWaitIdle(GraphicsQueue);
+        Vk.QueueSubmit(queue, 1, in submitInfo, default);
+        Vk.QueueWaitIdle(queue);
 
-        Vk.FreeCommandBuffers(Device, CommandPool, 1, in commandBuffer);
+        Vk.FreeCommandBuffers(Device, pool, 1, in commandBuffer);
     }
 
     private void CreateStagingBuffer()
     {
         StagingBuffer = new VkBuffer(this, StagingBufferSize, BufferUsageFlags.TransferSrcBit, MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, canGrow: true);
     }
 
-    public CommandBuffer AllocateCommandBuffer()
+    public CommandBuffer AllocateCommandBuffer(CommandPool pool)
     {
         var allocInfo = new CommandBufferAllocateInfo
         {
             SType = StructureType.CommandBufferAllocateInfo,
-            CommandPool = CommandPool,
+            CommandPool = pool,
             CommandBufferCount = 1,
             Level = CommandBufferLevel.Primary
         };
@@ -416,6 +501,7 @@ public void Dispose()
     {
         StagingBuffer.Destroy();
         Vk.DestroyCommandPool(Device, CommandPool, null);
+        Vk.DestroyCommandPool(Device, TransferCommandPool, null);
         Vk.DestroyDevice(Device, null);
         _debugUtils?.DestroyDebugUtilsMessenger(Instance, DebugUtilsMessenger, null);
         Vk.DestroyInstance(Instance, null);

Nanoforge/Render/Resources/Mesh.cs

@@ -18,19 +18,19 @@ public class Mesh
 
     public bool Destroyed { get; private set; } = false;
 
-    public Mesh(RenderContext context, Span<byte> vertices, Span<byte> indices, uint vertexCount, uint indexCount, uint indexSize)
+    public Mesh(RenderContext context, Span<byte> vertices, Span<byte> indices, uint vertexCount, uint indexCount, uint indexSize, CommandPool pool, Queue queue)
     {
         _context = context;
         VertexCount = vertexCount;
         IndexCount = indexCount;
 
         _vertexBuffer = new VkBuffer(_context, (ulong)vertices.Length, BufferUsageFlags.TransferDstBit | BufferUsageFlags.VertexBufferBit, MemoryPropertyFlags.DeviceLocalBit);
         _context.StagingBuffer.SetData(vertices);
-        _context.StagingBuffer.CopyTo(_vertexBuffer, (ulong)vertices.Length);
+        _context.StagingBuffer.CopyTo(_vertexBuffer, (ulong)vertices.Length, pool, queue);
 
         _indexBuffer = new VkBuffer(_context, (ulong)indices.Length, BufferUsageFlags.TransferDstBit | BufferUsageFlags.IndexBufferBit, MemoryPropertyFlags.DeviceLocalBit);
         _context.StagingBuffer.SetData(indices);
-        _context.StagingBuffer.CopyTo(_indexBuffer, (ulong)indices.Length);
+        _context.StagingBuffer.CopyTo(_indexBuffer, (ulong)indices.Length, pool, queue);
 
         _indexType = indexSize switch
         {

Nanoforge/Render/Resources/Texture2D.cs

@@ -94,24 +94,24 @@ private void CreateImage()
         Context.Vk.BindImageMemory(Context.Device, _textureImage, Memory, 0);
     }
 
-    public void SetPixels(byte[] pixels, bool generateMipMaps = false)
+    public void SetPixels(byte[] pixels, CommandPool pool, Queue queue, bool generateMipMaps = false)
     {
-        TransitionLayoutImmediate(ImageLayout.TransferDstOptimal);
+        TransitionLayoutImmediate(ImageLayout.TransferDstOptimal, pool, queue);
         Context.StagingBuffer.SetData(pixels);
-        Context.StagingBuffer.CopyToImage(_textureImage, _width, _height); 
-        TransitionLayoutImmediate(ImageLayout.ShaderReadOnlyOptimal);
+        Context.StagingBuffer.CopyToImage(_textureImage, _width, _height, pool, queue); 
+        TransitionLayoutImmediate(ImageLayout.ShaderReadOnlyOptimal, pool, queue);
 
         if (generateMipMaps)
         {
-            GenerateMipMaps(Context, _textureImage, _format, _width, _height, _mipLevels);
+            GenerateMipMaps(Context, _textureImage, _format, _width, _height, _mipLevels, pool, queue);
         }
     }
 
-    public void TransitionLayoutImmediate(ImageLayout newLayout)
+    public void TransitionLayoutImmediate(ImageLayout newLayout, CommandPool pool, Queue queue)
     {
-        CommandBuffer cmd = Context.BeginSingleTimeCommands();
+        CommandBuffer cmd = Context.BeginSingleTimeCommands(pool);
         TransitionLayout(cmd, newLayout);
-        Context.EndSingleTimeCommands(cmd);
+        Context.EndSingleTimeCommands(cmd, pool, queue);
     }
 
     public void TransitionLayout(CommandBuffer cmd, ImageLayout newLayout)
@@ -185,7 +185,7 @@ public void TransitionLayout(CommandBuffer cmd, ImageLayout newLayout)
         _currentLayout = newLayout;
     }
 
-    private static void GenerateMipMaps(RenderContext context, Image image, Format imageFormat, uint width, uint height, uint mipLevels)
+    private static void GenerateMipMaps(RenderContext context, Image image, Format imageFormat, uint width, uint height, uint mipLevels, CommandPool pool, Queue queue)
     {
         context.Vk.GetPhysicalDeviceFormatProperties(context.PhysicalDevice, imageFormat, out var formatProperties);
 
@@ -194,7 +194,7 @@ private static void GenerateMipMaps(RenderContext context, Image image, Format i
             throw new Exception("texture image format does not support linear blitting!");
         }
 
-        var commandBuffer = context.BeginSingleTimeCommands();
+        var commandBuffer = context.BeginSingleTimeCommands(pool);
 
         ImageMemoryBarrier barrier = new()
         {
@@ -287,7 +287,7 @@ private static void GenerateMipMaps(RenderContext context, Image image, Format i
             0, null,
             1, in barrier);
 
-        context.EndSingleTimeCommands(commandBuffer);
+        context.EndSingleTimeCommands(commandBuffer, pool, queue);
     }
 
     public void CreateImageView()
@@ -367,7 +367,7 @@ public void CopyToBuffer(CommandBuffer cmd, Buffer buffer)
         Vk.CmdCopyImageToBuffer(cmd, _textureImage, _currentLayout, buffer, 1, copyRegion);
     }
 
-    public static Texture2D FromFile(RenderContext context, string path)
+    public static Texture2D FromFile(RenderContext context, CommandPool pool, Queue queue, string path)
     {
         using var img = SixLabors.ImageSharp.Image.Load<SixLabors.ImageSharp.PixelFormats.Rgba32>(path);
         ulong imageSize = (ulong)(img.Width * img.Height * img.PixelType.BitsPerPixel / 8);
@@ -382,7 +382,7 @@ public static Texture2D FromFile(RenderContext context, string path)
         Texture2D texture = new Texture2D(context, (uint)img.Width, (uint)img.Height, mipLevels, vulkanFormat,
             ImageTiling.Optimal, ImageUsageFlags.TransferSrcBit | ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit, MemoryPropertyFlags.DeviceLocalBit,
             ImageAspectFlags.ColorBit);
-        texture.SetPixels(pixelData, generateMipMaps: false);
+        texture.SetPixels(pixelData, pool, queue, generateMipMaps: false);
         texture.CreateTextureSampler();
         texture.CreateImageView();