Improve Vulkan initialization;

- Check for layers before enabling
- Check for instance/device extensions before enabling

Fixes unfriendly errors when running on a system without validation layers
installed.

Uses same table approach as OpenXR code.

src/core/gpu_vk.c

@@ -210,6 +210,8 @@ static gpu_memory* gpu_allocate(gpu_memory_type type, VkMemoryRequirements info,
 static void gpu_release(gpu_memory* memory);
 static void condemn(void* handle, VkObjectType type);
 static void expunge(void);
+static bool hasLayer(VkLayerProperties* layers, uint32_t count, const char* layer);
+static bool hasExtension(VkExtensionProperties* extensions, uint32_t count, const char* extension);
 static void createSwapchain(uint32_t width, uint32_t height);
 static VkRenderPass getCachedRenderPass(gpu_pass_info* pass, bool exact);
 static VkFramebuffer getCachedFramebuffer(VkRenderPass pass, VkImageView images[9], uint32_t imageCount, uint32_t size[2]);
@@ -226,6 +228,7 @@ static bool check(bool condition, const char* message);
 
 // Functions that don't require an instance
 #define GPU_FOREACH_ANONYMOUS(X)\
+  X(vkEnumerateInstanceLayerProperties)\
   X(vkEnumerateInstanceExtensionProperties)\
   X(vkCreateInstance)
 
@@ -1840,51 +1843,86 @@ bool gpu_init(gpu_config* config) {
   GPU_FOREACH_ANONYMOUS(GPU_LOAD_ANONYMOUS);
 
   { // Instance
-    const char* extensions[32];
+    struct {
-    uint32_t extensionCount = 0;
+      bool validation;
+      bool portability;
+      bool debug;
+    } supports = { 0 };
 
-    if (state.config.vk.getInstanceExtensions) {
+    // Layers
-      const char** instanceExtensions = state.config.vk.getInstanceExtensions(&extensionCount);
+
-      CHECK(extensionCount < COUNTOF(extensions), "Too many instance extensions") return gpu_destroy(), false;
+    struct { const char* name; bool shouldEnable; bool* isEnabled; } layers[] = {
-      for (uint32_t i = 0; i < extensionCount; i++) {
+      { "VK_LAYER_KHRONOS_validation", config->debug, &supports.validation }
-        extensions[i] = instanceExtensions[i];
+    };
+
+    const char* enabledLayers[1];
+    uint32_t enabledLayerCount = 0;
+
+    VkLayerProperties layerInfo[32];
+    uint32_t count = COUNTOF(layerInfo);
+    VK(vkEnumerateInstanceLayerProperties(&count, layerInfo), "Failed to enumerate instance layers") return gpu_destroy(), false;
+
+    for (uint32_t i = 0; i < COUNTOF(layers); i++) {
+      if (!layers[i].shouldEnable) continue;
+      if (hasLayer(layerInfo, count, layers[i].name)) {
+        CHECK(enabledLayerCount < COUNTOF(enabledLayers), "Too many layers") return gpu_destroy(), false;
+        if (layers[i].isEnabled) *layers[i].isEnabled = true;
+        enabledLayers[enabledLayerCount++] = layers[i].name;
       }
     }
 
-    if (state.config.debug) {
+    // Extensions
-      CHECK(extensionCount < COUNTOF(extensions), "Too many instance extensions") return gpu_destroy(), false;
+
-      extensions[extensionCount++] = "VK_EXT_debug_utils";
+    struct { const char* name; bool shouldEnable; bool* isEnabled; } extensions[] = {
+      { "VK_KHR_portability_enumeration", true, &supports.portability },
+      { "VK_EXT_debug_utils", config->debug, &supports.debug }
+    };
+
+    const char* enabledExtensions[32];
+    uint32_t enabledExtensionCount = 0;
+
+    if (state.config.vk.getInstanceExtensions) {
+      const char** instanceExtensions = state.config.vk.getInstanceExtensions(&enabledExtensionCount);
+      CHECK(enabledExtensionCount < COUNTOF(enabledExtensions), "Too many instance extensions") return gpu_destroy(), false;
+      for (uint32_t i = 0; i < enabledExtensionCount; i++) {
+        enabledExtensions[i] = instanceExtensions[i];
+      }
     }
 
     VkExtensionProperties extensionInfo[256];
-    uint32_t count = COUNTOF(extensionInfo);
+    count = COUNTOF(extensionInfo);
     VK(vkEnumerateInstanceExtensionProperties(NULL, &count, extensionInfo), "Failed to enumerate instance extensions") return gpu_destroy(), false;
 
-    VkInstanceCreateFlags instanceFlags = 0;
+    for (uint32_t i = 0; i < COUNTOF(extensions); i++) {
-
+      if (!extensions[i].shouldEnable) continue;
-#ifdef VK_KHR_portability_enumeration
+      if (hasExtension(extensionInfo, count, extensions[i].name)) {
-    for (uint32_t i = 0; i < count; i++) {
+        CHECK(enabledExtensionCount < COUNTOF(enabledExtensions), "Too many instance extensions") return gpu_destroy(), false;
-      if (!strcmp(extensionInfo[i].extensionName, "VK_KHR_portability_enumeration")) {
+        if (extensions[i].isEnabled) *extensions[i].isEnabled = true;
-        CHECK(extensionCount < COUNTOF(extensions), "Too many instance extensions") return gpu_destroy(), false;
+        enabledExtensions[enabledExtensionCount++] = extensions[i].name;
-        extensions[extensionCount++] = "VK_KHR_portability_enumeration";
-        instanceFlags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;
       }
     }
-#endif
+
+    if (state.config.debug && !supports.validation && state.config.callback) {
+      state.config.callback(state.config.userdata, "Warning: GPU debug mode is enabled, but validation layer is not supported", false);
+    }
+
+    if (state.config.debug && !supports.debug) {
+      state.config.debug = false;
+    }
 
     VkInstanceCreateInfo instanceInfo = {
       .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
-      .flags = instanceFlags,
+      .flags = supports.portability ? VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR : 0,
       .pApplicationInfo = &(VkApplicationInfo) {
         .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
         .pEngineName = config->engineName,
         .engineVersion = VK_MAKE_VERSION(config->engineVersion[0], config->engineVersion[1], config->engineVersion[2]),
         .apiVersion = VK_MAKE_VERSION(1, 1, 0)
       },
-      .enabledLayerCount = state.config.debug ? 1 : 0,
+      .enabledLayerCount = enabledLayerCount,
-      .ppEnabledLayerNames = (const char*[]) { "VK_LAYER_KHRONOS_validation" },
+      .ppEnabledLayerNames = enabledLayers,
-      .enabledExtensionCount = extensionCount,
+      .enabledExtensionCount = enabledExtensionCount,
-      .ppEnabledExtensionNames = extensions
+      .ppEnabledExtensionNames = enabledExtensions
     };
 
     if (state.config.vk.createInstance) {
@@ -2056,23 +2094,33 @@ bool gpu_init(gpu_config* config) {
     }
     CHECK(state.queueFamilyIndex != ~0u, "Queue selection failed") return gpu_destroy(), false;
 
-    const char* extensions[4];
+    struct {
-    uint32_t extensionCount = 0;
+      bool swapchain;
+    } supports = { 0 };
 
-    if (state.surface) {
+    struct { const char* name; bool shouldEnable; bool* isEnabled; } extensions[] = {
-      extensions[extensionCount++] = "VK_KHR_swapchain";
+      { "VK_KHR_swapchain", state.surface, &supports.swapchain },
-    }
+      { "VK_KHR_portability_subset", true, NULL }
+    };
+
+    const char* enabledExtensions[4];
+    uint32_t enabledExtensionCount = 0;
 
     VkExtensionProperties extensionInfo[256];
     uint32_t count = COUNTOF(extensionInfo);
     VK(vkEnumerateDeviceExtensionProperties(state.adapter, NULL, &count, extensionInfo), "Failed to enumerate device extensions") return gpu_destroy(), false;
 
-    for (uint32_t i = 0; i < count; i++) {
+    for (uint32_t i = 0; i < COUNTOF(extensions); i++) {
-      if (!strcmp(extensionInfo[i].extensionName, "VK_KHR_portability_subset")) {
+      if (!extensions[i].shouldEnable) continue;
-        extensions[extensionCount++] = "VK_KHR_portability_subset";
+      if (hasExtension(extensionInfo, count, extensions[i].name)) {
+        CHECK(enabledExtensionCount < COUNTOF(enabledExtensions), "Too many device extensions") return gpu_destroy(), false;
+        if (extensions[i].isEnabled) *extensions[i].isEnabled = true;
+        enabledExtensions[enabledExtensionCount++] = extensions[i].name;
       }
     }
 
+    CHECK(supports.swapchain || !state.surface, "Swapchain extension not supported");
+
     VkDeviceCreateInfo deviceInfo = {
       .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
       .pNext = config->features ? &enabledFeatures : NULL,
@@ -2083,8 +2131,8 @@ bool gpu_init(gpu_config* config) {
         .pQueuePriorities = &(float) { 1.f },
         .queueCount = 1
       },
-      .enabledExtensionCount = extensionCount,
+      .enabledExtensionCount = enabledExtensionCount,
-      .ppEnabledExtensionNames = extensions
+      .ppEnabledExtensionNames = enabledExtensions
     };
 
     if (state.config.vk.createDevice) {
@@ -2586,6 +2634,24 @@ static void expunge() {
   }
 }
 
+static bool hasLayer(VkLayerProperties* layers, uint32_t count, const char* layer) {
+  for (uint32_t i = 0; i < count; i++) {
+    if (!strcmp(layers[i].layerName, layer)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool hasExtension(VkExtensionProperties* extensions, uint32_t count, const char* extension) {
+  for (uint32_t i = 0; i < count; i++) {
+    if (!strcmp(extensions[i].extensionName, extension)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 static void createSwapchain(uint32_t width, uint32_t height) {
   if (width == 0 || height == 0) {
     state.swapchainValid = false;