From 72b5d55687a8e16d744eaad9db6a6bd90f035552 Mon Sep 17 00:00:00 2001
From: Kenny Levinsen <kl@kl.wtf>
Date: Fri, 13 Sep 2024 14:52:38 +0200
Subject: [PATCH] config/output: Pick among all 8-bit and 10-bit formats

The usual 8-bit and 10-bit formats have a lot of variations:

        XRGB, XBGR, RGBX, BGRX, ARGB, ABGR, RGBA, BGRA

A particular display is likely to support multiple of these, and as we
need to allocate a buffer of a particular type to test it, we'd like to
avoid doing too many unnecessary tests.

If we go off on the assumption that a display device generally supports
a subset of format variations but otherwise support each variation
equally well - that is, if both RGB and BGR is supported, they will
either both succeed or both fail at any given time - then instead of
trying all formats, we can simply pick one supported format within each
format group to try. For example, XBGR2101010 and XRGB8888.

By being conservative in how many formats we try, we can aggressively
expand the list of candidate formats to handle odd hardware.

We can get rid of the the conservative aspect if KMS ever grows the
ability to commit-test a buffer format without allocating the buffer
first, allowing us to quickly test every format under the sun.
---
 sway/config/output.c | 76 ++++++++++++++++++++++++++++----------------
 1 file changed, 48 insertions(+), 28 deletions(-)

diff --git a/sway/config/output.c b/sway/config/output.c
index 8e2528c8..e7e34a08 100644
--- a/sway/config/output.c
+++ b/sway/config/output.c
@@ -410,10 +410,6 @@ static enum render_bit_depth bit_depth_from_format(uint32_t render_format) {
 	return RENDER_BIT_DEPTH_DEFAULT;
 }
 
-static bool render_format_is_bgr(uint32_t fmt) {
-	return fmt == DRM_FORMAT_XBGR2101010 || fmt == DRM_FORMAT_XBGR8888;
-}
-
 static bool output_config_is_disabling(struct output_config *oc) {
 	return oc && (!oc->enabled || oc->power == 0);
 }
@@ -785,44 +781,68 @@ static bool search_mode(struct search_context *ctx, size_t output_idx) {
 	return false;
 }
 
+struct fmtmap_entry {
+	enum render_bit_depth bit_depth;
+	const uint32_t *fmts;
+};
+
+static const struct fmtmap_entry map[] = {
+	{RENDER_BIT_DEPTH_10, (uint32_t[]){
+		DRM_FORMAT_XRGB2101010,
+		DRM_FORMAT_XBGR2101010,
+		DRM_FORMAT_RGBX1010102,
+		DRM_FORMAT_BGRX1010102,
+		DRM_FORMAT_ARGB2101010,
+		DRM_FORMAT_ABGR2101010,
+		DRM_FORMAT_RGBA1010102,
+		DRM_FORMAT_BGRA1010102,
+		DRM_FORMAT_INVALID,
+	}},
+	{RENDER_BIT_DEPTH_8, (uint32_t[]){
+		DRM_FORMAT_XRGB8888,
+		DRM_FORMAT_XBGR8888,
+		DRM_FORMAT_RGBX8888,
+		DRM_FORMAT_BGRX8888,
+		DRM_FORMAT_ARGB8888,
+		DRM_FORMAT_ABGR8888,
+		DRM_FORMAT_RGBA8888,
+		DRM_FORMAT_BGRA8888,
+		DRM_FORMAT_INVALID,
+	}},
+	{RENDER_BIT_DEPTH_6, (uint32_t[]){
+		DRM_FORMAT_RGB565,
+		DRM_FORMAT_INVALID,
+	}},
+};
+
 static bool search_render_format(struct search_context *ctx, size_t output_idx) {
 	struct matched_output_config *cfg = &ctx->configs[output_idx];
 	struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
 	struct wlr_output_state *state = &backend_state->base;
 	struct wlr_output *wlr_output = backend_state->output;
 
-	uint32_t fmts[] = {
-		DRM_FORMAT_XRGB2101010,
-		DRM_FORMAT_XBGR2101010,
-		DRM_FORMAT_XRGB8888,
-		DRM_FORMAT_ARGB8888,
-		DRM_FORMAT_RGB565,
-		DRM_FORMAT_INVALID,
-	};
-	if (render_format_is_bgr(wlr_output->render_format)) {
-		// Start with BGR in the unlikely event that we previously required it.
-		fmts[0] = DRM_FORMAT_XBGR2101010;
-		fmts[1] = DRM_FORMAT_XRGB2101010;
-	}
-
 	const struct wlr_drm_format_set *primary_formats =
 		wlr_output_get_primary_formats(wlr_output, WLR_BUFFER_CAP_DMABUF);
 	enum render_bit_depth needed_bits = RENDER_BIT_DEPTH_8;
 	if (cfg->config && cfg->config->render_bit_depth != RENDER_BIT_DEPTH_DEFAULT) {
 		needed_bits = cfg->config->render_bit_depth;
 	}
-	for (size_t idx = 0; fmts[idx] != DRM_FORMAT_INVALID; idx++) {
-		enum render_bit_depth format_bits = bit_depth_from_format(fmts[idx]);
-		if (needed_bits < format_bits) {
+
+	// Try one supported format from each category
+	for (size_t mapidx = 0; mapidx < sizeof(map); mapidx++) {
+		const struct fmtmap_entry *entry = &map[mapidx];
+		if (needed_bits < entry->bit_depth) {
+			// This format category has too many bits
 			continue;
 		}
-		if (!wlr_drm_format_set_get(primary_formats, fmts[idx])) {
-			// This is not a supported format for this output
-			continue;
-		}
-		wlr_output_state_set_render_format(state, fmts[idx]);
-		if (search_mode(ctx, output_idx)) {
-			return true;
+		for (size_t idx = 0; entry->fmts[idx] != DRM_FORMAT_INVALID; idx++) {
+			if (wlr_drm_format_set_get(primary_formats, entry->fmts[idx])) {
+				wlr_output_state_set_render_format(state, entry->fmts[idx]);
+				if (search_mode(ctx, output_idx)) {
+					return true;
+				}
+				break;
+			}
 		}
 	}
 	return false;