backend/drm: introduce wlr_drm_bo_handle_table

Using GBM to import DRM dumb buffers tends to not work well. By
using GBM we're calling some driver-specific functions in Mesa.
These functions check whether Mesa can work with the buffer.
Sometimes Mesa has requirements which differ from DRM dumb buffers
and the GBM import will fail (e.g. on amdgpu).

Instead, drop GBM and use drmPrimeFDToHandle directly. But there's
a twist: BO handles are not ref'counted by the kernel and need to
be ref'counted in user-space [1]. libdrm usually performs this
bookkeeping and is used under-the-hood by Mesa.

We can't re-use libdrm for this task without using driver-specific
APIs. So let's just re-implement the ref'counting logic in wlroots.
The wlroots implementation is inspired from amdgpu's in libdrm [2].

Closes: https://github.com/swaywm/wlroots/issues/2916

[1]: https://gitlab.freedesktop.org/mesa/drm/-/merge_requests/110
[2]: https://gitlab.freedesktop.org/mesa/drm/-/blob/1a4c0ec9aea13211997f982715fe5ffcf19dd067/amdgpu/handle_table.c

5 files changed, 28 insertions, 7 deletions

diff --git a/include/backend/drm/bo_handle_table.h b/include/backend/drm/bo_handle_table.h
new file mode 100644
index 00000000..d086df45
--- /dev/null
+++ b/include/backend/drm/bo_handle_table.h
@@ -0,0 +1,24 @@
+#ifndef BACKEND_DRM_BO_HANDLE_TABLE_H
+#define BACKEND_DRM_BO_HANDLE_TABLE_H
+
+/**
+ * Table performing reference counting for buffer object handles.
+ *
+ * The BO handles are allocated incrementally and are recycled by the kernel,
+ * so a simple array is used.
+ *
+ * This design is inspired from amdgpu's code in libdrm:
+ * https://gitlab.freedesktop.org/mesa/drm/-/blob/1a4c0ec9aea13211997f982715fe5ffcf19dd067/amdgpu/handle_table.c
+ */
+struct wlr_drm_bo_handle_table {
+	size_t *nrefs;
+	size_t len;
+};
+
+void drm_bo_handle_table_finish(struct wlr_drm_bo_handle_table *table);
+bool drm_bo_handle_table_ref(struct wlr_drm_bo_handle_table *table,
+	uint32_t handle);
+size_t drm_bo_handle_table_unref(struct wlr_drm_bo_handle_table *table,
+	uint32_t handle);
+
+#endif
diff --git a/include/backend/drm/drm.h b/include/backend/drm/drm.h
index 9aae5e0b..d57ddbf4 100644
--- a/include/backend/drm/drm.h
+++ b/include/backend/drm/drm.h
@@ -1,7 +1,6 @@
 #ifndef BACKEND_DRM_DRM_H
 #define BACKEND_DRM_DRM_H
 
-#include <gbm.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -12,6 +11,7 @@
 #include <wlr/backend/session.h>
 #include <wlr/render/drm_format_set.h>
 #include <xf86drmMode.h>
+#include "backend/drm/bo_handle_table.h"
 #include "backend/drm/iface.h"
 #include "backend/drm/properties.h"
 #include "backend/drm/renderer.h"
@@ -62,7 +62,7 @@ struct wlr_drm_backend {
 	int fd;
 	char *name;
 	struct wlr_device *dev;
-	struct gbm_device *gbm;
+	struct wlr_drm_bo_handle_table bo_handles;
 
 	size_t num_crtcs;
 	struct wlr_drm_crtc *crtcs;
diff --git a/include/backend/drm/iface.h b/include/backend/drm/iface.h
index e02c2199..98f7e06c 100644
--- a/include/backend/drm/iface.h
+++ b/include/backend/drm/iface.h
@@ -1,7 +1,6 @@
 #ifndef BACKEND_DRM_IFACE_H
 #define BACKEND_DRM_IFACE_H
 
-#include <gbm.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <xf86drm.h>
diff --git a/include/backend/drm/renderer.h b/include/backend/drm/renderer.h
index 70a9533b..d6f878c5 100644
--- a/include/backend/drm/renderer.h
+++ b/include/backend/drm/renderer.h
@@ -1,7 +1,6 @@
 #ifndef BACKEND_DRM_RENDERER_H
 #define BACKEND_DRM_RENDERER_H
 
-#include <gbm.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <wlr/backend.h>
@@ -30,9 +29,10 @@ struct wlr_drm_surface {
 struct wlr_drm_fb {
 	struct wlr_buffer *wlr_buf;
 	struct wlr_addon addon;
+	struct wlr_drm_backend *backend;
 	struct wl_list link; // wlr_drm_backend.fbs
 
-	struct gbm_bo *bo;
+	uint32_t handles[WLR_DMABUF_MAX_PLANES];
 	uint32_t id;
 };
 
diff --git a/include/backend/drm/util.h b/include/backend/drm/util.h
index 15895ec6..b4cdee7d 100644
--- a/include/backend/drm/util.h
+++ b/include/backend/drm/util.h
@@ -13,8 +13,6 @@ void parse_edid(struct wlr_output *restrict output, size_t len,
 	const uint8_t *data);
 // Returns the string representation of a DRM output type
 const char *conn_get_name(uint32_t type_id);
-// Returns the DRM framebuffer id for a gbm_bo
-uint32_t get_fb_for_bo(struct gbm_bo *bo, bool with_modifiers);
 
 // Part of match_obj
 enum {