backend/drm: get rid of BO handle table

The BO handle table exists to avoid double-closing a BO handle,
which aren't reference-counted by the kernel. But if we can
guarantee that there is only ever a single ref for each BO handle,
then we don't need the BO handle table anymore.

This is possible if we create the handle right before the ADDFB2
IOCTL, and close the handle right after. The handles are very
short-lived and we don't need to track their lifetime.

Because of multi-planar FBs, we need to be a bit careful: some
FB planes might share the same handle. But with a small check, it's
easy to avoid double-closing the same handle (which wouldn't be a
big deal anyways).

There's one gotcha though: drmModeSetCursor2 takes a BO handle as
input. Saving the handles until drmModeSetCursor2 time would require
us to track BO handle lifetimes, so we wouldn't be able to get rid
of the BO handle table. As a workaround, use drmModeGetFB to turn the
FB ID back to a BO handle, call drmModeSetCursor2 and then immediately
close the BO handle. The overhead should be minimal since these IOCTLs
are pretty cheap.

Closes: https://github.com/swaywm/wlroots/issues/3164

4 files changed, 7 insertions, 27 deletions

diff --git a/include/backend/drm/bo_handle_table.h b/include/backend/drm/bo_handle_table.h
deleted file mode 100644
index d086df45..00000000
--- a/include/backend/drm/bo_handle_table.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#ifndef BACKEND_DRM_BO_HANDLE_TABLE_H
-#define BACKEND_DRM_BO_HANDLE_TABLE_H
-
-/**
- * Table performing reference counting for buffer object handles.
- *
- * The BO handles are allocated incrementally and are recycled by the kernel,
- * so a simple array is used.
- *
- * This design is inspired from amdgpu's code in libdrm:
- * https://gitlab.freedesktop.org/mesa/drm/-/blob/1a4c0ec9aea13211997f982715fe5ffcf19dd067/amdgpu/handle_table.c
- */
-struct wlr_drm_bo_handle_table {
-	size_t *nrefs;
-	size_t len;
-};
-
-void drm_bo_handle_table_finish(struct wlr_drm_bo_handle_table *table);
-bool drm_bo_handle_table_ref(struct wlr_drm_bo_handle_table *table,
-	uint32_t handle);
-size_t drm_bo_handle_table_unref(struct wlr_drm_bo_handle_table *table,
-	uint32_t handle);
-
-#endif
diff --git a/include/backend/drm/drm.h b/include/backend/drm/drm.h
index 6d70d844..4b2a1f7c 100644
--- a/include/backend/drm/drm.h
+++ b/include/backend/drm/drm.h
@@ -11,7 +11,6 @@
 #include <wlr/backend/session.h>
 #include <wlr/render/drm_format_set.h>
 #include <xf86drmMode.h>
-#include "backend/drm/bo_handle_table.h"
 #include "backend/drm/iface.h"
 #include "backend/drm/properties.h"
 #include "backend/drm/renderer.h"
@@ -63,7 +62,6 @@ struct wlr_drm_backend {
 	int fd;
 	char *name;
 	struct wlr_device *dev;
-	struct wlr_drm_bo_handle_table bo_handles;
 
 	size_t num_crtcs;
 	struct wlr_drm_crtc *crtcs;
diff --git a/include/backend/drm/renderer.h b/include/backend/drm/renderer.h
index d6f878c5..52a2a053 100644
--- a/include/backend/drm/renderer.h
+++ b/include/backend/drm/renderer.h
@@ -32,7 +32,6 @@ struct wlr_drm_fb {
 	struct wlr_drm_backend *backend;
 	struct wl_list link; // wlr_drm_backend.fbs
 
-	uint32_t handles[WLR_DMABUF_MAX_PLANES];
 	uint32_t id;
 };
 
diff --git a/include/backend/drm/util.h b/include/backend/drm/util.h
index b4cdee7d..6e97505c 100644
--- a/include/backend/drm/util.h
+++ b/include/backend/drm/util.h
@@ -36,4 +36,11 @@ size_t match_obj(size_t num_objs, const uint32_t objs[static restrict num_objs],
 		size_t num_res, const uint32_t res[static restrict num_res],
 		uint32_t out[static restrict num_res]);
 
+/**
+ * Close a GEM buffer handle.
+ *
+ * TODO: replace with drmCloseBufferHandle.
+ */
+void close_bo_handle(int drm_fd, uint32_t handle);
+
 #endif