render/pixel_format: import pixel_format_info_check_stride()

We'll use this function from wlr_shm too.

Add some assertions, use int32_t (since the wire protocol uses that,
and we don't want to use 16-bit integers on exotic systems) and
switch the stride check to be overflow-safe.

2 files changed, 21 insertions, 17 deletions

diff --git a/render/gles2/texture.c b/render/gles2/texture.c
index f304df93..9dccc526 100644
--- a/render/gles2/texture.c
+++ b/render/gles2/texture.c
@@ -28,21 +28,6 @@ struct wlr_gles2_texture *gles2_get_texture(
 	return (struct wlr_gles2_texture *)wlr_texture;
 }
 
-static bool check_stride(const struct wlr_pixel_format_info *fmt,
-		uint32_t stride, uint32_t width) {
-	if (stride % (fmt->bpp / 8) != 0) {
-		wlr_log(WLR_ERROR, "Invalid stride %d (incompatible with %d "
-			"bytes-per-pixel)", stride, fmt->bpp / 8);
-		return false;
-	}
-	if (stride < width * (fmt->bpp / 8)) {
-		wlr_log(WLR_ERROR, "Invalid stride %d (too small for %d "
-			"bytes-per-pixel and width %d)", stride, fmt->bpp / 8, width);
-		return false;
-	}
-	return true;
-}
-
 static bool gles2_texture_update_from_buffer(struct wlr_texture *wlr_texture,
 		struct wlr_buffer *buffer, const pixman_region32_t *damage) {
 	struct wlr_gles2_texture *texture = gles2_get_texture(wlr_texture);
@@ -72,7 +57,7 @@ static bool gles2_texture_update_from_buffer(struct wlr_texture *wlr_texture,
 		drm_get_pixel_format_info(texture->drm_format);
 	assert(drm_fmt);
 
-	if (!check_stride(drm_fmt, stride, buffer->width)) {
+	if (!pixel_format_info_check_stride(drm_fmt, stride, buffer->width)) {
 		wlr_buffer_end_data_ptr_access(buffer);
 		return false;
 	}
@@ -212,7 +197,7 @@ static struct wlr_texture *gles2_texture_from_pixels(
 		drm_get_pixel_format_info(drm_format);
 	assert(drm_fmt);
 
-	if (!check_stride(drm_fmt, stride, width)) {
+	if (!pixel_format_info_check_stride(drm_fmt, stride, width)) {
 		return NULL;
 	}
 
diff --git a/render/pixel_format.c b/render/pixel_format.c
index 586d2a09..330a59cc 100644
--- a/render/pixel_format.c
+++ b/render/pixel_format.c
@@ -1,4 +1,6 @@
+#include <assert.h>
 #include <drm_fourcc.h>
+#include <wlr/util/log.h>
 #include "render/pixel_format.h"
 
 static const struct wlr_pixel_format_info pixel_format_info[] = {
@@ -176,3 +178,20 @@ enum wl_shm_format convert_drm_format_to_wl_shm(uint32_t fmt) {
 		return (enum wl_shm_format)fmt;
 	}
 }
+
+bool pixel_format_info_check_stride(const struct wlr_pixel_format_info *fmt,
+		int32_t stride, int32_t width) {
+	assert(fmt->bpp > 0 && fmt->bpp % 8 == 0);
+	int32_t bytes_per_pixel = (int32_t)(fmt->bpp / 8);
+	if (stride % bytes_per_pixel != 0) {
+		wlr_log(WLR_DEBUG, "Invalid stride %d (incompatible with %d "
+			"bytes-per-pixel)", stride, bytes_per_pixel);
+		return false;
+	}
+	if (stride / bytes_per_pixel < width) {
+		wlr_log(WLR_DEBUG, "Invalid stride %d (too small for %d "
+			"bytes-per-pixel and width %d)", stride, bytes_per_pixel, width);
+		return false;
+	}
+	return true;
+}