3 files changed, 167 insertions, 163 deletions

diff --git a/include/wlr/types/wlr_xdg_shell_v6.h b/include/wlr/types/wlr_xdg_shell_v6.h
index e2a58669..0ab57108 100644
--- a/include/wlr/types/wlr_xdg_shell_v6.h
+++ b/include/wlr/types/wlr_xdg_shell_v6.h
@@ -302,18 +302,45 @@ struct wlr_xdg_surface_v6 *wlr_xdg_surface_v6_popup_at(
 		struct wlr_xdg_surface_v6 *surface, double sx, double sy,
 		double *popup_sx, double *popup_sy);
 
+/**
+ * Get the geometry for this positioner based on the anchor rect, gravity, and
+ * size of this positioner.
+ */
 struct wlr_box wlr_xdg_positioner_v6_get_geometry(
 		struct wlr_xdg_positioner_v6_attributes *positioner);
 
 /**
- * Get the anchor point for this popup in the root parent's coordinate system.
+ * Get the anchor point for this popup in the toplevel parent's coordinate system.
  */
 void wlr_xdg_popup_v6_get_anchor_point(struct wlr_xdg_popup_v6 *popup,
-		int *root_sx, int *root_sy);
+		int *toplevel_sx, int *toplevel_sy);
+
+/**
+ * Convert the given coordinates in the popup coordinate system to the toplevel
+ * surface coordinate system.
+ */
+void wlr_xdg_popup_v6_get_toplevel_coords(struct wlr_xdg_popup_v6 *popup,
+		int popup_sx, int popup_sy, int *toplevel_sx, int *toplevel_sy);
+
+/**
+ * Set the geometry of this popup to unconstrain it according to its
+ * xdg-positioner rules. The box should be in the popup's toplevel
+ * parent surface coordinate system.
+ */
+void wlr_xdg_popup_v6_unconstrain_from_box(struct wlr_xdg_popup_v6 *popup,
+		struct wlr_box *toplevel_box);
 
+/**
+  Invert the right/left anchor and gravity for this positioner. This can be
+  used to "flip" the positioner around the anchor rect in the x direction.
+ */
 void wlr_positioner_v6_invert_x(
 		struct wlr_xdg_positioner_v6_attributes *positioner);
 
+/**
+  Invert the top/bottom anchor and gravity for this positioner. This can be
+  used to "flip" the positioner around the anchor rect in the y direction.
+ */
 void wlr_positioner_v6_invert_y(
 		struct wlr_xdg_positioner_v6_attributes *positioner);
 
diff --git a/rootston/xdg_shell_v6.c b/rootston/xdg_shell_v6.c
index 9bcdb4c9..22309e8e 100644
--- a/rootston/xdg_shell_v6.c
+++ b/rootston/xdg_shell_v6.c
@@ -50,34 +50,17 @@ static void popup_handle_new_popup(struct wl_listener *listener, void *data) {
 	popup_create(popup->view_child.view, wlr_popup);
 }
 
-static void popup_get_coords(struct wlr_xdg_popup_v6 *popup,
-		double *sx, double *sy) {
-	struct wlr_xdg_surface_v6 *parent = popup->parent;
-	double popup_sx = popup->geometry.x;
-	double popup_sy = popup->geometry.y;
-	while (parent->role != WLR_XDG_SURFACE_V6_ROLE_TOPLEVEL) {
-		popup_sx += parent->popup->geometry.x;
-		popup_sy += parent->popup->geometry.y;
-		parent = parent->popup->parent;
-	}
-
-	*sx = popup_sx + parent->geometry.x;
-	*sy = popup_sy + parent->geometry.y;
-}
-
-static void popup_constraint_offset(struct roots_xdg_popup_v6 *popup,
-		int *offset_x, int *offset_y) {
+static void popup_unconstrain(struct roots_xdg_popup_v6 *popup) {
 	struct roots_view *view = popup->view_child.view;
 	struct wlr_output_layout *layout = view->desktop->layout;
 	struct wlr_xdg_popup_v6 *wlr_popup = popup->wlr_popup;
-	int popup_width = wlr_popup->geometry.width;
-	int popup_height = wlr_popup->geometry.height;
 
 	int anchor_lx, anchor_ly;
 	wlr_xdg_popup_v6_get_anchor_point(wlr_popup, &anchor_lx, &anchor_ly);
 
-	double popup_lx, popup_ly;
-	popup_get_coords(wlr_popup, &popup_lx, &popup_ly);
+	int popup_lx, popup_ly;
+	wlr_xdg_popup_v6_get_toplevel_coords(wlr_popup, wlr_popup->geometry.x,
+		wlr_popup->geometry.y, &popup_lx, &popup_ly);
 	popup_lx += view->x;
 	popup_ly += view->y;
 
@@ -93,108 +76,17 @@ static void popup_constraint_offset(struct roots_xdg_popup_v6 *popup,
 	// XXX: handle empty output layout
 	assert(output);
 
-	struct wlr_box *output_box = wlr_output_layout_get_box(layout, output);
-
-	*offset_x = *offset_y = 0;
-
-	if (popup_lx < output_box->x) {
-		*offset_x = output_box->x - popup_lx;
-	} else if (popup_lx + popup_width > output_box->x + output_box->width) {
-		*offset_x = output_box->x + output_box->width - (popup_lx + popup_width);
-	}
-
-	if (popup_ly < output_box->y) {
-		*offset_y = output_box->y - popup_ly;
-	} else if (popup_ly + popup_height > output_box->y + output_box->height) {
-		*offset_y = output_box->y + output_box->height - (popup_ly + popup_height);
-	}
-}
-
-static bool popup_unconstrain_flip(struct roots_xdg_popup_v6 *popup) {
-	int offset_x, offset_y;
-	popup_constraint_offset(popup, &offset_y, &offset_y);
-
-	if (!offset_x && !offset_y) {
-		return true;
-	}
-
-	if (offset_x) {
-		wlr_positioner_v6_invert_x(&popup->wlr_popup->positioner);
-	}
-	if (offset_y) {
-		wlr_positioner_v6_invert_y(&popup->wlr_popup->positioner);
-	}
-
-	popup->wlr_popup->geometry =
-		wlr_xdg_positioner_v6_get_geometry(&popup->wlr_popup->positioner);
-
-	popup_constraint_offset(popup, &offset_x, &offset_y);
-
-	if (!offset_x && !offset_y) {
-		// no longer constrained
-		return true;
-	}
-
-	// revert the positioner back if it didn't fix it and go to the next part
-	if (offset_x) {
-		wlr_positioner_v6_invert_x(&popup->wlr_popup->positioner);
-	}
-	if (offset_y) {
-		wlr_positioner_v6_invert_y(&popup->wlr_popup->positioner);
-	}
+	int width = 0, height = 0;
+	wlr_output_effective_resolution(output, &width, &height);
 
-	popup->wlr_popup->geometry =
-		wlr_xdg_positioner_v6_get_geometry(&popup->wlr_popup->positioner);
+	struct wlr_box toplevel_box = {
+		.x = output->lx - view->x,
+		.y = output->ly - view->y,
+		.width = width,
+		.height = height
+	};
 
-	return false;
-}
-
-static bool popup_unconstrain_slide(struct roots_xdg_popup_v6 *popup) {
-	int offset_x, offset_y;
-	popup_constraint_offset(popup, &offset_x, &offset_y);
-
-	if (!offset_x && !offset_y) {
-		return true;
-	}
-
-	if (offset_x) {
-		popup->wlr_popup->geometry.x += offset_x;
-	}
-
-	if (offset_y) {
-		popup->wlr_popup->geometry.y += offset_y;
-	}
-
-	popup_constraint_offset(popup, &offset_y, &offset_y);
-
-	return !offset_x && !offset_y;
-}
-
-static bool popup_unconstrain_resize(struct roots_xdg_popup_v6 *popup) {
-	int offset_x, offset_y;
-	popup_constraint_offset(popup, &offset_x, &offset_y);
-
-	if (!offset_x && !offset_y) {
-		return true;
-	}
-
-	if (offset_x) {
-		popup->wlr_popup->geometry.width -= offset_x;
-	}
-	if (offset_y) {
-		popup->wlr_popup->geometry.height -= offset_y;
-	}
-
-	popup_constraint_offset(popup, &offset_y, &offset_y);
-
-	return !offset_x && !offset_y;
-
-}
-
-static void popup_unconstrain(struct roots_xdg_popup_v6 *popup) {
-	popup_unconstrain_flip(popup);
-	popup_unconstrain_slide(popup);
-	popup_unconstrain_resize(popup);
+	wlr_xdg_popup_v6_unconstrain_from_box(popup->wlr_popup, &toplevel_box);
 }
 
 static struct roots_xdg_popup_v6 *popup_create(struct roots_view *view,
diff --git a/types/wlr_xdg_shell_v6.c b/types/wlr_xdg_shell_v6.c
index 0a8bbadf..0cf215ab 100644
--- a/types/wlr_xdg_shell_v6.c
+++ b/types/wlr_xdg_shell_v6.c
@@ -1674,48 +1674,133 @@ void wlr_xdg_popup_v6_get_anchor_point(struct wlr_xdg_popup_v6 *popup,
 
 	*root_sx = sx;
 	*root_sy = sy;
+}
 
-	/*
-	// XXX: THIS IS WILL WORK WITH XDG SHELL STABLE
-	switch (popup->positioner.anchor) {
-	case WLR_POSITIONER_ANCHOR_NONE:
-		sx = (rect.x + rect.width) / 2;
-		sy = (rect.y + rect.height) / 2;
-		break;
-	case WLR_POSITIONER_ANCHOR_TOP:
-		sx = (rect.x + rect.width) / 2;
-		sy = rect.y;
-		break;
-	case WLR_POSITIONER_ANCHOR_BOTTOM:
-		sx = (rect.x + rect.width) / 2;
-		sy = rect.y + rect.height;
-		break;
-	case WLR_POSITIONER_ANCHOR_LEFT:
-		sx = rect.x;
-		sy = (rect.y + rect.height) / 2;
-		break;
-	case WLR_POSITIONER_ANCHOR_RIGHT:
-		sx = rect.x + rect.width;
-		sy = (rect.y + rect.height) / 2;
-		break;
-	case WLR_POSITIONER_ANCHOR_TOP_LEFT:
-		sx = rect.x;
-		sy = rect.y;
-		break;
-	case WLR_POSITIONER_ANCHOR_BOTTOM_LEFT:
-		sx = rect.x;
-		sy = rect.y + rect.height;
-		break;
-	case WLR_POSITIONER_ANCHOR_TOP_RIGHT:
-		sx = rect.x + rect.width;
-		sy = rect.y;
-		break;
-	case WLR_POSITIONER_ANCHOR_BOTTOM_RIGHT:
-		sx = rect.x + rect.width;
-		sy = rect.y + rect.height;
-		break;
+void wlr_xdg_popup_v6_get_toplevel_coords(struct wlr_xdg_popup_v6 *popup,
+		int popup_sx, int popup_sy, int *toplevel_sx, int *toplevel_sy) {
+	struct wlr_xdg_surface_v6 *parent = popup->parent;
+	while (parent->role != WLR_XDG_SURFACE_V6_ROLE_TOPLEVEL) {
+		popup_sx += parent->popup->geometry.x;
+		popup_sy += parent->popup->geometry.y;
+		parent = parent->popup->parent;
+	}
+
+	*toplevel_sx = popup_sx + parent->geometry.x;
+	*toplevel_sy = popup_sy + parent->geometry.y;
+
+}
+
+static void wlr_xdg_popup_v6_box_constraints(struct wlr_xdg_popup_v6 *popup,
+		struct wlr_box *toplevel_box, int *offset_x, int *offset_y) {
+	int popup_width = popup->geometry.width;
+	int popup_height = popup->geometry.height;
+	int anchor_sx = 0, anchor_sy = 0;
+	wlr_xdg_popup_v6_get_anchor_point(popup, &anchor_sx, &anchor_sy);
+	int popup_sx = 0, popup_sy = 0;
+	wlr_xdg_popup_v6_get_toplevel_coords(popup, popup->geometry.x,
+		popup->geometry.y, &popup_sx, &popup_sy);
+	*offset_x = 0, *offset_y = 0;
+
+	if (popup_sx < toplevel_box->x) {
+		*offset_x = toplevel_box->x - popup_sx;
+	} else if (popup_sx + popup_width > toplevel_box->x + toplevel_box->width) {
+		*offset_x = toplevel_box->x + toplevel_box->width - (popup_sx + popup_width);
+	}
+
+	if (popup_sy < toplevel_box->y) {
+		*offset_y = toplevel_box->y - popup_sy;
+	} else if (popup_sy + popup_height > toplevel_box->y + toplevel_box->height) {
+		*offset_y = toplevel_box->y + toplevel_box->height - (popup_sy + popup_height);
+	}
+}
+
+static bool wlr_xdg_popup_v6_unconstrain_flip(struct wlr_xdg_popup_v6 *popup,
+		struct wlr_box *toplevel_box) {
+	int offset_x = 0, offset_y = 0;
+	wlr_xdg_popup_v6_box_constraints(popup, toplevel_box, &offset_y, &offset_y);
+
+	if (!offset_x && !offset_y) {
+		return true;
+	}
+
+	if (offset_x) {
+		wlr_positioner_v6_invert_x(&popup->positioner);
+	}
+	if (offset_y) {
+		wlr_positioner_v6_invert_y(&popup->positioner);
+	}
+
+	popup->geometry =
+		wlr_xdg_positioner_v6_get_geometry(&popup->positioner);
+
+	wlr_xdg_popup_v6_box_constraints(popup, toplevel_box, &offset_x, &offset_y);
+
+	if (!offset_x && !offset_y) {
+		// no longer constrained
+		return true;
 	}
-	*/
+
+	// revert the positioner back if it didn't fix it and go to the next part
+	if (offset_x) {
+		wlr_positioner_v6_invert_x(&popup->positioner);
+	}
+	if (offset_y) {
+		wlr_positioner_v6_invert_y(&popup->positioner);
+	}
+
+	popup->geometry =
+		wlr_xdg_positioner_v6_get_geometry(&popup->positioner);
+
+	return false;
+}
+
+static bool wlr_xdg_popup_v6_unconstrain_slide(struct wlr_xdg_popup_v6 *popup,
+		struct wlr_box *toplevel_box) {
+	int offset_x, offset_y;
+	wlr_xdg_popup_v6_box_constraints(popup, toplevel_box, &offset_x, &offset_y);
+
+	if (!offset_x && !offset_y) {
+		return true;
+	}
+
+	if (offset_x) {
+		popup->geometry.x += offset_x;
+	}
+
+	if (offset_y) {
+		popup->geometry.y += offset_y;
+	}
+
+	wlr_xdg_popup_v6_box_constraints(popup, toplevel_box, &offset_x, &offset_y);
+
+	return !offset_x && !offset_y;
+}
+
+static bool wlr_xdg_popup_v6_unconstrain_resize(struct wlr_xdg_popup_v6 *popup,
+		struct wlr_box *toplevel_box) {
+	int offset_x, offset_y;
+	wlr_xdg_popup_v6_box_constraints(popup, toplevel_box, &offset_x, &offset_y);
+
+	if (!offset_x && !offset_y) {
+		return true;
+	}
+
+	if (offset_x) {
+		popup->geometry.width -= offset_x;
+	}
+	if (offset_y) {
+		popup->geometry.height -= offset_y;
+	}
+
+	wlr_xdg_popup_v6_box_constraints(popup, toplevel_box, &offset_y, &offset_y);
+
+	return !offset_x && !offset_y;
+}
+
+void wlr_xdg_popup_v6_unconstrain_from_box(struct wlr_xdg_popup_v6 *popup,
+		struct wlr_box *toplevel_box) {
+	wlr_xdg_popup_v6_unconstrain_flip(popup, toplevel_box);
+	wlr_xdg_popup_v6_unconstrain_slide(popup, toplevel_box);
 }
 
 void wlr_positioner_v6_invert_x(struct wlr_xdg_positioner_v6_attributes *positioner) {