[fix] Handle auto layout resize with multiple slave groups

1 files changed, 73 insertions, 59 deletions

diff --git a/sway/commands/resize.c b/sway/commands/resize.c
index c850575b..9a756e81 100644
--- a/sway/commands/resize.c
+++ b/sway/commands/resize.c
@@ -63,35 +63,6 @@ static bool resize_floating(int amount, bool use_width) {
 }
 
 /**
- * returns the index of the container's child that is first in a group.
- * This index is > to the <after> argument.
- * This makes the function usable to walk through the groups in a container.
- */
-static int next_group_index(swayc_t *container, int after) {
-	if (after < 0) {
-		return 0;
-	} else if (is_auto_layout(container->layout)) {
-		if ((uint_fast32_t) after < container->nb_master) {
-			return container->nb_master;
-		} else {
-			uint_fast32_t grp_idx = 0;
-			for (int i = container->nb_master; i < container->children->length; ) {
-				uint_fast32_t grp_sz = (container->children->length - i) /
-					(container->nb_slave_groups - grp_idx);
-				if (after - i < (int) grp_sz) {
-					return i + grp_sz;
-				}
-				i += grp_sz;
-			}
-			return container->children->length;
-		}
-	} else {
-		//		return after + 1;
-		return container->children->length;
-	}
-}
-
-/**
  * Return the number of children in the slave groups. This corresponds to the children
  * that are not members of the master group.
  */
@@ -105,36 +76,46 @@ static inline uint_fast32_t slave_count(swayc_t *container) {
  * which index is a member of.
  */
 static int group_start_index(swayc_t *container, int index) {
-	if (index < 0 || ! is_auto_layout(container->layout) || (uint_fast32_t) index < container->nb_master) {
+	if ((index < 0) || (! is_auto_layout(container->layout)) ||
+	    ((uint_fast32_t) index < container->nb_master)) {
 		return 0;
 	} else {
 		uint_fast32_t grp_sz = slave_count(container) / container->nb_slave_groups;
 		uint_fast32_t remainder = slave_count(container) % container->nb_slave_groups;
-		if ((index - container->nb_master) / grp_sz < container->nb_slave_groups - remainder) {
-			return ((index - container->nb_master) / grp_sz) * grp_sz + container->nb_master;
+		int start_idx;
+		int idx2 = (container->nb_slave_groups - remainder) * grp_sz + container->nb_master;
+		if (index < idx2) {
+			start_idx = ((index - container->nb_master) / grp_sz) * grp_sz + container->nb_master;
 		} else {
-			int idx2 = (container->nb_slave_groups - remainder) * grp_sz + container->nb_master;
-			return idx2 + ((idx2 - index) / (grp_sz + 1)) * (grp_sz + 1);
+			start_idx = idx2 + ((index - idx2) / (grp_sz + 1)) * (grp_sz + 1);
 		}
+		return MIN(start_idx, container->children->length);
 	}
 }
 
 /**
  * given the index of a container's child, return the index of the first child of the group
  * that follows the one which index is a member of.
+ * This makes the function usable to walk through the groups in a container.
  */
 static int group_end_index(swayc_t *container, int index) {
 	if (index < 0 || ! is_auto_layout(container->layout)) {
 		return container->children->length;
 	} else {
-		uint_fast32_t grp_sz = slave_count(container) / container->nb_slave_groups;
-		uint_fast32_t remainder = slave_count(container) % container->nb_slave_groups;
-		if ((index - container->nb_master) / grp_sz < container->nb_slave_groups - remainder) {
-			return ((index - container->nb_master) / grp_sz + 1) * grp_sz + container->nb_master;
+		int nxt_idx;
+		if ((uint_fast32_t)index < container->nb_master) {
+			nxt_idx = container->nb_master;
 		} else {
+			uint_fast32_t grp_sz = slave_count(container) / container->nb_slave_groups;
+			uint_fast32_t remainder = slave_count(container) % container->nb_slave_groups;
 			int idx2 = (container->nb_slave_groups - remainder) * grp_sz + container->nb_master;
-			return idx2 + ((idx2 - index) / (grp_sz + 1) + 1) * (grp_sz + 1);
+			if (index < idx2) {
+				nxt_idx = ((index - container->nb_master) / grp_sz + 1) * grp_sz + container->nb_master;
+			} else {
+				nxt_idx = idx2 + ((index - idx2) / (grp_sz + 1) + 1) * (grp_sz + 1);
+			}
 		}
+		return MIN(nxt_idx, container->children->length);
 	}
 }
 
@@ -150,22 +131,26 @@ static inline uint_fast32_t group_count(swayc_t *container) {
  * The index is the order of the group along the container's major axis (starting at 0).
  */
 static uint_fast32_t group_index(swayc_t *container, int index) {
+	if (index < 0) {
+		return 0;
+	}
 	bool master_first = (container->layout == L_AUTO_LEFT || container->layout == L_AUTO_TOP);
 	int nb_slaves = slave_count(container);
-	if (index < (int) container->nb_master) {
+	if ((uint_fast32_t) index < container->nb_master) {
 		if (master_first || nb_slaves <= 0) {
 			return 0;
 		} else {
 			return MIN(container->nb_slave_groups, nb_slaves);
 		}
 	} else {
-		uint_fast32_t grp_idx = 0;
-		for (int i = container->nb_master; i < container->children->length; ) {
-			uint_fast32_t grp_sz = (container->children->length - i) /
-				(container->nb_slave_groups - grp_idx);
-			if (index - i < (int) grp_sz) {
-				break;
-			}
+		uint_fast32_t grp_sz = slave_count(container) / container->nb_slave_groups;
+		uint_fast32_t remainder = slave_count(container) % container->nb_slave_groups;
+		uint_fast32_t grp_idx;
+		int idx2 = (container->nb_slave_groups - remainder) * grp_sz + container->nb_master;
+		if (index < idx2) {
+			grp_idx = (index - container->nb_master) / grp_sz;
+		} else {
+			grp_idx = (container->nb_slave_groups - remainder) + (index - idx2) / (grp_sz + 1) ;
 		}
 		return grp_idx + (master_first ? 1 : 0);
 	}
@@ -204,7 +189,13 @@ static bool resize_tiled(int amount, bool use_width) {
 				nb_after = group_count(parent) - nb_before - 1;
 			} else {
 				nb_before = idx_focused - group_start_index(parent, idx_focused);
-				nb_after = next_group_index(parent, idx_focused) - idx_focused - 1;
+				nb_after = group_end_index(parent, idx_focused) - idx_focused - 1;
+				sway_log(L_DEBUG, "+++ focused: %d, start: %d, end: %d, before: %d, after: %d",
+					 idx_focused,
+					 (int)group_start_index(parent, idx_focused),
+					 (int)group_end_index(parent, idx_focused),
+					 (int)nb_before, (int)nb_after);
+
 			}
 			if (nb_before || nb_after) {
 				break;
@@ -223,6 +214,7 @@ static bool resize_tiled(int amount, bool use_width) {
 	swayc_t *focused = parent->children->items[idx_focused];
 	int start = use_major ? 0 : group_start_index(parent, idx_focused);
 	int end = use_major ? parent->children->length : group_end_index(parent, idx_focused);
+	sway_log(L_DEBUG, "Check children of container %p [%d,%d[", container, start, end);
 	for (int i = start; i < end; ) {
 		swayc_t *sibling = parent->children->items[i];
 		double pixels = amount;
@@ -235,17 +227,21 @@ static bool resize_tiled(int amount, bool use_width) {
 				pixels /= 2;
 			}
 		}
+		sway_log(L_DEBUG, "Check container %p: width %g vs %d, height %g vs %d", sibling, sibling->width + pixels, min_sane_w, sibling->height + pixels, min_sane_h);
 		if (use_width ?
 		    sibling->width + pixels < min_sane_w :
 		    sibling->height + pixels < min_sane_h) {
 			valid = false;
+			sway_log(L_DEBUG, "Container size no longer sane");
 			break;
 		}
-		i = use_major ? next_group_index(parent, i) : (i + 1);
+		i = use_major ? group_end_index(parent, i) : (i + 1);
+		sway_log(L_DEBUG, "+++++ check %i", i);
 	}
 	// 3. Apply the size change
 	if (valid) {
-		for (int i = 0; i < parent->children->length; ++i) {
+		for (int i = start; i < end; ) {
+			int next_i = use_major ? group_end_index(parent, i) : (i + 1);
 			swayc_t *sibling = parent->children->items[i];
 			double pixels = amount;
 			bool is_before = use_width ? sibling->x < focused->x : sibling->y < focused->y;
@@ -257,17 +253,35 @@ static bool resize_tiled(int amount, bool use_width) {
 					pixels /= 2;
 				}
 				sway_log(L_DEBUG, "%p: %s", sibling, is_before ? "before" : "after");
-				recursive_resize(sibling, pixels,
-						 use_width ?
-						 (is_before ? WLC_RESIZE_EDGE_RIGHT : WLC_RESIZE_EDGE_LEFT) :
-						 (is_before ? WLC_RESIZE_EDGE_BOTTOM : WLC_RESIZE_EDGE_TOP));
+				if (use_major) {
+					for (int j = i; j < next_i; ++j) {
+						recursive_resize(parent->children->items[j], pixels,
+								 use_width ?
+								 (is_before ? WLC_RESIZE_EDGE_RIGHT : WLC_RESIZE_EDGE_LEFT) :
+								 (is_before ? WLC_RESIZE_EDGE_BOTTOM : WLC_RESIZE_EDGE_TOP));
+					}
+				} else {
+					recursive_resize(sibling, pixels,
+							use_width ?
+							(is_before ? WLC_RESIZE_EDGE_RIGHT : WLC_RESIZE_EDGE_LEFT) :
+							(is_before ? WLC_RESIZE_EDGE_BOTTOM : WLC_RESIZE_EDGE_TOP));
+				}
 			} else {
-				sway_log(L_DEBUG, "%p: same pos", sibling);
-				recursive_resize(sibling, pixels,
-						 use_width ? WLC_RESIZE_EDGE_LEFT : WLC_RESIZE_EDGE_TOP);
-				recursive_resize(sibling, pixels,
-						 use_width ? WLC_RESIZE_EDGE_RIGHT : WLC_RESIZE_EDGE_BOTTOM);
+				if (use_major) {
+					for (int j = i; j < next_i; ++j) {
+						recursive_resize(parent->children->items[j], pixels,
+								 use_width ? WLC_RESIZE_EDGE_LEFT : WLC_RESIZE_EDGE_TOP);
+						recursive_resize(parent->children->items[j], pixels,
+								 use_width ? WLC_RESIZE_EDGE_RIGHT : WLC_RESIZE_EDGE_BOTTOM);
+					}
+				} else {
+					recursive_resize(sibling, pixels,
+							 use_width ? WLC_RESIZE_EDGE_LEFT : WLC_RESIZE_EDGE_TOP);
+					recursive_resize(sibling, pixels,
+							 use_width ? WLC_RESIZE_EDGE_RIGHT : WLC_RESIZE_EDGE_BOTTOM);
+				}
 			}
+			i = next_i;
 		}
 		// Recursive resize does not handle positions, let arrange_windows
 		// take care of that.