diff --git a/sway/input/seat.c b/sway/input/seat.c
index 1dd99cb2..f4098eb0 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -159,14 +159,6 @@ static void handle_seat_node_destroy(struct wl_listener *listener, void *data) {
 		return;
 	}
 
-	if (node->type == N_CONTAINER &&
-			!workspace_is_visible(node->sway_container->workspace)) {
-		// Do not change the focus stack on a non-visible workspace. The focus
-		// will either be set when the workspace becomes visible or the
-		// workspace is now empty and will be destroyed
-		return;
-	}
-
 	// Find new focus_inactive (ie. sibling, or workspace if no siblings left)
 	struct sway_node *next_focus = NULL;
 	while (next_focus == NULL) {
@@ -182,6 +174,12 @@ static void handle_seat_node_destroy(struct wl_listener *listener, void *data) {
 		parent = node_get_parent(parent);
 	}
 
+	if (next_focus->type == N_WORKSPACE &&
+			!workspace_is_visible(next_focus->sway_workspace)) {
+		// Do not change focus to a non-visible workspace
+		return;
+	}
+
 	if (needs_new_focus) {
 		// The structure change might have caused it to move up to the top of
 		// the focus stack without sending focus notifications to the view