diff --git a/sway/desktop/layer_shell.c b/sway/desktop/layer_shell.c
index b60aa487..2d355b74 100644
--- a/sway/desktop/layer_shell.c
+++ b/sway/desktop/layer_shell.c
@@ -257,6 +257,11 @@ static void unmap(struct sway_layer_surface *sway_layer) {
 	}
 	output_damage_surface(output, sway_layer->geo.x, sway_layer->geo.y,
 		sway_layer->layer_surface->surface, true);
+
+	struct sway_seat *seat = input_manager_current_seat(input_manager);
+	if (seat->focused_layer == sway_layer->layer_surface) {
+		seat_set_focus_layer(seat, NULL);
+	}
 }
 
 static void handle_destroy(struct wl_listener *listener, void *data) {
diff --git a/sway/desktop/xwayland.c b/sway/desktop/xwayland.c
index 75bfb7b2..6447b711 100644
--- a/sway/desktop/xwayland.c
+++ b/sway/desktop/xwayland.c
@@ -77,6 +77,21 @@ static void unmanaged_handle_unmap(struct wl_listener *listener, void *data) {
 	desktop_damage_surface(xsurface->surface, xsurface->x, xsurface->y, true);
 	wl_list_remove(&surface->link);
 	wl_list_remove(&surface->commit.link);
+
+	if (!wlr_xwayland_surface_is_unmanaged(xsurface)) {
+		struct sway_seat *seat = input_manager_current_seat(input_manager);
+		if (seat->wlr_seat->keyboard_state.focused_surface ==
+				xsurface->surface) {
+			// Restore focus
+			struct sway_container *previous =
+				seat_get_focus_inactive(seat, &root_container);
+			if (previous) {
+				// Hack to get seat to re-focus the return value of get_focus
+				seat_set_focus(seat, previous->parent);
+				seat_set_focus(seat, previous);
+			}
+		}
+	}
 }
 
 static void unmanaged_handle_destroy(struct wl_listener *listener, void *data) {
diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index 16e5427b..d6e17ae1 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -264,7 +264,7 @@ void dispatch_cursor_button(struct sway_cursor *cursor,
 		if (new_ws != old_ws) {
 			seat_set_focus(cursor->seat, cont);
 		}
-	} else {
+	} else if (cont) {
 		seat_set_focus(cursor->seat, cont);
 	}
 
diff --git a/sway/input/seat.c b/sway/input/seat.c
index 0e539b70..071ef020 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -658,7 +658,8 @@ void seat_set_focus_layer(struct sway_seat *seat,
 		struct wlr_layer_surface *layer) {
 	if (!layer && seat->focused_layer) {
 		seat->focused_layer = NULL;
-		struct sway_container *previous = seat_get_focus(seat);
+		struct sway_container *previous =
+			seat_get_focus_inactive(seat, &root_container);
 		if (previous) {
 			wlr_log(L_DEBUG, "Returning focus to %p %s '%s'", previous,
 					container_type_to_str(previous->type), previous->name);