diff --git a/sway/desktop/xwayland.c b/sway/desktop/xwayland.c
index 55c42227..6447b711 100644
--- a/sway/desktop/xwayland.c
+++ b/sway/desktop/xwayland.c
@@ -79,14 +79,17 @@ static void unmanaged_handle_unmap(struct wl_listener *listener, void *data) {
 	wl_list_remove(&surface->commit.link);
 
 	if (!wlr_xwayland_surface_is_unmanaged(xsurface)) {
-		// Restore focus
 		struct sway_seat *seat = input_manager_current_seat(input_manager);
-		struct sway_container *previous =
-			seat_get_focus_inactive(seat, &root_container);
-		if (previous) {
-			// Hack to get seat to re-focus the return value of get_focus
-			seat_set_focus(seat, previous->parent);
-			seat_set_focus(seat, previous);
+		if (seat->wlr_seat->keyboard_state.focused_surface ==
+				xsurface->surface) {
+			// Restore focus
+			struct sway_container *previous =
+				seat_get_focus_inactive(seat, &root_container);
+			if (previous) {
+				// Hack to get seat to re-focus the return value of get_focus
+				seat_set_focus(seat, previous->parent);
+				seat_set_focus(seat, previous);
+			}
 		}
 	}
 }