diff --git a/sway/desktop/xwayland.c b/sway/desktop/xwayland.c
index 270cf08f..9a661405 100644
--- a/sway/desktop/xwayland.c
+++ b/sway/desktop/xwayland.c
@@ -289,7 +289,9 @@ static void set_activated(struct sway_view *view, bool activated) {
 	}
 
 	wlr_xwayland_surface_activate(surface, activated);
-	wlr_xwayland_surface_restack(surface, NULL, XCB_STACK_MODE_ABOVE);
+	if (activated) {
+		wlr_xwayland_surface_restack(surface, NULL, XCB_STACK_MODE_ABOVE);
+	}
 }
 
 static void set_tiled(struct sway_view *view, bool tiled) {