diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index 16e5427b..d6e17ae1 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -264,7 +264,7 @@ void dispatch_cursor_button(struct sway_cursor *cursor,
 		if (new_ws != old_ws) {
 			seat_set_focus(cursor->seat, cont);
 		}
-	} else {
+	} else if (cont) {
 		seat_set_focus(cursor->seat, cont);
 	}