diff --git a/sway/input/text_input.c b/sway/input/text_input.c
index c38a3bb2..ba8c0b99 100644
--- a/sway/input/text_input.c
+++ b/sway/input/text_input.c
@@ -66,11 +66,13 @@ static void handle_im_keyboard_grab_destroy(struct wl_listener *listener, void *
 	struct sway_input_method_relay *relay = wl_container_of(listener, relay,
 		input_method_keyboard_grab_destroy);
 	struct wlr_input_method_keyboard_grab_v2 *keyboard_grab = data;
+	struct wlr_seat *wlr_seat = keyboard_grab->input_method->seat;
 	wl_list_remove(&relay->input_method_keyboard_grab_destroy.link);
 
 	if (keyboard_grab->keyboard) {
 		// send modifier state to original client
-		wlr_seat_keyboard_notify_modifiers(keyboard_grab->input_method->seat,
+		wlr_seat_set_keyboard(wlr_seat, keyboard_grab->keyboard);
+		wlr_seat_keyboard_notify_modifiers(wlr_seat,
 			&keyboard_grab->keyboard->modifiers);
 	}
 }