diff --git a/include/sway/input/seat.h b/include/sway/input/seat.h
index e3a468726..4abe91f71 100644
--- a/include/sway/input/seat.h
+++ b/include/sway/input/seat.h
@@ -72,6 +72,7 @@ struct sway_drag_icon {
 	struct wl_list link; // sway_root::drag_icons
 
 	double x, y; // in layout-local coordinates
+	int dx, dy; // offset in surface-local coordinates
 
 	struct wl_listener surface_commit;
 	struct wl_listener map;
diff --git a/sway/input/seat.c b/sway/input/seat.c
index 43b207799..c263eb825 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -382,8 +382,8 @@ void drag_icon_update_position(struct sway_drag_icon *icon) {
 	case WLR_DRAG_GRAB_KEYBOARD:
 		return;
 	case WLR_DRAG_GRAB_KEYBOARD_POINTER:
-		icon->x = cursor->x + wlr_icon->surface->sx;
-		icon->y = cursor->y + wlr_icon->surface->sy;
+		icon->x = cursor->x + icon->dx;
+		icon->y = cursor->y + icon->dy;
 		break;
 	case WLR_DRAG_GRAB_KEYBOARD_TOUCH:;
 		struct wlr_touch_point *point =
@@ -391,8 +391,8 @@ void drag_icon_update_position(struct sway_drag_icon *icon) {
 		if (point == NULL) {
 			return;
 		}
-		icon->x = seat->touch_x + wlr_icon->surface->sx;
-		icon->y = seat->touch_y + wlr_icon->surface->sy;
+		icon->x = seat->touch_x + icon->dx;
+		icon->y = seat->touch_y + icon->dy;
 	}
 
 	drag_icon_damage_whole(icon);
@@ -402,6 +402,9 @@ static void drag_icon_handle_surface_commit(struct wl_listener *listener,
 		void *data) {
 	struct sway_drag_icon *icon =
 		wl_container_of(listener, icon, surface_commit);
+	struct wlr_drag_icon *wlr_icon = icon->wlr_drag_icon;
+	icon->dx += wlr_icon->surface->current.dx;
+	icon->dy += wlr_icon->surface->current.dy;
 	drag_icon_update_position(icon);
 }