diff --git a/include/sway/server.h b/include/sway/server.h
index 0f30653f8..9242ceb7c 100644
--- a/include/sway/server.h
+++ b/include/sway/server.h
@@ -24,6 +24,7 @@ struct sway_server {
 	const char *socket;
 
 	struct wlr_backend *backend;
+	struct wlr_backend *noop_backend;
 
 	struct wlr_compositor *compositor;
 	struct wlr_data_device_manager *data_device_manager;
diff --git a/include/sway/tree/root.h b/include/sway/tree/root.h
index ceccc9209..8f4deaa7f 100644
--- a/include/sway/tree/root.h
+++ b/include/sway/tree/root.h
@@ -31,7 +31,9 @@ struct sway_root {
 
 	list_t *outputs; // struct sway_output
 	list_t *scratchpad; // struct sway_container
-	list_t *saved_workspaces; // For when there's no connected outputs
+
+	// For when there's no connected outputs
+	struct sway_output *noop_output;
 
 	struct {
 		struct wl_signal new_node;
diff --git a/sway/desktop/output.c b/sway/desktop/output.c
index edf77fbc5..f18a118f1 100644
--- a/sway/desktop/output.c
+++ b/sway/desktop/output.c
@@ -386,7 +386,7 @@ static void damage_handle_frame(struct wl_listener *listener, void *data) {
 void output_damage_whole(struct sway_output *output) {
 	// The output can exist with no wlr_output if it's just been disconnected
 	// and the transaction to evacuate it has't completed yet.
-	if (output && output->wlr_output) {
+	if (output && output->wlr_output && output->damage) {
 		wlr_output_damage_add_whole(output->damage);
 	}
 }
diff --git a/sway/server.c b/sway/server.c
index 5ae9bb01c..43dc39002 100644
--- a/sway/server.c
+++ b/sway/server.c
@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <wayland-server.h>
 #include <wlr/backend.h>
+#include <wlr/backend/noop.h>
 #include <wlr/backend/session.h>
 #include <wlr/render/wlr_renderer.h>
 #include <wlr/types/wlr_compositor.h>
@@ -25,6 +26,7 @@
 #include "sway/config.h"
 #include "sway/desktop/idle_inhibit_v1.h"
 #include "sway/input/input-manager.h"
+#include "sway/output.h"
 #include "sway/server.h"
 #include "sway/tree/root.h"
 #if HAVE_XWAYLAND
@@ -36,6 +38,7 @@ bool server_privileged_prepare(struct sway_server *server) {
 	server->wl_display = wl_display_create();
 	server->wl_event_loop = wl_display_get_event_loop(server->wl_display);
 	server->backend = wlr_backend_autocreate(server->wl_display, NULL);
+	server->noop_backend = wlr_noop_backend_create(server->wl_display);
 
 	if (!server->backend) {
 		sway_log(SWAY_ERROR, "Unable to create backend");
@@ -116,6 +119,9 @@ bool server_init(struct sway_server *server) {
 		return false;
 	}
 
+	struct wlr_output *wlr_output = wlr_noop_add_output(server->noop_backend);
+	root->noop_output = output_create(wlr_output);
+
 	// This may have been set already via -Dtxn-timeout
 	if (!server->txn_timeout_ms) {
 		server->txn_timeout_ms = 200;
diff --git a/sway/tree/output.c b/sway/tree/output.c
index 50a2c5352..5a992f2db 100644
--- a/sway/tree/output.c
+++ b/sway/tree/output.c
@@ -57,12 +57,12 @@ static void restore_workspaces(struct sway_output *output) {
 	}
 
 	// Saved workspaces
-	for (int i = 0; i < root->saved_workspaces->length; ++i) {
-		struct sway_workspace *ws = root->saved_workspaces->items[i];
+	while (root->noop_output->workspaces->length) {
+		struct sway_workspace *ws = root->noop_output->workspaces->items[0];
+		workspace_detach(ws);
 		output_add_workspace(output, ws);
 		ipc_event_workspace(NULL, ws, "move");
 	}
-	root->saved_workspaces->length = 0;
 
 	output_sort_workspaces(output);
 }
@@ -177,6 +177,9 @@ static void output_evacuate(struct sway_output *output) {
 		if (!new_output) {
 			new_output = fallback_output;
 		}
+		if (!new_output) {
+			new_output = root->noop_output;
+		}
 
 		if (workspace_is_empty(workspace)) {
 			// If floating is not empty, there are sticky containers to move
@@ -187,14 +190,10 @@ static void output_evacuate(struct sway_output *output) {
 			continue;
 		}
 
-		if (new_output) {
-			workspace_output_add_priority(workspace, new_output);
-			output_add_workspace(new_output, workspace);
-			output_sort_workspaces(new_output);
-			ipc_event_workspace(NULL, workspace, "move");
-		} else {
-			list_add(root->saved_workspaces, workspace);
-		}
+		workspace_output_add_priority(workspace, new_output);
+		output_add_workspace(new_output, workspace);
+		output_sort_workspaces(new_output);
+		ipc_event_workspace(NULL, workspace, "move");
 	}
 }
 
diff --git a/sway/tree/root.c b/sway/tree/root.c
index d4b97be31..ec6bccf62 100644
--- a/sway/tree/root.c
+++ b/sway/tree/root.c
@@ -39,7 +39,6 @@ struct sway_root *root_create(void) {
 	wl_signal_init(&root->events.new_node);
 	root->outputs = create_list();
 	root->scratchpad = create_list();
-	root->saved_workspaces = create_list();
 
 	root->output_layout_change.notify = output_layout_handle_change;
 	wl_signal_add(&root->output_layout->events.change,
@@ -50,7 +49,6 @@ struct sway_root *root_create(void) {
 void root_destroy(struct sway_root *root) {
 	wl_list_remove(&root->output_layout_change.link);
 	list_free(root->scratchpad);
-	list_free(root->saved_workspaces);
 	list_free(root->outputs);
 	wlr_output_layout_destroy(root->output_layout);
 	free(root);
@@ -327,8 +325,8 @@ void root_for_each_container(void (*f)(struct sway_container *con, void *data),
 	}
 
 	// Saved workspaces
-	for (int i = 0; i < root->saved_workspaces->length; ++i) {
-		struct sway_workspace *ws = root->saved_workspaces->items[i];
+	for (int i = 0; i < root->noop_output->workspaces->length; ++i) {
+		struct sway_workspace *ws = root->noop_output->workspaces->items[i];
 		workspace_for_each_container(ws, f, data);
 	}
 }
@@ -380,8 +378,8 @@ struct sway_container *root_find_container(
 	}
 
 	// Saved workspaces
-	for (int i = 0; i < root->saved_workspaces->length; ++i) {
-		struct sway_workspace *ws = root->saved_workspaces->items[i];
+	for (int i = 0; i < root->noop_output->workspaces->length; ++i) {
+		struct sway_workspace *ws = root->noop_output->workspaces->items[i];
 		if ((result = workspace_find_container(ws, test, data))) {
 			return result;
 		}
diff --git a/sway/tree/view.c b/sway/tree/view.c
index bc252521b..edbfca975 100644
--- a/sway/tree/view.c
+++ b/sway/tree/view.c
@@ -518,9 +518,10 @@ static struct sway_workspace *select_workspace(struct sway_view *view) {
 		return node->sway_container->workspace;
 	}
 
-	// If there's no focus_inactive workspace then we must be running without
-	// any outputs connected
-	return root->saved_workspaces->items[0];
+	// When there's no outputs connected, the above should match a workspace on
+	// the noop output.
+	sway_assert(false, "Expected to find a workspace");
+	return NULL;
 }
 
 static bool should_focus(struct sway_view *view) {
diff --git a/sway/tree/workspace.c b/sway/tree/workspace.c
index e89c08492..8b3eb2ad2 100644
--- a/sway/tree/workspace.c
+++ b/sway/tree/workspace.c
@@ -158,13 +158,7 @@ void workspace_begin_destroy(struct sway_workspace *workspace) {
 
 	if (workspace->output) {
 		workspace_detach(workspace);
-	} else {
-		int index = list_find(root->saved_workspaces, workspace);
-		if (index != -1) {
-			list_del(root->saved_workspaces, index);
-		}
 	}
-
 	workspace->node.destroying = true;
 	node_set_dirty(&workspace->node);
 }