diff --git a/src/modules/audio/audio.c b/src/modules/audio/audio.c
index 9105ae49..a9598024 100644
--- a/src/modules/audio/audio.c
+++ b/src/modules/audio/audio.c
@@ -95,6 +95,9 @@ void lovrAudioUpdate() {
         alSourcePlay(id);
       }
     } else if (isStopped) {
+      // in case we'll play this source in the future, rewind it now. This also frees up queued raw buffers.
+      lovrAudioStreamRewind(lovrSourceGetStream(source));
+
       arr_splice(&state.sources, i, 1);
       lovrRelease(Source, source);
     }
diff --git a/src/modules/audio/source.c b/src/modules/audio/source.c
index 04858537..7575aadc 100644
--- a/src/modules/audio/source.c
+++ b/src/modules/audio/source.c
@@ -165,11 +165,6 @@ void lovrSourcePlay(Source* source) {
     return;
   }
 
-  // in case we're replaying an already-used stream, make sure to rewind it if applicable
-  if (!lovrAudioStreamIsRaw(source->stream)) {
-    lovrAudioStreamRewind(source->stream);
-  }
-
   // in case we have some queued buffers, make sure to unqueue them before streaming more data into them.
   ALint processed;
   alGetSourcei(lovrSourceGetId(source), AL_BUFFERS_PROCESSED, &processed);
diff --git a/src/modules/data/audioStream.c b/src/modules/data/audioStream.c
index 422deec5..afcd7a48 100644
--- a/src/modules/data/audioStream.c
+++ b/src/modules/data/audioStream.c
@@ -129,9 +129,16 @@ bool lovrAudioStreamIsRaw(AudioStream* stream)
 }
 
 void lovrAudioStreamRewind(AudioStream* stream) {
-  lovrAssert(!lovrAudioStreamIsRaw(stream), "Can't rewind raw stream");
   stb_vorbis* decoder = (stb_vorbis*) stream->decoder;
-  stb_vorbis_seek_start(decoder);
+  if (decoder) {
+    stb_vorbis_seek_start(decoder);
+  } else {
+    stream->queueLengthInSamples = 0;
+    for (int i = 0; i < stream->queuedRawBuffers.length; i++) {
+      lovrRelease(Blob, stream->queuedRawBuffers.data[i]);
+    }
+    arr_clear(&stream->queuedRawBuffers);
+  }
 }
 
 void lovrAudioStreamSeek(AudioStream* stream, size_t sample) {