Skip to content

Commit

Permalink
Overhauled Web audio with AudioWorklet
Browse files Browse the repository at this point in the history
  • Loading branch information
chrisd1100 committed May 21, 2023
1 parent b602a68 commit 87bc841
Show file tree
Hide file tree
Showing 2 changed files with 166 additions and 79 deletions.
201 changes: 125 additions & 76 deletions src/unix/web/matoya-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,112 +291,156 @@ const MTY_GL_API = {

// Audio

function mty_audio_queued_ms() {
let queued_ms = Math.round((MTY.audio.next_time - MTY.audio.ctx.currentTime) * 1000.0);
let buffered_ms = Math.round((MTY.audio.offset / 4) / MTY.audio.frames_per_ms);
function mty_mutex_lock_nb(mutex, index)
{
return Atomics.compareExchange(mutex, index, 0, 1) == 0;
}

function mty_mutex_lock(mutex, index) {
while (true) {
if (mty_mutex_lock_nb(mutex, index))
return;

return (queued_ms < 0 ? 0 : queued_ms) + buffered_ms;
Atomics.wait(mutex, index, 1);
}
}

const MTY_AUDIO_API = {
MTY_AudioCreate: function (sampleRate, minBuffer, maxBuffer, channels, deviceID, fallback) {
MTY.audio = {};
MTY.audio.flushing = false;
MTY.audio.playing = false;
MTY.audio.sample_rate = sampleRate;
MTY.audio.channels = channels;
function mty_mutex_unlock(mutex, index, notify) {
Atomics.compareExchange(mutex, index, 1, 0);

MTY.audio.frames_per_ms = Math.round(sampleRate / 1000.0);
MTY.audio.min_buffer = minBuffer * MTY.audio.frames_per_ms;
MTY.audio.max_buffer = maxBuffer * MTY.audio.frames_per_ms;
if (notify)
Atomics.notify(mutex, index, 1);
}

MTY.audio.offset = 0;
MTY.audio.buf = mty_alloc(sampleRate * 2 * MTY.audio.channels);
const MTY_AUDIO_API = {
MTY_AudioCreate: function (sampleRate, minBuffer, maxBuffer, channels, deviceID, fallback) {
MTY.audio = {
sampleRate,
minBuffer,
maxBuffer,
channels,
};

return 0xCDD;
},
MTY_AudioDestroy: function (audio) {
if (!MTY.audio)
if (!audio || !mty_get_uint32(audio))
return;

mty_free(MTY.audio.buf);
postMessage({type: 'audio-destroy'});
mty_set_uint32(audio, 0);
MTY.audio = null;
},
MTY_AudioQueue: function (ctx, frames, count) {
// Initialize on first queue otherwise the browser may complain about user interaction
if (!MTY.audio.ctx)
MTY.audio.ctx = new AudioContext();
const buf = new Int16Array(MTY_MEMORY.buffer, frames, count * MTY.audio.channels);

let queued_frames = MTY.audio.frames_per_ms * mty_audio_queued_ms();
mty_mutex_lock(MTY.audioObjs.control, 0);

// Stop playing and flush if we've exceeded the maximum buffer
if (queued_frames > MTY.audio.max_buffer) {
MTY.audio.playing = false;
MTY.audio.flushing = true;
if (buf.length <= MTY.audioObjs.buf.length - MTY.audioObjs.control[1]) {
MTY.audioObjs.buf.set(buf, MTY.audioObjs.control[1]);
MTY.audioObjs.control[1] += buf.length;
}

// Stop flushing when the queue reaches zero
if (queued_frames == 0) {
MTY.audio.flushing = false;
MTY.audio.playing = false;
}
mty_mutex_unlock(MTY.audioObjs.control, 0, false);

// Convert PCM int16_t to float
if (!MTY.audio.flushing) {
let size = count * 2 * MTY.audio.channels;
mty_memcpy(MTY.audio.buf + MTY.audio.offset, new Uint8Array(MTY_MEMORY.buffer, frames, size));
MTY.audio.offset += size;
}
postMessage({type: 'audio-queue', ...MTY.audio});
},
MTY_AudioReset: function (ctx) {
mty_mutex_lock(MTY.audioObjs.control, 0);

MTY.audioObjs.control[1] = 0;

mty_mutex_unlock(MTY.audioObjs.control, 0, false);
},
MTY_AudioGetQueued: function (ctx) {
return Atomics.load(MTY.audioObjs.control, 2);
},
};


// Audio Worklet

if (typeof AudioWorkletGlobalScope != 'undefined') {

function mty_int16_to_float(i) {
return i < 0 ? i / 32768 : i / 32767;
}

class MTY_Audio extends AudioWorkletProcessor {
constructor(options) {
super();

const frames_per_ms = Math.round(sampleRate / 1000.0);

this.minBuffer = Math.round(options.processorOptions.minBuffer * frames_per_ms);
this.maxBuffer = Math.round(options.processorOptions.maxBuffer * frames_per_ms);
this.channels = options.outputChannelCount[0];
this.playing = false;

// Begin playing again if the buffer has accumulated past the min
if (!MTY.audio.playing && !MTY.audio.flushing &&
MTY.audio.offset / (2 * MTY.audio.channels) > MTY.audio.min_buffer)
{
MTY.audio.next_time = MTY.audio.ctx.currentTime;
MTY.audio.playing = true;
this.ibuf = new Int16Array(new ArrayBuffer(1024 * 1024));
this.ibufLen = 0;

this.port.onmessage = (evt) => {
this.sbuf = evt.data.buf;
this.control = evt.data.control;
};
}

process(inputs, outputs, parameters) {
// Copy from staging buffer to internal buffer
if (mty_mutex_lock_nb(this.control, 0)) {
if (this.control[1] <= this.ibuf.length - this.ibufLen) {
this.ibuf.set(new Int16Array(this.sbuf.buffer, 0, this.control[1]), this.ibufLen);
this.ibufLen += this.control[1];
this.control[1] = 0;
}

mty_mutex_unlock(this.control, 0, true);
}

// Queue the audio if playing
if (MTY.audio.playing) {
const src = new Int16Array(MTY_MEMORY.buffer, MTY.audio.buf);
const bcount = MTY.audio.offset / (2 * MTY.audio.channels);
let queued = this.ibufLen / this.channels;

const buf = MTY.audio.ctx.createBuffer(MTY.audio.channels, bcount, MTY.audio.sample_rate);
// Queued audio has reached the min buffer, begin playing
if (!this.playing && queued >= this.minBuffer)
this.playing = true;

const chans = [];
for (let x = 0; x < MTY.audio.channels; x++)
chans[x] = buf.getChannelData(x);
// No audio left, pause and let buffer refill
if (this.playing && this.ibufLen == 0)
this.playing = false;

let offset = 0;
for (let x = 0; x < bcount * MTY.audio.channels; x += MTY.audio.channels) {
for (y = 0; y < MTY.audio.channels; y++) {
chans[y][offset] = src[x + y] / 32768;
offset++;
}
// Fill output with buffered audio
if (this.playing) {
const l = outputs[0][0];
const r = outputs[0][1];

let x = 0;
for (let o = 0; x < this.ibufLen && o < l.length && o < r.length; x += this.channels, o++) {
l[o] = mty_int16_to_float(this.ibuf[x]);
r[o] = mty_int16_to_float(this.ibuf[x + 1]);
}

const source = MTY.audio.ctx.createBufferSource();
source.buffer = buf;
source.connect(MTY.audio.ctx.destination);
source.start(MTY.audio.next_time);
// Essentially a 'memmove' to bring remaining audio to front of the buffer
this.ibufLen -= x;
this.ibuf.set(new Int16Array(this.ibuf.buffer, x * 2, this.ibufLen));
}

queued = this.ibufLen / this.channels;

MTY.audio.next_time += buf.duration;
MTY.audio.offset = 0;
// If buffer has exceeded the max, reset
if (this.playing && queued > this.maxBuffer) {
this.playing = false;
this.ibufLen = 0;
}
},
MTY_AudioReset: function (ctx) {
MTY.audio.playing = false;
MTY.audio.flushing = false;
MTY.audio.offset = 0;
},
MTY_AudioGetQueued: function (ctx) {
if (MTY.audio.ctx)
return mty_audio_queued_ms();

return 0;
},
};
// Store queued frames
Atomics.store(this.control, 2, queued);

return true;
}
}

registerProcessor('MTY_Audio', MTY_Audio);

}


// Net
Expand Down Expand Up @@ -1021,6 +1065,8 @@ const MTY_WASI_API = {

// Entry

if (typeof WorkerGlobalScope != 'undefined') {

async function mty_instantiate_wasm(wasmBuf, userEnv) {
// Imports
const imports = {
Expand Down Expand Up @@ -1084,6 +1130,7 @@ onmessage = async (ev) => {
MTY.fdIndex = 64;
MTY.kbMap = msg.kbMap;
MTY.psync = msg.psync;
MTY.audioObjs = msg.audioObjs;
MTY.initWindowInfo = msg.windowInfo;
MTY.sync = new Int32Array(new SharedArrayBuffer(4));
MTY.sleeper = new Int32Array(new SharedArrayBuffer(4));
Expand Down Expand Up @@ -1189,3 +1236,5 @@ onmessage = async (ev) => {
}
}
};

}
44 changes: 41 additions & 3 deletions src/unix/web/matoya.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,30 @@ function mty_poll_gamepads() {
}


// Audio

async function mty_audio_queue(ctx, sampleRate, minBuffer, maxBuffer, channels) {
// Initialize on first queue otherwise the browser may complain about user interaction
if (!MTY.audioCtx) {
MTY.audioCtx = new AudioContext({sampleRate: sampleRate});

const baseFile = MTY_CURRENT_SCRIPT.pathname;
await MTY.audioCtx.audioWorklet.addModule(baseFile.replace('.js', '-worker.js'));

const node = new AudioWorkletNode(MTY.audioCtx, 'MTY_Audio', {
outputChannelCount: [channels],
processorOptions: {
minBuffer,
maxBuffer,
},
});

node.connect(MTY.audioCtx.destination);
node.port.postMessage(MTY.audioObjs);
}
}


// Image

async function mty_decode_image(input) {
Expand Down Expand Up @@ -782,7 +806,7 @@ function mty_update_interval(thread) {
});
}

function mty_thread_start(threadId, bin, wasmBuf, memory, startArg, userEnv, kbMap, psync, name) {
function mty_thread_start(threadId, bin, wasmBuf, memory, startArg, userEnv, kbMap, psync, audioObjs, name) {
const baseFile = MTY_CURRENT_SCRIPT.pathname;
const worker = new Worker(baseFile.replace('.js', '-worker.js'), {name: name});

Expand All @@ -802,6 +826,7 @@ function mty_thread_start(threadId, bin, wasmBuf, memory, startArg, userEnv, kbM
startArg: startArg,
threadId: threadId,
memory: memory,
audioObjs,
});

return worker;
Expand All @@ -814,6 +839,10 @@ async function MTY_Start(bin, container, userEnv) {
MTY.bin = bin;
MTY.userEnv = userEnv;
MTY.psync = new Int32Array(new SharedArrayBuffer(4));
MTY.audioObjs = {
buf: new Int16Array(new SharedArrayBuffer(1024 * 1024)),
control: new Int32Array(new SharedArrayBuffer(32)),
};

// Drawing surface
MTY.canvas = document.createElement('canvas');
Expand Down Expand Up @@ -846,7 +875,7 @@ async function MTY_Start(bin, container, userEnv) {

// Main thread
MTY.mainThread = mty_thread_start(MTY.threadId, bin, MTY.wasmBuf, MTY_MEMORY,
0, userEnv, MTY.kbMap, MTY.psync, 'main');
0, userEnv, MTY.kbMap, MTY.psync, MTY.audioObjs, 'main');

// Init position, update loop
MTY.posX = window.screenX;
Expand Down Expand Up @@ -880,7 +909,7 @@ async function mty_thread_message(ev) {
MTY.threadId++;

const worker = mty_thread_start(MTY.threadId, MTY.bin, MTY.wasmBuf, MTY_MEMORY,
msg.startArg, MTY.userEnv, MTY.kbMap, MTY.psync, 'thread-' + MTY.threadId);
msg.startArg, MTY.userEnv, MTY.kbMap, MTY.psync, MTY.audioObjs, 'thread-' + MTY.threadId);

msg.sab[0] = MTY.threadId;
mty_signal(msg.sync);
Expand Down Expand Up @@ -1039,6 +1068,15 @@ async function mty_thread_message(ev) {
mty_signal(msg.sync);
break;
}
case 'audio-queue':
mty_audio_queue(MTY.audio, msg.sampleRate, msg.minBuffer,
msg.maxBuffer, msg.channels);
break;
case 'audio-destroy':
if (MTY.audioCtx)
MTY.audioCtx.close();
delete MTY.audioCtx;
break;
case 'async-copy':
msg.sab8.set(this.tmp);
delete this.tmp;
Expand Down

0 comments on commit 87bc841

Please sign in to comment.