From b36a955d207a47e4df9947bd54cb2d288213e38a Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Fri, 29 Apr 2016 17:33:05 -0700 Subject: [PATCH 1/5] GUACAMOLE-25: Use linear interpolation for resampling input audio. --- .../src/main/webapp/modules/AudioRecorder.js | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js index f9de4dc8b..f64e6eef6 100644 --- a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js +++ b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js @@ -183,6 +183,43 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { */ var maxSampleValue = (format.bytesPerSample === 1) ? 128 : 32768; + /** + * Determines the value of the waveform represented by the audio data at + * the given location. If the value cannot be determined exactly as it does + * not correspond to an exact sample within the audio data, the value will + * be derived through interpolating nearby samples. + * + * @param {Float32Array} audioData + * An array of audio data, as returned by AudioBuffer.getChannelData(). + * + * @param {Number} t + * The relative location within the waveform from which the value + * should be retrieved, represented as a floating point number between + * 0 and 1 inclusive, where 0 represents the earliest point in time and + * 1 represents the latest. + * + * @returns {Number} + * The value of the waveform at the given location. + */ + var interpolateSample = function getValueAt(audioData, t) { + + // Convert [0, 1] range to [0, audioData.length - 1] + var index = (audioData.length - 1) * t; + + // Get the closest whole integer samples indices + var left = Math.floor(index); + var right = Math.ceil(index); + + // Pull the values of the closest samples + var leftValue = audioData[left]; + var rightValue = audioData[right]; + + // Determine the value of the sample at the given non-integer location + // through linear interpolation of the nearest samples + return leftValue + (rightValue - leftValue) / (right - left) * (index - left); + + }; + /** * Converts the given AudioBuffer into an audio packet, ready for streaming * along the underlying output stream. Unlike the raw audio packets used by @@ -215,13 +252,8 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { // Fill array with data from audio buffer channel var offset = channel; for (var i = 0; i < outSamples; i++) { - - // Apply naiive resampling - var inOffset = Math.floor(i / outSamples * inSamples); - data[offset] = Math.floor(audioData[inOffset] * maxSampleValue); - + data[offset] = interpolateSample(audioData, i / (outSamples - 1)) * maxSampleValue; offset += format.channels; - } } From 8442f7c33f7afdb348c362536468c8d1258b3aab Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Fri, 29 Apr 2016 18:20:25 -0700 Subject: [PATCH 2/5] GUACAMOLE-25: Compensate for underflow/overflow induced by rounding error. --- .../src/main/webapp/modules/AudioRecorder.js | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js index f64e6eef6..704d7e756 100644 --- a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js +++ b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js @@ -183,6 +183,24 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { */ var maxSampleValue = (format.bytesPerSample === 1) ? 128 : 32768; + /** + * The total number of audio samples read from the local audio input device + * over the life of this audio recorder. + * + * @private + * @type {Number} + */ + var readSamples = 0; + + /** + * The total number of audio samples written to the underlying Guacamole + * connection over the life of this audio recorder. + * + * @private + * @type {Number} + */ + var writtenSamples = 0; + /** * Determines the value of the waveform represented by the audio data at * the given location. If the value cannot be determined exactly as it does @@ -237,9 +255,18 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { */ var toSampleArray = function toSampleArray(audioBuffer) { - // Calculate the number of samples in both input and output + // Track overall amount of data read var inSamples = audioBuffer.length; - var outSamples = Math.floor(audioBuffer.duration * format.rate); + readSamples += inSamples; + + // Calculate the total number of samples that should be written as of + // the audio data just received and adjust the size of the output + // packet accordingly + var expectedWrittenSamples = Math.round(readSamples * format.rate / audioBuffer.sampleRate); + var outSamples = expectedWrittenSamples - writtenSamples; + + // Update number of samples written + writtenSamples += outSamples; // Get array for raw PCM storage var data = new SampleArray(outSamples * format.channels); From 0c0ee96aaa18d70502f063e734d701597d11e800 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Fri, 29 Apr 2016 19:09:12 -0700 Subject: [PATCH 3/5] GUACAMOLE-25: Clean up media source and processor node on end. Keep reference while streaming (prevent faulty garbage collection of the nodes). --- .../src/main/webapp/modules/AudioRecorder.js | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js index 704d7e756..38e2cad47 100644 --- a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js +++ b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js @@ -201,6 +201,23 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { */ var writtenSamples = 0; + /** + * The source node providing access to the local audio input device. + * + * @private + * @type {MediaStreamAudioSourceNode} + */ + var source = null; + + /** + * The script processing node which receives audio input from the media + * stream source node as individual audio buffers. + * + * @private + * @type {ScriptProcessorNode} + */ + var processor = null; + /** * Determines the value of the waveform represented by the audio data at * the given location. If the value cannot be determined exactly as it does @@ -294,15 +311,29 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { // Abort stream if rejected if (status.code !== Guacamole.Status.Code.SUCCESS) { + + // Disconnect media source node from script processor + if (source) + source.disconnect(); + + // Disconnect associated script processor node + if (processor) + processor.disconnect(); + + // Remove references to now-unneeded components + processor = null; + source = null; + writer.sendEnd(); return; + } // Attempt to retrieve an audio input stream from the browser getUserMedia({ 'audio' : true }, function streamReceived(mediaStream) { // Create processing node which receives appropriately-sized audio buffers - var processor = context.createScriptProcessor(BUFFER_SIZE, format.channels, format.channels); + processor = context.createScriptProcessor(BUFFER_SIZE, format.channels, format.channels); processor.connect(context.destination); // Send blobs when audio buffers are received @@ -311,7 +342,7 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { }; // Connect processing node to user's audio input source - var source = context.createMediaStreamSource(mediaStream); + source = context.createMediaStreamSource(mediaStream); source.connect(processor); }, function streamDenied() { From c137312963aa084b54b46b8ffaa710971be5b24d Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Sun, 1 May 2016 00:23:15 -0700 Subject: [PATCH 4/5] GUACAMOLE-25: Migrate to Lanczos interpolation (a = 3). --- .../src/main/webapp/modules/AudioRecorder.js | 84 +++++++++++++++++-- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js index 38e2cad47..1f7c83d73 100644 --- a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js +++ b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js @@ -126,6 +126,17 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { */ var BUFFER_SIZE = 512; + /** + * The window size to use when applying Lanczos interpolation, commonly + * denoted by the variable "a". + * See: https://en.wikipedia.org/wiki/Lanczos_resampling + * + * @private + * @contant + * @type Number + */ + var LANCZOS_WINDOW_SIZE = 3; + /** * The format of audio this recorder will encode. * @@ -218,12 +229,65 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { */ var processor = null; + /** + * The normalized sinc function. The normalized sinc function is defined as + * 1 for x=0 and sin(PI * x) / (PI * x) for all other values of x. + * + * See: https://en.wikipedia.org/wiki/Sinc_function + * + * @private + * @param {Number} x + * The point at which the normalized sinc function should be computed. + * + * @returns {Number} + * The value of the normalized sinc function at x. + */ + var sinc = function sinc(x) { + + // The value of sinc(0) is defined as 1 + if (x === 0) + return 1; + + // Otherwise, normlized sinc(x) is sin(PI * x) / (PI * x) + var piX = Math.PI * x; + return Math.sin(piX) / piX; + + }; + + /** + * Calculates the value of the Lanczos kernal at point x for a given window + * size. See: https://en.wikipedia.org/wiki/Lanczos_resampling + * + * @private + * @param {Number} x + * The point at which the value of the Lanczos kernel should be + * computed. + * + * @param {Number} a + * The window size to use for the Lanczos kernel. + * + * @returns {Number} + * The value of the Lanczos kernel at the given point for the given + * window size. + */ + var lanczos = function lanczos(x, a) { + + // Lanczos is sinc(x) * sinc(x / a) for -a < x < a ... + if (-a < x && x < a) + return sinc(x) * sinc(x / a); + + // ... and 0 otherwise + return 0; + + }; + /** * Determines the value of the waveform represented by the audio data at * the given location. If the value cannot be determined exactly as it does * not correspond to an exact sample within the audio data, the value will * be derived through interpolating nearby samples. * + * @private * @param {Float32Array} audioData * An array of audio data, as returned by AudioBuffer.getChannelData(). * @@ -241,17 +305,19 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { // Convert [0, 1] range to [0, audioData.length - 1] var index = (audioData.length - 1) * t; - // Get the closest whole integer samples indices - var left = Math.floor(index); - var right = Math.ceil(index); + // Determine the start and end points for the summation used by the + // Lanczos interpolation algorithm (see: https://en.wikipedia.org/wiki/Lanczos_resampling) + var start = Math.floor(index) - LANCZOS_WINDOW_SIZE + 1; + var end = Math.floor(index) + LANCZOS_WINDOW_SIZE; - // Pull the values of the closest samples - var leftValue = audioData[left]; - var rightValue = audioData[right]; + // Calculate the value of the Lanczos interpolation function for the + // required range + var sum = 0; + for (var i = start; i <= end; i++) { + sum += (audioData[i] || 0) * lanczos(index - i, LANCZOS_WINDOW_SIZE); + } - // Determine the value of the sample at the given non-integer location - // through linear interpolation of the nearest samples - return leftValue + (rightValue - leftValue) / (right - left) * (index - left); + return sum; }; From efa6bf3c72000af95ae91a939442ab40205aa779 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 2 May 2016 01:48:16 -0700 Subject: [PATCH 5/5] GUACAMOLE-25: Increase audio recording buffer to 2048 bytes. --- guacamole-common-js/src/main/webapp/modules/AudioRecorder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js index 1f7c83d73..a75dbe102 100644 --- a/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js +++ b/guacamole-common-js/src/main/webapp/modules/AudioRecorder.js @@ -124,7 +124,7 @@ Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) { * @constant * @type {Number} */ - var BUFFER_SIZE = 512; + var BUFFER_SIZE = 2048; /** * The window size to use when applying Lanczos interpolation, commonly