GUAC-1354: Dynamically split and reassemble audio packets to minimize clicking.

guacamole-common-js/src/main/webapp/modules/AudioPlayer.js

@@ -180,6 +180,18 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
      */
     var reader = new Guacamole.ArrayBufferReader(stream);
 
+    /**
+     * The minimum size of an audio packet split by splitAudioPacket(), in
+     * seconds. Audio packets smaller than this will not be split, nor will the
+     * split result of a larger packet ever be smaller in size than this
+     * minimum.
+     *
+     * @private
+     * @constant
+     * @type Number
+     */
+    var MIN_SPLIT_SIZE = 0.02;
+
     /**
      * The maximum amount of latency to allow between the buffered data stream
      * and the playback position, in seconds. Initially, this is set to
@@ -190,32 +202,209 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
      */
     var maxLatency = 0.3;
 
-    // Play each received raw packet of audio immediately
+    /**
-    reader.ondata = function playReceivedAudio(data) {
+     * The type of typed array that will be used to represent each audio packet
+     * internally. This will be either Int8Array or Int16Array, depending on
+     * whether the raw audio format is 8-bit or 16-bit.
+     *
+     * @private
+     * @constructor
+     */
+    var SampleArray = (format.bytesPerSample === 1) ? window.Int8Array : window.Int16Array;
+
+    /**
+     * The maximum absolute value of any sample within a raw audio packet
+     * received by this audio player. This depends only on the size of each
+     * sample, and will be 128 for 8-bit audio and 32768 for 16-bit audio.
+     *
+     * @private
+     * @type Number
+     */
+    var maxSampleValue = (format.bytesPerSample === 1) ? 128 : 32768;
+
+    /**
+     * The queue of all pending audio packets, as an array of sample arrays.
+     * Audio packets which are pending playback will be added to this queue for
+     * further manipulation prior to scheduling via the Web Audio API. Once an
+     * audio packet leaves this queue and is scheduled via the Web Audio API,
+     * no further modifications can be made to that packet.
+     *
+     * @private
+     * @type SampleArray[]
+     */
+    var packetQueue = [];
+
+    /**
+     * Given an array of audio packets, returns a single audio packet
+     * containing the concatenation of those packets.
+     *
+     * @private
+     * @param {SampleArray[]} packets
+     *     The array of audio packets to concatenate.
+     *
+     * @returns {SampleArray}
+     *     A single audio packet containing the concatenation of all given
+     *     audio packets. If no packets are provided, this will be undefined.
+     */
+    var joinAudioPackets = function joinAudioPackets(packets) {
+
+        // Do not bother joining if one or fewer packets are in the queue
+        if (packets.length <= 1)
+            return packets[0];
+
+        // Determine total sample length of the entire queue
+        var totalLength = 0;
+        packets.forEach(function addPacketLengths(packet) {
+            totalLength += packet.length;
+        });
+
+        // Append each packet within queue
+        var offset = 0;
+        var joined = new SampleArray(totalLength);
+        packets.forEach(function appendPacket(packet) {
+            joined.set(packet, offset);
+            offset += packet.length;
+        });
+
+        return joined;
+
+    };
+
+    /**
+     * Given a single packet of audio data, splits off an arbitrary length of
+     * audio data from the beginning of that packet, returning the split result
+     * as an array of two packets. The split location is determined through an
+     * algorithm intended to minimize the liklihood of audible clicking between
+     * packets. If no such split location is possible, an array containing only
+     * the originally-provided audio packet is returned.
+     *
+     * @private
+     * @param {SampleArray} data
+     *     The audio packet to split.
+     *
+     * @returns {SampleArray[]}
+     *     An array of audio packets containing the result of splitting the
+     *     provided audio packet. If splitting is possible, this array will
+     *     contain two packets. If splitting is not possible, this array will
+     *     contain only the originally-provided packet.
+     */
+    var splitAudioPacket = function splitAudioPacket(data) {
+
+        var minValue = Number.MAX_VALUE;
+        var optimalSplitLength = data.length;
+
+        // Calculate number of whole samples in the provided audio packet AND
+        // in the minimum possible split packet
+        var samples = Math.floor(data.length / format.channels);
+        var minSplitSamples = Math.floor(format.rate * MIN_SPLIT_SIZE);
+
+        // Calculate the beginning of the "end" of the audio packet
+        var start = Math.max(
+            format.channels * minSplitSamples,
+            format.channels * (samples - minSplitSamples)
+        );
+
+        // For all samples at the end of the given packet, find a point where
+        // the perceptible volume across all channels is lowest (and thus is
+        // the optimal point to split)
+        for (var offset = start; offset < data.length; offset += format.channels) {
+
+            // Calculate the sum of all values across all channels (the result
+            // will be proportional to the average volume of a sample)
+            var totalValue = 0;
+            for (var channel = 0; channel < format.channels; channel++) {
+                totalValue += Math.abs(data[offset + channel]);
+            }
+
+            // If this is the smallest average value thus far, set the split
+            // length such that the first packet ends with the current sample
+            if (totalValue <= minValue) {
+                optimalSplitLength = offset + format.channels;
+                minValue = totalValue;
+            }
+
+        }
+
+        // If packet is not split, return the supplied packet untouched
+        if (optimalSplitLength === data.length)
+            return [data];
+
+        // Otherwise, split the packet into two new packets according to the
+        // calculated optimal split length
+        return [
+            data.slice(0, optimalSplitLength),
+            data.slice(optimalSplitLength)
+        ];
+
+    };
+
+    /**
+     * Pushes the given packet of audio data onto the playback queue. Unlike
+     * other private functions within Guacamole.RawAudioPlayer, the type of the
+     * ArrayBuffer packet of audio data here need not be specific to the type
+     * of audio (as with SampleArray). The ArrayBuffer type provided by a
+     * Guacamole.ArrayBufferReader, for example, is sufficient. Any necessary
+     * conversions will be performed automatically internally.
+     *
+     * @private
+     * @param {ArrayBuffer} data
+     *     A raw packet of audio data that should be pushed onto the audio
+     *     playback queue.
+     */
+    var pushAudioPacket = function pushAudioPacket(data) {
+        packetQueue.push(new SampleArray(data));
+    };
+
+    /**
+     * Shifts off and returns a packet of audio data from the beginning of the
+     * playback queue. The length of this audio packet is determined
+     * dynamically according to the click-reduction algorithm implemented by
+     * splitAudioPacket().
+     *
+     * @returns {SampleArray}
+     *     A packet of audio data pulled from the beginning of the playback
+     *     queue.
+     */
+    var shiftAudioPacket = function shiftAudioPacket() {
+
+        // Flatten data in packet queue
+        var data = joinAudioPackets(packetQueue);
+        if (!data)
+            return null;
+
+        // Pull an appropriate amount of data from the front of the queue
+        packetQueue = splitAudioPacket(data);
+        data = packetQueue.shift();
+
+        return data;
+
+    };
+
+    /**
+     * Converts the given audio packet into an AudioBuffer, ready for playback
+     * by the Web Audio API. Unlike the raw audio packets received by this
+     * audio player, AudioBuffers require floating point samples and are split
+     * into isolated planes of channel-specific data.
+     *
+     * @private
+     * @param {SampleArray} data
+     *     The raw audio packet that should be converted into a Web Audio API
+     *     AudioBuffer.
+     *
+     * @returns {AudioBuffer}
+     *     A new Web Audio API AudioBuffer containing the provided audio data,
+     *     converted to the format used by the Web Audio API.
+     */
+    var toAudioBuffer = function toAudioBuffer(data) {
 
         // Calculate total number of samples
-        var samples = data.byteLength / format.channels / format.bytesPerSample;
+        var samples = data.length / format.channels;
-
-        // Calculate overall duration (in seconds)
-        var duration = samples / format.rate;
 
         // Determine exactly when packet CAN play
         var packetTime = context.currentTime;
         if (nextPacketTime < packetTime)
             nextPacketTime = packetTime;
 
-        // Obtain typed array view based on defined bytes per sample
-        var maxValue;
-        var source;
-        if (format.bytesPerSample === 1) {
-            source = new Int8Array(data);
-            maxValue = 128;
-        }
-        else {
-            source = new Int16Array(data);
-            maxValue = 32768;
-        }
-
         // Get audio buffer for specified format
         var audioBuffer = context.createBuffer(format.channels, samples, format.rate);
 
@@ -227,12 +416,33 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
             // Fill audio buffer with data for channel
             var offset = channel;
             for (var i = 0; i < samples; i++) {
-                audioData[i] = source[offset] / maxValue;
+                audioData[i] = data[offset] / maxSampleValue;
                 offset += format.channels;
             }
 
         }
 
+        return audioBuffer;
+
+    };
+
+    // Defer playback of received audio packets slightly
+    reader.ondata = function playReceivedAudio(data) {
+
+        // Push received samples onto queue
+        pushAudioPacket(new SampleArray(data));
+
+        // Shift off an arbitrary packet of audio data from the queue (this may
+        // be different in size from the packet just pushed)
+        var packet = shiftAudioPacket();
+        if (!packet)
+            return;
+
+        // Determine exactly when packet CAN play
+        var packetTime = context.currentTime;
+        if (nextPacketTime < packetTime)
+            nextPacketTime = packetTime;
+
         // Set up buffer source
         var source = context.createBufferSource();
         source.connect(context.destination);
@@ -242,11 +452,11 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
             source.start = source.noteOn;
 
         // Schedule packet
-        source.buffer = audioBuffer;
+        source.buffer = toAudioBuffer(packet);
         source.start(nextPacketTime);
 
-        // Update timeline
+        // Update timeline by duration of scheduled packet
-        nextPacketTime += duration;
+        nextPacketTime += packet.length / format.channels / format.rate;
 
     };