safebox/guacamole-client
4
0
Fork 0
mirror of https://github.com/gyurix1968/guacamole-client.git synced 2025-09-06 13:17:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

GUAC-1354: Dynamically split and reassemble audio packets to minimize clicking.

Browse Source
This commit is contained in:
Michael Jumper
2015-10-02 16:42:24 -07:00
parent a3dd959dc4
commit 079e3dad8c
1 changed files with 232 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
guacamole-common-js/src/main/webapp/modules/AudioPlayer.js
View File

@@ -180,6 +180,18 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
*/
var reader = new Guacamole.ArrayBufferReader(stream);
/**
* The minimum size of an audio packet split by splitAudioPacket(), in
* seconds. Audio packets smaller than this will not be split, nor will the
* split result of a larger packet ever be smaller in size than this
* minimum.
*
* @private
* @constant
* @type Number
*/
var MIN_SPLIT_SIZE = 0.02;
/**
* The maximum amount of latency to allow between the buffered data stream
* and the playback position, in seconds. Initially, this is set to
@@ -190,32 +202,209 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
*/
var maxLatency = 0.3;
// Play each received raw packet of audio immediately
reader.ondata = function playReceivedAudio(data) {
/**
* The type of typed array that will be used to represent each audio packet
* internally. This will be either Int8Array or Int16Array, depending on
* whether the raw audio format is 8-bit or 16-bit.
*
* @private
* @constructor
*/
var SampleArray = (format.bytesPerSample === 1) ? window.Int8Array : window.Int16Array;
/**
* The maximum absolute value of any sample within a raw audio packet
* received by this audio player. This depends only on the size of each
* sample, and will be 128 for 8-bit audio and 32768 for 16-bit audio.
*
* @private
* @type Number
*/
var maxSampleValue = (format.bytesPerSample === 1) ? 128 : 32768;
/**
* The queue of all pending audio packets, as an array of sample arrays.
* Audio packets which are pending playback will be added to this queue for
* further manipulation prior to scheduling via the Web Audio API. Once an
* audio packet leaves this queue and is scheduled via the Web Audio API,
* no further modifications can be made to that packet.
*
* @private
* @type SampleArray[]
*/
var packetQueue = [];
/**
* Given an array of audio packets, returns a single audio packet
* containing the concatenation of those packets.
*
* @private
* @param {SampleArray[]} packets
* The array of audio packets to concatenate.
*
* @returns {SampleArray}
* A single audio packet containing the concatenation of all given
* audio packets. If no packets are provided, this will be undefined.
*/
var joinAudioPackets = function joinAudioPackets(packets) {
// Do not bother joining if one or fewer packets are in the queue
if (packets.length <= 1)
return packets[0];
// Determine total sample length of the entire queue
var totalLength = 0;
packets.forEach(function addPacketLengths(packet) {
totalLength += packet.length;
});
// Append each packet within queue
var offset = 0;
var joined = new SampleArray(totalLength);
packets.forEach(function appendPacket(packet) {
joined.set(packet, offset);
offset += packet.length;
});
return joined;
};
/**
* Given a single packet of audio data, splits off an arbitrary length of
* audio data from the beginning of that packet, returning the split result
* as an array of two packets. The split location is determined through an
* algorithm intended to minimize the liklihood of audible clicking between
* packets. If no such split location is possible, an array containing only
* the originally-provided audio packet is returned.
*
* @private
* @param {SampleArray} data
* The audio packet to split.
*
* @returns {SampleArray[]}
* An array of audio packets containing the result of splitting the
* provided audio packet. If splitting is possible, this array will
* contain two packets. If splitting is not possible, this array will
* contain only the originally-provided packet.
*/
var splitAudioPacket = function splitAudioPacket(data) {
var minValue = Number.MAX_VALUE;
var optimalSplitLength = data.length;
// Calculate number of whole samples in the provided audio packet AND
// in the minimum possible split packet
var samples = Math.floor(data.length / format.channels);
var minSplitSamples = Math.floor(format.rate * MIN_SPLIT_SIZE);
// Calculate the beginning of the "end" of the audio packet
var start = Math.max(
format.channels * minSplitSamples,
format.channels * (samples - minSplitSamples)
);
// For all samples at the end of the given packet, find a point where
// the perceptible volume across all channels is lowest (and thus is
// the optimal point to split)
for (var offset = start; offset < data.length; offset += format.channels) {
// Calculate the sum of all values across all channels (the result
// will be proportional to the average volume of a sample)
var totalValue = 0;
for (var channel = 0; channel < format.channels; channel++) {
totalValue += Math.abs(data[offset + channel]);
}
// If this is the smallest average value thus far, set the split
// length such that the first packet ends with the current sample
if (totalValue <= minValue) {
optimalSplitLength = offset + format.channels;
minValue = totalValue;
}
}
// If packet is not split, return the supplied packet untouched
if (optimalSplitLength === data.length)
return [data];
// Otherwise, split the packet into two new packets according to the
// calculated optimal split length
return [
data.slice(0, optimalSplitLength),
data.slice(optimalSplitLength)
];
};
/**
* Pushes the given packet of audio data onto the playback queue. Unlike
* other private functions within Guacamole.RawAudioPlayer, the type of the
* ArrayBuffer packet of audio data here need not be specific to the type
* of audio (as with SampleArray). The ArrayBuffer type provided by a
* Guacamole.ArrayBufferReader, for example, is sufficient. Any necessary
* conversions will be performed automatically internally.
*
* @private
* @param {ArrayBuffer} data
* A raw packet of audio data that should be pushed onto the audio
* playback queue.
*/
var pushAudioPacket = function pushAudioPacket(data) {
packetQueue.push(new SampleArray(data));
};
/**
* Shifts off and returns a packet of audio data from the beginning of the
* playback queue. The length of this audio packet is determined
* dynamically according to the click-reduction algorithm implemented by
* splitAudioPacket().
*
* @returns {SampleArray}
* A packet of audio data pulled from the beginning of the playback
* queue.
*/
var shiftAudioPacket = function shiftAudioPacket() {
// Flatten data in packet queue
var data = joinAudioPackets(packetQueue);
if (!data)
return null;
// Pull an appropriate amount of data from the front of the queue
packetQueue = splitAudioPacket(data);
data = packetQueue.shift();
return data;
};
/**
* Converts the given audio packet into an AudioBuffer, ready for playback
* by the Web Audio API. Unlike the raw audio packets received by this
* audio player, AudioBuffers require floating point samples and are split
* into isolated planes of channel-specific data.
*
* @private
* @param {SampleArray} data
* The raw audio packet that should be converted into a Web Audio API
* AudioBuffer.
*
* @returns {AudioBuffer}
* A new Web Audio API AudioBuffer containing the provided audio data,
* converted to the format used by the Web Audio API.
*/
var toAudioBuffer = function toAudioBuffer(data) {
// Calculate total number of samples
var samples = data.byteLength / format.channels / format.bytesPerSample;
// Calculate overall duration (in seconds)
var duration = samples / format.rate;
var samples = data.length / format.channels;
// Determine exactly when packet CAN play
var packetTime = context.currentTime;
if (nextPacketTime < packetTime)
nextPacketTime = packetTime;
// Obtain typed array view based on defined bytes per sample
var maxValue;
var source;
if (format.bytesPerSample === 1) {
source = new Int8Array(data);
maxValue = 128;
}
else {
source = new Int16Array(data);
maxValue = 32768;
}
// Get audio buffer for specified format
var audioBuffer = context.createBuffer(format.channels, samples, format.rate);
@@ -227,12 +416,33 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
// Fill audio buffer with data for channel
var offset = channel;
for (var i = 0; i < samples; i++) {
audioData[i] = source[offset] / maxValue;
audioData[i] = data[offset] / maxSampleValue;
offset += format.channels;
}
}
return audioBuffer;
};
// Defer playback of received audio packets slightly
reader.ondata = function playReceivedAudio(data) {
// Push received samples onto queue
pushAudioPacket(new SampleArray(data));
// Shift off an arbitrary packet of audio data from the queue (this may
// be different in size from the packet just pushed)
var packet = shiftAudioPacket();
if (!packet)
return;
// Determine exactly when packet CAN play
var packetTime = context.currentTime;
if (nextPacketTime < packetTime)
nextPacketTime = packetTime;
// Set up buffer source
var source = context.createBufferSource();
source.connect(context.destination);
@@ -242,11 +452,11 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
source.start = source.noteOn;
// Schedule packet
source.buffer = audioBuffer;
source.buffer = toAudioBuffer(packet);
source.start(nextPacketTime);
// Update timeline
nextPacketTime += duration;
// Update timeline by duration of scheduled packet
nextPacketTime += packet.length / format.channels / format.rate;
};