GUAC-1511: Merge front end audio support changes.

This commit is contained in:
James Muehlner
2016-03-30 13:19:35 -07:00
5 changed files with 570 additions and 203 deletions

View File

@@ -126,9 +126,9 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
* The format of audio this player will decode.
*
* @private
* @type {Guacamole.RawAudioPlayer._Format}
* @type {Guacamole.RawAudioFormat}
*/
var format = Guacamole.RawAudioPlayer._Format.parse(mimetype);
var format = Guacamole.RawAudioFormat.parse(mimetype);
/**
* An instance of a Web Audio API AudioContext object, or null if the
@@ -474,133 +474,6 @@ Guacamole.RawAudioPlayer = function RawAudioPlayer(stream, mimetype) {
Guacamole.RawAudioPlayer.prototype = new Guacamole.AudioPlayer();
/**
* A description of the format of raw PCM audio received by a
* Guacamole.RawAudioPlayer. This object describes the number of bytes per
* sample, the number of channels, and the overall sample rate.
*
* @private
* @constructor
* @param {Guacamole.RawAudioPlayer._Format|Object} template
* The object whose properties should be copied into the corresponding
* properties of the new Guacamole.RawAudioPlayer._Format.
*/
Guacamole.RawAudioPlayer._Format = function _Format(template) {
/**
* The number of bytes in each sample of audio data. This value is
* independent of the number of channels.
*
* @type {Number}
*/
this.bytesPerSample = template.bytesPerSample;
/**
* The number of audio channels (ie: 1 for mono, 2 for stereo).
*
* @type {Number}
*/
this.channels = template.channels;
/**
* The number of samples per second, per channel.
*
* @type {Number}
*/
this.rate = template.rate;
};
/**
* Parses the given mimetype, returning a new Guacamole.RawAudioPlayer._Format
* which describes the type of raw audio data represented by that mimetype. If
* the mimetype is not supported by Guacamole.RawAudioPlayer, null is returned.
*
* @private
* @param {String} mimetype
* The audio mimetype to parse.
*
* @returns {Guacamole.RawAudioPlayer._Format}
* A new Guacamole.RawAudioPlayer._Format which describes the type of raw
* audio data represented by the given mimetype, or null if the given
* mimetype is not supported.
*/
Guacamole.RawAudioPlayer._Format.parse = function parseFormat(mimetype) {
var bytesPerSample;
// Rate is absolutely required - if null is still present later, the
// mimetype must not be supported
var rate = null;
// Default for both "audio/L8" and "audio/L16" is one channel
var channels = 1;
// "audio/L8" has one byte per sample
if (mimetype.substring(0, 9) === 'audio/L8;') {
mimetype = mimetype.substring(9);
bytesPerSample = 1;
}
// "audio/L16" has two bytes per sample
else if (mimetype.substring(0, 10) === 'audio/L16;') {
mimetype = mimetype.substring(10);
bytesPerSample = 2;
}
// All other types are unsupported
else
return null;
// Parse all parameters
var parameters = mimetype.split(',');
for (var i = 0; i < parameters.length; i++) {
var parameter = parameters[i];
// All parameters must have an equals sign separating name from value
var equals = parameter.indexOf('=');
if (equals === -1)
return null;
// Parse name and value from parameter string
var name = parameter.substring(0, equals);
var value = parameter.substring(equals+1);
// Handle each supported parameter
switch (name) {
// Number of audio channels
case 'channels':
channels = parseInt(value);
break;
// Sample rate
case 'rate':
rate = parseInt(value);
break;
// All other parameters are unsupported
default:
return null;
}
};
// The rate parameter is required
if (rate === null)
return null;
// Return parsed format details
return new Guacamole.RawAudioPlayer._Format({
bytesPerSample : bytesPerSample,
channels : channels,
rate : rate
});
};
/**
* Determines whether the given mimetype is supported by
* Guacamole.RawAudioPlayer.
@@ -618,7 +491,7 @@ Guacamole.RawAudioPlayer.isSupportedType = function isSupportedType(mimetype) {
if (!window.AudioContext && !window.webkitAudioContext)
return false;
return Guacamole.RawAudioPlayer._Format.parse(mimetype) !== null;
return Guacamole.RawAudioFormat.parse(mimetype) !== null;
};

View File

@@ -0,0 +1,324 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
var Guacamole = Guacamole || {};
/**
* Abstract audio recorder which streams arbitrary audio data to an underlying
* Guacamole.OutputStream. It is up to implementations of this class to provide
* some means of handling this Guacamole.OutputStream. Data produced by the
* recorder is to be sent along the provided stream immediately.
*
* @constructor
*/
Guacamole.AudioRecorder = function AudioRecorder() {
// AudioRecorder currently provides no functions
};
/**
* Determines whether the given mimetype is supported by any built-in
* implementation of Guacamole.AudioRecorder, and thus will be properly handled
* by Guacamole.AudioRecorder.getInstance().
*
* @param {String} mimetype
* The mimetype to check.
*
* @returns {Boolean}
* true if the given mimetype is supported by any built-in
* Guacamole.AudioRecorder, false otherwise.
*/
Guacamole.AudioRecorder.isSupportedType = function isSupportedType(mimetype) {
return Guacamole.RawAudioRecorder.isSupportedType(mimetype);
};
/**
* Returns a list of all mimetypes supported by any built-in
* Guacamole.AudioRecorder, in rough order of priority. Beware that only the
* core mimetypes themselves will be listed. Any mimetype parameters, even
* required ones, will not be included in the list. For example, "audio/L8" is
* a supported raw audio mimetype that is supported, but it is invalid without
* additional parameters. Something like "audio/L8;rate=44100" would be valid,
* however (see https://tools.ietf.org/html/rfc4856).
*
* @returns {String[]}
* A list of all mimetypes supported by any built-in
* Guacamole.AudioRecorder, excluding any parameters.
*/
Guacamole.AudioRecorder.getSupportedTypes = function getSupportedTypes() {
return Guacamole.RawAudioRecorder.getSupportedTypes();
};
/**
* Returns an instance of Guacamole.AudioRecorder providing support for the
* given audio format. If support for the given audio format is not available,
* null is returned.
*
* @param {Guacamole.OutputStream} stream
* The Guacamole.OutputStream to send audio data through.
*
* @param {String} mimetype
* The mimetype of the audio data to be sent along the provided stream.
*
* @return {Guacamole.AudioRecorder}
* A Guacamole.AudioRecorder instance supporting the given mimetype and
* writing to the given stream, or null if support for the given mimetype
* is absent.
*/
Guacamole.AudioRecorder.getInstance = function getInstance(stream, mimetype) {
// Use raw audio recorder if possible
if (Guacamole.RawAudioRecorder.isSupportedType(mimetype))
return new Guacamole.RawAudioRecorder(stream, mimetype);
// No support for given mimetype
return null;
};
/**
* Implementation of Guacamole.AudioRecorder providing support for raw PCM
* format audio. This recorder relies only on the Web Audio API and does not
* require any browser-level support for its audio formats.
*
* @constructor
* @augments Guacamole.AudioRecorder
* @param {Guacamole.OutputStream} stream
* The Guacamole.OutputStream to write audio data to.
*
* @param {String} mimetype
* The mimetype of the audio data to send along the provided stream, which
* must be a "audio/L8" or "audio/L16" mimetype with necessary parameters,
* such as: "audio/L16;rate=44100,channels=2".
*/
Guacamole.RawAudioRecorder = function RawAudioRecorder(stream, mimetype) {
/**
* The format of audio this recorder will encode.
*
* @private
* @type {Guacamole.RawAudioFormat}
*/
var format = Guacamole.RawAudioFormat.parse(mimetype);
/**
* An instance of a Web Audio API AudioContext object, or null if the
* Web Audio API is not supported.
*
* @private
* @type {AudioContext}
*/
var context = (function getAudioContext() {
// Fallback to Webkit-specific AudioContext implementation
var AudioContext = window.AudioContext || window.webkitAudioContext;
// Get new AudioContext instance if Web Audio API is supported
if (AudioContext) {
try {
return new AudioContext();
}
catch (e) {
// Do not use Web Audio API if not allowed by browser
}
}
// Web Audio API not supported
return null;
})();
/**
* A function which directly invokes the browser's implementation of
* navigator.getUserMedia() with all provided parameters.
*
* @type Function
*/
var getUserMedia = (navigator.getUserMedia
|| navigator.webkitGetUserMedia
|| navigator.mozGetUserMedia
|| navigator.msGetUserMedia).bind(navigator);
/**
* Guacamole.ArrayBufferWriter wrapped around the audio output stream
* provided when this Guacamole.RawAudioRecorder was created.
*
* @private
* @type {Guacamole.ArrayBufferWriter}
*/
var writer = new Guacamole.ArrayBufferWriter(stream);
/**
* The type of typed array that will be used to represent each audio packet
* internally. This will be either Int8Array or Int16Array, depending on
* whether the raw audio format is 8-bit or 16-bit.
*
* @private
* @constructor
*/
var SampleArray = (format.bytesPerSample === 1) ? window.Int8Array : window.Int16Array;
/**
* The maximum absolute value of any sample within a raw audio packet sent
* by this audio recorder. This depends only on the size of each sample,
* and will be 128 for 8-bit audio and 32768 for 16-bit audio.
*
* @private
* @type {Number}
*/
var maxSampleValue = (format.bytesPerSample === 1) ? 128 : 32768;
/**
* The size of audio buffer to request from the Web Audio API when
* recording audio. This must be a power of two between 256 and 16384
* inclusive, as required by AudioContext.createScriptProcessor().
*
* @private
* @type {Number}
*/
var bufferSize = format.bytesPerSample * 4096;
/**
* Converts the given AudioBuffer into an audio packet, ready for streaming
* along the underlying output stream. Unlike the raw audio packets used by
* this audio recorder, AudioBuffers require floating point samples and are
* split into isolated planes of channel-specific data.
*
* @private
* @param {AudioBuffer} audioBuffer
* The Web Audio API AudioBuffer that should be converted to a raw
* audio packet.
*
* @returns {SampleArray}
* A new raw audio packet containing the audio data from the provided
* AudioBuffer.
*/
var toSampleArray = function toSampleArray(audioBuffer) {
// Get array for raw PCM storage
var data = new SampleArray(audioBuffer.length);
// Convert each channel
for (var channel = 0; channel < format.channels; channel++) {
var audioData = audioBuffer.getChannelData(channel);
// Fill array with data from audio buffer channel
var offset = channel;
for (var i = 0; i < audioData.length; i++) {
data[offset] = audioData[i] * maxSampleValue;
offset += format.channels;
}
}
return data;
};
// Once audio stream is successfully open, request and begin reading audio
writer.onack = function audioStreamAcknowledged(status) {
// Abort stream if rejected
if (status.code !== Guacamole.Status.Code.SUCCESS) {
writer.sendEnd();
return;
}
// Attempt to retrieve an audio input stream from the browser
getUserMedia({ 'audio' : true }, function streamReceived(mediaStream) {
// Create processing node which receives appropriately-sized audio buffers
var processor = context.createScriptProcessor(bufferSize, format.channels, format.channels);
processor.connect(context.destination);
// Send blobs when audio buffers are received
processor.onaudioprocess = function processAudio(e) {
writer.sendData(toSampleArray(e.inputBuffer));
};
// Connect processing node to user's audio input source
var source = context.createMediaStreamSource(mediaStream);
source.connect(processor);
}, function streamDenied() {
// Simply end stream if audio access is not allowed
writer.sendEnd();
});
};
};
Guacamole.RawAudioRecorder.prototype = new Guacamole.AudioRecorder();
/**
* Determines whether the given mimetype is supported by
* Guacamole.RawAudioRecorder.
*
* @param {String} mimetype
* The mimetype to check.
*
* @returns {Boolean}
* true if the given mimetype is supported by Guacamole.RawAudioRecorder,
* false otherwise.
*/
Guacamole.RawAudioRecorder.isSupportedType = function isSupportedType(mimetype) {
// No supported types if no Web Audio API
if (!window.AudioContext && !window.webkitAudioContext)
return false;
return Guacamole.RawAudioFormat.parse(mimetype) !== null;
};
/**
* Returns a list of all mimetypes supported by Guacamole.RawAudioRecorder. Only
* the core mimetypes themselves will be listed. Any mimetype parameters, even
* required ones, will not be included in the list. For example, "audio/L8" is
* a raw audio mimetype that may be supported, but it is invalid without
* additional parameters. Something like "audio/L8;rate=44100" would be valid,
* however (see https://tools.ietf.org/html/rfc4856).
*
* @returns {String[]}
* A list of all mimetypes supported by Guacamole.RawAudioRecorder,
* excluding any parameters. If the necessary JavaScript APIs for recording
* raw audio are absent, this list will be empty.
*/
Guacamole.RawAudioRecorder.getSupportedTypes = function getSupportedTypes() {
// No supported types if no Web Audio API
if (!window.AudioContext && !window.webkitAudioContext)
return [];
// We support 8-bit and 16-bit raw PCM
return [
'audio/L8',
'audio/L16'
];
};

View File

@@ -232,90 +232,99 @@ Guacamole.Client = function(tunnel) {
};
/**
* Allocates an available stream index and creates a new
* Guacamole.OutputStream using that index, associating the resulting
* stream with this Guacamole.Client. Note that this stream will not yet
* exist as far as the other end of the Guacamole connection is concerned.
* Streams exist within the Guacamole protocol only when referenced by an
* instruction which creates the stream, such as a "clipboard", "file", or
* "pipe" instruction.
*
* @returns {Guacamole.OutputStream}
* A new Guacamole.OutputStream with a newly-allocated index and
* associated with this Guacamole.Client.
*/
this.createOutputStream = function createOutputStream() {
// Allocate index
var index = stream_indices.next();
// Return new stream
var stream = output_streams[index] = new Guacamole.OutputStream(guac_client, index);
return stream;
};
/**
* Opens a new audio stream for writing, where audio data having the give
* mimetype will be sent along the returned stream. The instruction
* necessary to create this stream will automatically be sent.
*
* @param {String} mimetype
* The mimetype of the audio data that will be sent along the returned
* stream.
*
* @return {Guacamole.OutputStream}
* The created audio stream.
*/
this.createAudioStream = function(mimetype) {
// Allocate and associate stream with audio metadata
var stream = guac_client.createOutputStream();
tunnel.sendMessage("audio", stream.index, mimetype);
return stream;
};
/**
* Opens a new file for writing, having the given index, mimetype and
* filename.
*
* filename. The instruction necessary to create this stream will
* automatically be sent.
*
* @param {String} mimetype The mimetype of the file being sent.
* @param {String} filename The filename of the file being sent.
* @return {Guacamole.OutputStream} The created file stream.
*/
this.createFileStream = function(mimetype, filename) {
// Allocate index
var index = stream_indices.next();
// Create new stream
tunnel.sendMessage("file", index, mimetype, filename);
var stream = output_streams[index] = new Guacamole.OutputStream(guac_client, index);
// Override sendEnd() of stream to automatically free index
var old_end = stream.sendEnd;
stream.sendEnd = function() {
old_end();
stream_indices.free(index);
delete output_streams[index];
};
// Return new, overridden stream
// Allocate and associate stream with file metadata
var stream = guac_client.createOutputStream();
tunnel.sendMessage("file", stream.index, mimetype, filename);
return stream;
};
/**
* Opens a new pipe for writing, having the given name and mimetype.
*
* Opens a new pipe for writing, having the given name and mimetype. The
* instruction necessary to create this stream will automatically be sent.
*
* @param {String} mimetype The mimetype of the data being sent.
* @param {String} name The name of the pipe.
* @return {Guacamole.OutputStream} The created file stream.
*/
this.createPipeStream = function(mimetype, name) {
// Allocate index
var index = stream_indices.next();
// Create new stream
tunnel.sendMessage("pipe", index, mimetype, name);
var stream = output_streams[index] = new Guacamole.OutputStream(guac_client, index);
// Override sendEnd() of stream to automatically free index
var old_end = stream.sendEnd;
stream.sendEnd = function() {
old_end();
stream_indices.free(index);
delete output_streams[index];
};
// Return new, overridden stream
// Allocate and associate stream with pipe metadata
var stream = guac_client.createOutputStream();
tunnel.sendMessage("pipe", stream.index, mimetype, name);
return stream;
};
/**
* Opens a new clipboard object for writing, having the given mimetype.
*
* Opens a new clipboard object for writing, having the given mimetype. The
* instruction necessary to create this stream will automatically be sent.
*
* @param {String} mimetype The mimetype of the data being sent.
* @param {String} name The name of the pipe.
* @return {Guacamole.OutputStream} The created file stream.
*/
this.createClipboardStream = function(mimetype) {
// Allocate index
var index = stream_indices.next();
// Create new stream
tunnel.sendMessage("clipboard", index, mimetype);
var stream = output_streams[index] = new Guacamole.OutputStream(guac_client, index);
// Override sendEnd() of stream to automatically free index
var old_end = stream.sendEnd;
stream.sendEnd = function() {
old_end();
stream_indices.free(index);
delete output_streams[index];
};
// Return new, overridden stream
// Allocate and associate stream with clipboard metadata
var stream = guac_client.createOutputStream();
tunnel.sendMessage("clipboard", stream.index, mimetype);
return stream;
};
@@ -323,7 +332,8 @@ Guacamole.Client = function(tunnel) {
/**
* Creates a new output stream associated with the given object and having
* the given mimetype and name. The legality of a mimetype and name is
* dictated by the object itself.
* dictated by the object itself. The instruction necessary to create this
* stream will automatically be sent.
*
* @param {Number} index
* The index of the object for which the output stream is being
@@ -341,22 +351,9 @@ Guacamole.Client = function(tunnel) {
*/
this.createObjectOutputStream = function createObjectOutputStream(index, mimetype, name) {
// Allocate index
var streamIndex = stream_indices.next();
// Create new stream
tunnel.sendMessage("put", index, streamIndex, mimetype, name);
var stream = output_streams[streamIndex] = new Guacamole.OutputStream(guac_client, streamIndex);
// Override sendEnd() of stream to automatically free index
var oldEnd = stream.sendEnd;
stream.sendEnd = function freeStreamIndex() {
oldEnd();
stream_indices.free(streamIndex);
delete output_streams[streamIndex];
};
// Return new, overridden stream
// Allocate and ssociate stream with object metadata
var stream = guac_client.createOutputStream();
tunnel.sendMessage("put", index, stream.index, mimetype, name);
return stream;
};
@@ -415,9 +412,13 @@ Guacamole.Client = function(tunnel) {
};
/**
* Marks a currently-open stream as complete.
* Marks a currently-open stream as complete. The other end of the
* Guacamole connection will be notified via an "end" instruction that the
* stream is closed, and the index will be made available for reuse in
* future streams.
*
* @param {Number} index The index of the stream to end.
* @param {Number} index
* The index of the stream to end.
*/
this.endStream = function(index) {
@@ -425,7 +426,15 @@ Guacamole.Client = function(tunnel) {
if (!isConnected())
return;
// Explicitly close stream by sending "end" instruction
tunnel.sendMessage("end", index);
// Free associated index and stream if they exist
if (output_streams[index]) {
stream_indices.free(index);
delete output_streams[index];
}
};
/**

View File

@@ -0,0 +1,146 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
var Guacamole = Guacamole || {};
/**
* A description of the format of raw PCM audio, such as that used by
* Guacamole.RawAudioPlayer and Guacamole.RawAudioRecorder. This object
* describes the number of bytes per sample, the number of channels, and the
* overall sample rate.
*
* @constructor
* @param {Guacamole.RawAudioFormat|Object} template
* The object whose properties should be copied into the corresponding
* properties of the new Guacamole.RawAudioFormat.
*/
Guacamole.RawAudioFormat = function RawAudioFormat(template) {
/**
* The number of bytes in each sample of audio data. This value is
* independent of the number of channels.
*
* @type {Number}
*/
this.bytesPerSample = template.bytesPerSample;
/**
* The number of audio channels (ie: 1 for mono, 2 for stereo).
*
* @type {Number}
*/
this.channels = template.channels;
/**
* The number of samples per second, per channel.
*
* @type {Number}
*/
this.rate = template.rate;
};
/**
* Parses the given mimetype, returning a new Guacamole.RawAudioFormat
* which describes the type of raw audio data represented by that mimetype. If
* the mimetype is not a supported raw audio data mimetype, null is returned.
*
* @param {String} mimetype
* The audio mimetype to parse.
*
* @returns {Guacamole.RawAudioFormat}
* A new Guacamole.RawAudioFormat which describes the type of raw
* audio data represented by the given mimetype, or null if the given
* mimetype is not supported.
*/
Guacamole.RawAudioFormat.parse = function parseFormat(mimetype) {
var bytesPerSample;
// Rate is absolutely required - if null is still present later, the
// mimetype must not be supported
var rate = null;
// Default for both "audio/L8" and "audio/L16" is one channel
var channels = 1;
// "audio/L8" has one byte per sample
if (mimetype.substring(0, 9) === 'audio/L8;') {
mimetype = mimetype.substring(9);
bytesPerSample = 1;
}
// "audio/L16" has two bytes per sample
else if (mimetype.substring(0, 10) === 'audio/L16;') {
mimetype = mimetype.substring(10);
bytesPerSample = 2;
}
// All other types are unsupported
else
return null;
// Parse all parameters
var parameters = mimetype.split(',');
for (var i = 0; i < parameters.length; i++) {
var parameter = parameters[i];
// All parameters must have an equals sign separating name from value
var equals = parameter.indexOf('=');
if (equals === -1)
return null;
// Parse name and value from parameter string
var name = parameter.substring(0, equals);
var value = parameter.substring(equals+1);
// Handle each supported parameter
switch (name) {
// Number of audio channels
case 'channels':
channels = parseInt(value);
break;
// Sample rate
case 'rate':
rate = parseInt(value);
break;
// All other parameters are unsupported
default:
return null;
}
};
// The rate parameter is required
if (rate === null)
return null;
// Return parsed format details
return new Guacamole.RawAudioFormat({
bytesPerSample : bytesPerSample,
channels : channels,
rate : rate
});
};

View File

@@ -149,6 +149,15 @@ angular.module('client').factory('ManagedClient', ['$rootScope', '$injector',
};
/**
* The mimetype of audio data to be sent along the Guacamole connection if
* audio input is supported.
*
* @constant
* @type String
*/
ManagedClient.AUDIO_INPUT_MIMETYPE = 'audio/L16;rate=44100,channels=2';
/**
* Returns a promise which resolves with the string of connection
* parameters to be passed to the Guacamole client during connection. This
@@ -352,6 +361,12 @@ angular.module('client').factory('ManagedClient', ['$rootScope', '$injector',
case 3:
ManagedClientState.setConnectionState(managedClient.clientState,
ManagedClientState.ConnectionState.CONNECTED);
// Begin streaming audio input if possible
var stream = client.createAudioStream(ManagedClient.AUDIO_INPUT_MIMETYPE);
if (!Guacamole.AudioRecorder.getInstance(stream, ManagedClient.AUDIO_INPUT_MIMETYPE))
stream.sendEnd();
break;
// Update history when disconnecting