From ff5687c01ec3d93d4cdd4ed569a26250dfc41ef9 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 28 Sep 2015 12:37:09 -0700 Subject: [PATCH] GUAC-1354: Refactor Guacamole.AudioChannel to Guacamole.AudioPlayer. --- .../src/main/webapp/modules/AudioChannel.js | 291 ------------ .../src/main/webapp/modules/AudioPlayer.js | 418 ++++++++++++++++++ .../src/main/webapp/modules/Client.js | 88 ++-- 3 files changed, 465 insertions(+), 332 deletions(-) delete mode 100644 guacamole-common-js/src/main/webapp/modules/AudioChannel.js create mode 100644 guacamole-common-js/src/main/webapp/modules/AudioPlayer.js diff --git a/guacamole-common-js/src/main/webapp/modules/AudioChannel.js b/guacamole-common-js/src/main/webapp/modules/AudioChannel.js deleted file mode 100644 index 0f64c4d70..000000000 --- a/guacamole-common-js/src/main/webapp/modules/AudioChannel.js +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (C) 2015 Glyptodon LLC - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -var Guacamole = Guacamole || {}; - -/** - * Abstract audio channel which queues and plays arbitrary audio data. - * - * @constructor - */ -Guacamole.AudioChannel = function AudioChannel() { - - /** - * Reference to this AudioChannel. - * - * @private - * @type Guacamole.AudioChannel - */ - var channel = this; - - /** - * The earliest possible time that the next packet could play without - * overlapping an already-playing packet, in milliseconds. - * - * @private - * @type Number - */ - var nextPacketTime = Guacamole.AudioChannel.getTimestamp(); - - /** - * The last time that sync() was called, in milliseconds. If sync() has - * never been called, this will be the time the Guacamole.AudioChannel - * was created. - * - * @type Number - */ - var lastSync = nextPacketTime; - - /** - * Notifies this Guacamole.AudioChannel that all audio up to the current - * point in time has been given via play(), and that any difference in time - * between queued audio packets and the current time can be considered - * latency. - */ - this.sync = function sync() { - - // Calculate elapsed time since last sync - var now = Guacamole.AudioChannel.getTimestamp(); - var elapsed = now - lastSync; - - // Reschedule future playback time such that playback latency is - // bounded within the duration of the last audio frame - nextPacketTime = Math.min(nextPacketTime, now + elapsed); - - // Record sync time - lastSync = now; - - }; - - /** - * Queues up the given data for playing by this channel once all previously - * queued data has been played. If no data has been queued, the data will - * play immediately. - * - * @param {String} mimetype - * The mimetype of the audio data provided. - * - * @param {Number} duration - * The duration of the data provided, in milliseconds. - * - * @param {Blob} data - * The blob of audio data to play. - */ - this.play = function play(mimetype, duration, data) { - - var packet = new Guacamole.AudioChannel.Packet(mimetype, data); - - // Determine exactly when packet CAN play - var packetTime = Guacamole.AudioChannel.getTimestamp(); - if (nextPacketTime < packetTime) - nextPacketTime = packetTime; - - // Schedule packet - packet.play(nextPacketTime); - - // Update timeline - nextPacketTime += duration; - - }; - -}; - -// Define context if available -if (window.AudioContext) { - try {Guacamole.AudioChannel.context = new AudioContext();} - catch (e){} -} - -// Fallback to Webkit-specific AudioContext implementation -else if (window.webkitAudioContext) { - try {Guacamole.AudioChannel.context = new webkitAudioContext();} - catch (e){} -} - -/** - * Returns a base timestamp which can be used for scheduling future audio - * playback. Scheduling playback for the value returned by this function plus - * N will cause the associated audio to be played back N milliseconds after - * the function is called. - * - * @return {Number} An arbitrary channel-relative timestamp, in milliseconds. - */ -Guacamole.AudioChannel.getTimestamp = function() { - - // If we have an audio context, use its timestamp - if (Guacamole.AudioChannel.context) - return Guacamole.AudioChannel.context.currentTime * 1000; - - // If we have high-resolution timers, use those - if (window.performance) { - - if (window.performance.now) - return window.performance.now(); - - if (window.performance.webkitNow) - return window.performance.webkitNow(); - - } - - // Fallback to millisecond-resolution system time - return new Date().getTime(); - -}; - -/** - * Abstract representation of an audio packet. - * - * @constructor - * - * @param {String} mimetype The mimetype of the data contained by this packet. - * @param {Blob} data The blob of sound data contained by this packet. - */ -Guacamole.AudioChannel.Packet = function(mimetype, data) { - - /** - * Schedules this packet for playback at the given time. - * - * @function - * @param {Number} when The time this packet should be played, in - * milliseconds. - */ - this.play = function(when) { /* NOP */ }; // Defined conditionally depending on support - - // If audio API available, use it. - if (Guacamole.AudioChannel.context) { - - var readyBuffer = null; - - // By default, when decoding finishes, store buffer for future - // playback - var handleReady = function(buffer) { - readyBuffer = buffer; - }; - - // Read data and start decoding - var reader = new FileReader(); - reader.onload = function() { - Guacamole.AudioChannel.context.decodeAudioData( - reader.result, - function(buffer) { handleReady(buffer); } - ); - }; - reader.readAsArrayBuffer(data); - - // Set up buffer source - var source = Guacamole.AudioChannel.context.createBufferSource(); - source.connect(Guacamole.AudioChannel.context.destination); - - // Use noteOn() instead of start() if necessary - if (!source.start) - source.start = source.noteOn; - - var play_when; - - function playDelayed(buffer) { - source.buffer = buffer; - source.start(play_when / 1000); - } - - /** @ignore */ - this.play = function(when) { - - play_when = when; - - // If buffer available, play it NOW - if (readyBuffer) - playDelayed(readyBuffer); - - // Otherwise, play when decoded - else - handleReady = playDelayed; - - }; - - } - - else { - - var play_on_load = false; - - // Create audio element to house and play the data - var audio = null; - try { audio = new Audio(); } - catch (e) {} - - if (audio) { - - // Read data and start decoding - var reader = new FileReader(); - reader.onload = function() { - - var binary = ""; - var bytes = new Uint8Array(reader.result); - - // Produce binary string from bytes in buffer - for (var i=0; i + * @type Object. */ - var audioChannels = {}; + var audioPlayers = {}; // No initial parsers var parsers = []; @@ -440,6 +440,25 @@ Guacamole.Client = function(tunnel) { */ this.onerror = null; + /** + * Fired when a audio stream is created. The stream provided to this event + * handler will contain its own event handlers for received data. + * + * @event + * @param {Guacamole.InputStream} stream + * The stream that will receive audio data from the server. + * + * @param {String} mimetype + * The mimetype of the audio data which will be received. + * + * @return {Guacamole.AudioPlayer} + * An object which implements the Guacamole.AudioPlayer interface and + * has been initialied to play the data in the provided stream, or null + * if the built-in audio players of the Guacamole client should be + * used. + */ + this.onaudio = null; + /** * Fired when the clipboard of the remote client is changing. * @@ -499,27 +518,6 @@ Guacamole.Client = function(tunnel) { */ this.onsync = null; - /** - * Returns the audio channel having the given index, creating a new channel - * if necessary. - * - * @param {Number} index - * The index of the audio channel to retrieve. - * - * @returns {Guacamole.AudioChannel} - * The audio channel having the given index. - */ - var getAudioChannel = function getAudioChannel(index) { - - // Get audio channel, creating it first if necessary - var audio_channel = audioChannels[index]; - if (!audio_channel) - audio_channel = audioChannels[index] = new Guacamole.AudioChannel(); - - return audio_channel; - - }; - /** * Returns the layer with the given index, creating it if necessary. * Positive indices refer to visible layers, an index of zero refers to @@ -626,24 +624,32 @@ Guacamole.Client = function(tunnel) { "audio": function(parameters) { var stream_index = parseInt(parameters[0]); - var channel = getAudioChannel(parseInt(parameters[1])); - var mimetype = parameters[2]; - var duration = parseFloat(parameters[3]); + var mimetype = parameters[1]; // Create stream var stream = streams[stream_index] = new Guacamole.InputStream(guac_client, stream_index); - // Assemble entire stream as a blob - var blob_reader = new Guacamole.BlobReader(stream, mimetype); + // Get player instance via callback + var audioPlayer = null; + if (guac_client.onaudio) + audioPlayer = guac_client.onaudio(stream, mimetype); - // Play blob as audio - blob_reader.onend = function() { - channel.play(mimetype, duration, blob_reader.getBlob()); - }; + // If unsuccessful, use a default implementation + if (!audioPlayer) { + if (Guacamole.RawAudioPlayer.isSupportedType(mimetype)) + audioPlayer = new Guacamole.RawAudioPlayer(stream, mimetype); + } - // Send success response - guac_client.sendAck(stream_index, "OK", 0x0000); + // If player somehow successfully retrieved, send success response + if (audioPlayer) { + audioPlayers[stream_index] = audioPlayer; + guac_client.sendAck(stream_index, "OK", 0x0000); + } + + // Otherwise, mimetype must be unsupported + else + guac_client.sendAck(stream_index, "BAD TYPE", 0x030F); }, @@ -1113,11 +1119,11 @@ Guacamole.Client = function(tunnel) { // Flush display, send sync when done display.flush(function displaySyncComplete() { - // Synchronize all audio channels - for (var index in audioChannels) { - var audioChannel = audioChannels[index]; - if (audioChannel) - audioChannel.sync(); + // Synchronize all audio players + for (var index in audioPlayers) { + var audioPlayer = audioPlayers[index]; + if (audioPlayer) + audioPlayer.sync(); } // Send sync response to server