GUACAMOLE-267: Merge new failover functionality for balancing groups.

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-base/src/main/java/org/apache/guacamole/auth/jdbc/tunnel/AbstractGuacamoleTunnelService.java

@@ -39,8 +39,10 @@ import org.apache.guacamole.auth.jdbc.connection.ConnectionModel;
 import org.apache.guacamole.auth.jdbc.connection.ConnectionRecordModel;
 import org.apache.guacamole.auth.jdbc.connection.ConnectionParameterModel;
 import org.apache.guacamole.GuacamoleException;
+import org.apache.guacamole.GuacamoleResourceConflictException;
 import org.apache.guacamole.GuacamoleResourceNotFoundException;
 import org.apache.guacamole.GuacamoleSecurityException;
+import org.apache.guacamole.GuacamoleUpstreamException;
 import org.apache.guacamole.auth.jdbc.JDBCEnvironment;
 import org.apache.guacamole.auth.jdbc.connection.ConnectionMapper;
 import org.apache.guacamole.environment.Environment;
@@ -60,6 +62,9 @@ import org.apache.guacamole.auth.jdbc.sharingprofile.ModeledSharingProfile;
 import org.apache.guacamole.auth.jdbc.sharingprofile.SharingProfileParameterMapper;
 import org.apache.guacamole.auth.jdbc.sharingprofile.SharingProfileParameterModel;
 import org.apache.guacamole.auth.jdbc.user.RemoteAuthenticatedUser;
+import org.apache.guacamole.protocol.FailoverGuacamoleSocket;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -69,6 +74,11 @@ import org.apache.guacamole.auth.jdbc.user.RemoteAuthenticatedUser;
  */
 public abstract class AbstractGuacamoleTunnelService implements GuacamoleTunnelService {
 
+    /**
+     * Logger for this class.
+     */
+    private final Logger logger = LoggerFactory.getLogger(AbstractGuacamoleTunnelService.class);
+
     /**
      * The environment of the Guacamole server.
      */
@@ -439,6 +449,10 @@ public abstract class AbstractGuacamoleTunnelService implements GuacamoleTunnelS
      *     Information describing the Guacamole client connecting to the given
      *     connection.
      *
+     * @param interceptErrors
+     *     Whether errors from the upstream remote desktop should be
+     *     intercepted and rethrown as GuacamoleUpstreamExceptions.
+     *
      * @return
      *     A new GuacamoleTunnel which is configured and connected to the given
      *     connection.
@@ -448,7 +462,7 @@ public abstract class AbstractGuacamoleTunnelService implements GuacamoleTunnelS
      *     while connection configuration information is being retrieved.
      */
     private GuacamoleTunnel assignGuacamoleTunnel(ActiveConnectionRecord activeConnection,
-            GuacamoleClientInformation info) throws GuacamoleException {
+            GuacamoleClientInformation info, boolean interceptErrors) throws GuacamoleException {
 
         // Record new active connection
         Runnable cleanupTask = new ConnectionCleanupTask(activeConnection);
@@ -488,7 +502,10 @@ public abstract class AbstractGuacamoleTunnelService implements GuacamoleTunnelS
                 getUnconfiguredGuacamoleSocket(cleanupTask), config, info);
 
             // Assign and return new tunnel
-            return activeConnection.assignGuacamoleTunnel(socket);
+            if (interceptErrors)
+                return activeConnection.assignGuacamoleTunnel(new FailoverGuacamoleSocket(socket), socket.getConnectionID());
+            else
+                return activeConnection.assignGuacamoleTunnel(socket, socket.getConnectionID());
             
         }
 
@@ -633,7 +650,7 @@ public abstract class AbstractGuacamoleTunnelService implements GuacamoleTunnelS
         // Connect only if the connection was successfully acquired
         ActiveConnectionRecord connectionRecord = activeConnectionRecordProvider.get();
         connectionRecord.init(user, connection);
-        return assignGuacamoleTunnel(connectionRecord, info);
+        return assignGuacamoleTunnel(connectionRecord, info, false);
 
     }
 
@@ -653,29 +670,50 @@ public abstract class AbstractGuacamoleTunnelService implements GuacamoleTunnelS
         if (connections.isEmpty())
             throw new GuacamoleSecurityException("Permission denied.");
 
-        // Acquire group
-        acquire(user, connectionGroup);
+        do {
 
-        // Attempt to acquire to any child
-        ModeledConnection connection;
-        try {
-            connection = acquire(user, connections);
-        }
+            // Acquire group
+            acquire(user, connectionGroup);
 
-        // Ensure connection group is always released if child acquire fails
-        catch (GuacamoleException e) {
-            release(user, connectionGroup);
-            throw e;
-        }
+            // Attempt to acquire to any child
+            ModeledConnection connection;
+            try {
+                connection = acquire(user, connections);
+            }
 
-        // If session affinity is enabled, prefer this connection going forward
-        if (connectionGroup.isSessionAffinityEnabled())
-            user.preferConnection(connection.getIdentifier());
+            // Ensure connection group is always released if child acquire fails
+            catch (GuacamoleException e) {
+                release(user, connectionGroup);
+                throw e;
+            }
 
-        // Connect to acquired child
-        ActiveConnectionRecord connectionRecord = activeConnectionRecordProvider.get();
-        connectionRecord.init(user, connectionGroup, connection);
-        return assignGuacamoleTunnel(connectionRecord, info);
+            try {
+
+                // Connect to acquired child
+                ActiveConnectionRecord connectionRecord = activeConnectionRecordProvider.get();
+                connectionRecord.init(user, connectionGroup, connection);
+                GuacamoleTunnel tunnel = assignGuacamoleTunnel(connectionRecord, info, connections.size() > 1);
+
+                // If session affinity is enabled, prefer this connection going forward
+                if (connectionGroup.isSessionAffinityEnabled())
+                    user.preferConnection(connection.getIdentifier());
+
+                return tunnel;
+
+            }
+
+            // If connection failed due to an upstream error, retry other
+            // connections
+            catch (GuacamoleUpstreamException e) {
+                logger.info("Upstream error intercepted for connection \"{}\". Failing over to next connection in group...", connection.getIdentifier());
+                logger.debug("Upstream remote desktop reported an error during connection.", e);
+                connections.remove(connection);
+            }
+
+        } while (!connections.isEmpty());
+
+        // All connection possibilities have been exhausted
+        throw new GuacamoleResourceConflictException("Cannot connect. All upstream connections are unavailable.");
 
     }
 
@@ -703,7 +741,7 @@ public abstract class AbstractGuacamoleTunnelService implements GuacamoleTunnelS
                 definition.getSharingProfile());
 
         // Connect to shared connection described by the created record
-        GuacamoleTunnel tunnel = assignGuacamoleTunnel(connectionRecord, info);
+        GuacamoleTunnel tunnel = assignGuacamoleTunnel(connectionRecord, info, false);
 
         // Register tunnel, such that it is closed when the
         // SharedConnectionDefinition is invalidated

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-base/src/main/java/org/apache/guacamole/auth/jdbc/tunnel/ActiveConnectionRecord.java

@@ -366,13 +366,17 @@ public class ActiveConnectionRecord implements ConnectionRecord {
      * given socket.
      *
      * @param socket
-     *     The ConfiguredGuacamoleSocket to use to create the tunnel associated
-     *     with this connection record.
+     *     The GuacamoleSocket to use to create the tunnel associated with this
+     *     connection record.
+     *
+     * @param connectionID
+     *     The connection ID assigned to this connection by guacd.
      *
      * @return
      *     The newly-created tunnel associated with this connection record.
      */
-    public GuacamoleTunnel assignGuacamoleTunnel(final ConfiguredGuacamoleSocket socket) {
+    public GuacamoleTunnel assignGuacamoleTunnel(final GuacamoleSocket socket,
+            String connectionID) {
 
         // Create tunnel with given socket
         this.tunnel = new AbstractGuacamoleTunnel() {
@@ -391,7 +395,7 @@ public class ActiveConnectionRecord implements ConnectionRecord {
 
         // Store connection ID of the primary connection only
         if (isPrimaryConnection())
-            this.connectionID = socket.getConnectionID();
+            this.connectionID = connectionID;
 
         // Return newly-created tunnel
         return this.tunnel;

guacamole-common/src/main/java/org/apache/guacamole/protocol/FailoverGuacamoleSocket.java

@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.guacamole.protocol;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import org.apache.guacamole.GuacamoleException;
+import org.apache.guacamole.GuacamoleUpstreamException;
+import org.apache.guacamole.GuacamoleUpstreamNotFoundException;
+import org.apache.guacamole.GuacamoleUpstreamTimeoutException;
+import org.apache.guacamole.GuacamoleUpstreamUnavailableException;
+import org.apache.guacamole.io.GuacamoleReader;
+import org.apache.guacamole.io.GuacamoleWriter;
+import org.apache.guacamole.net.GuacamoleSocket;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * GuacamoleSocket which intercepts errors received early in the Guacamole
+ * session. Upstream errors which are intercepted early enough result in
+ * exceptions thrown immediately within the FailoverGuacamoleSocket's
+ * constructor, allowing a different socket to be substituted prior to
+ * fulfilling the connection.
+ */
+public class FailoverGuacamoleSocket implements GuacamoleSocket {
+
+    /**
+     * Logger for this class.
+     */
+    private static final Logger logger =
+            LoggerFactory.getLogger(FailoverGuacamoleSocket.class);
+
+    /**
+     * The maximum number of characters of Guacamole instruction data to store
+     * within the instruction queue while searching for errors. Once this limit
+     * is exceeded, the connection is assumed to be successful.
+     */
+    private static final int INSTRUCTION_QUEUE_LIMIT = 2048;
+
+    /**
+     * The wrapped socket being used.
+     */
+    private final GuacamoleSocket socket;
+
+    /**
+     * Queue of all instructions read while this FailoverGuacamoleSocket was
+     * being constructed.
+     */
+    private final Queue<GuacamoleInstruction> instructionQueue =
+            new LinkedList<GuacamoleInstruction>();
+
+    /**
+     * Parses the given "error" instruction, throwing an exception if the
+     * instruction represents an error from the upstream remote desktop.
+     *
+     * @param instruction
+     *     The "error" instruction to parse.
+     *
+     * @throws GuacamoleUpstreamException
+     *     If the "error" instruction represents an error from the upstream
+     *     remote desktop.
+     */
+    private static void handleUpstreamErrors(GuacamoleInstruction instruction)
+            throws GuacamoleUpstreamException {
+
+        // Ignore error instructions which are missing the status code
+        List<String> args = instruction.getArgs();
+        if (args.size() < 2) {
+            logger.debug("Received \"error\" instruction without status code.");
+            return;
+        }
+
+        // Parse the status code from the received error instruction
+        int statusCode;
+        try {
+            statusCode = Integer.parseInt(args.get(1));
+        }
+        catch (NumberFormatException e) {
+            logger.debug("Received \"error\" instruction with non-numeric status code.", e);
+            return;
+        }
+
+        // Translate numeric status code into a GuacamoleStatus
+        GuacamoleStatus status = GuacamoleStatus.fromGuacamoleStatusCode(statusCode);
+        if (status == null) {
+            logger.debug("Received \"error\" instruction with unknown/invalid status code: {}", statusCode);
+            return;
+        }
+
+        // Only handle error instructions related to the upstream remote desktop
+        switch (status) {
+
+            // Generic upstream error
+            case UPSTREAM_ERROR:
+                throw new GuacamoleUpstreamException(args.get(0));
+
+            // Upstream is unreachable
+            case UPSTREAM_NOT_FOUND:
+                throw new GuacamoleUpstreamNotFoundException(args.get(0));
+
+            // Upstream did not respond
+            case UPSTREAM_TIMEOUT:
+                throw new GuacamoleUpstreamTimeoutException(args.get(0));
+
+            // Upstream is refusing the connection
+            case UPSTREAM_UNAVAILABLE:
+                throw new GuacamoleUpstreamUnavailableException(args.get(0));
+
+        }
+
+    }
+
+    /**
+     * Creates a new FailoverGuacamoleSocket which reads Guacamole instructions
+     * from the given socket, searching for errors from the upstream remote
+     * desktop. If an upstream error is encountered, it is thrown as a
+     * GuacamoleUpstreamException. This constructor will block until an error
+     * is encountered, or until the connection appears to have been successful.
+     * Once the FailoverGuacamoleSocket has been created, all reads, writes,
+     * etc. will be delegated to the provided socket.
+     *
+     * @param socket
+     *     The GuacamoleSocket of the Guacamole connection this
+     *     FailoverGuacamoleSocket should handle.
+     *
+     * @throws GuacamoleException
+     *     If an error occurs while reading data from the provided socket.
+     *
+     * @throws GuacamoleUpstreamException
+     *     If the connection to guacd succeeded, but an error occurred while
+     *     connecting to the remote desktop.
+     */
+    public FailoverGuacamoleSocket(GuacamoleSocket socket)
+            throws GuacamoleException, GuacamoleUpstreamException {
+
+        int totalQueueSize = 0;
+
+        GuacamoleInstruction instruction;
+        GuacamoleReader reader = socket.getReader();
+
+        // Continuously read instructions, searching for errors
+        while ((instruction = reader.readInstruction()) != null) {
+
+            // Add instruction to tail of instruction queue
+            instructionQueue.add(instruction);
+
+            // If instruction is a "sync" instruction, stop reading
+            String opcode = instruction.getOpcode();
+            if (opcode.equals("sync"))
+                break;
+
+            // If instruction is an "error" instruction, parse its contents and
+            // stop reading
+            if (opcode.equals("error")) {
+                handleUpstreamErrors(instruction);
+                break;
+            }
+
+            // Otherwise, track total data parsed, and assume connection is
+            // successful if no error encountered within reasonable space
+            totalQueueSize += instruction.toString().length();
+            if (totalQueueSize >= INSTRUCTION_QUEUE_LIMIT)
+                break;
+
+        }
+
+        this.socket = socket;
+
+    }
+
+    /**
+     * GuacamoleReader which reads instructions from the queue populated when
+     * the FailoverGuacamoleSocket was constructed. Once the queue has been
+     * emptied, reads are delegated directly to the reader of the wrapped
+     * socket.
+     */
+    private final GuacamoleReader queuedReader = new GuacamoleReader() {
+
+        @Override
+        public boolean available() throws GuacamoleException {
+            return !instructionQueue.isEmpty() || socket.getReader().available();
+        }
+
+        @Override
+        public char[] read() throws GuacamoleException {
+
+            // Read instructions from queue before finally delegating to
+            // underlying reader (received when FailoverGuacamoleSocket was
+            // being constructed)
+            if (!instructionQueue.isEmpty()) {
+                GuacamoleInstruction instruction = instructionQueue.remove();
+                return instruction.toString().toCharArray();
+            }
+
+            return socket.getReader().read();
+
+        }
+
+        @Override
+        public GuacamoleInstruction readInstruction()
+                throws GuacamoleException {
+
+            // Read instructions from queue before finally delegating to
+            // underlying reader (received when FailoverGuacamoleSocket was
+            // being constructed)
+            if (!instructionQueue.isEmpty())
+                return instructionQueue.remove();
+
+            return socket.getReader().readInstruction();
+
+        }
+
+    };
+
+    @Override
+    public GuacamoleReader getReader() {
+        return queuedReader;
+    }
+
+    @Override
+    public GuacamoleWriter getWriter() {
+        return socket.getWriter();
+    }
+
+    @Override
+    public void close() throws GuacamoleException {
+        socket.close();
+    }
+
+    @Override
+    public boolean isOpen() {
+        return socket.isOpen();
+    }
+    
+}

guacamole-common/src/main/java/org/apache/guacamole/protocol/GuacamoleInstruction.java

@@ -33,12 +33,18 @@ public class GuacamoleInstruction {
     /**
      * The opcode of this instruction.
      */
-    private String opcode;
+    private final String opcode;
 
     /**
      * All arguments of this instruction, in order.
      */
-    private List<String> args;
+    private final List<String> args;
+
+    /**
+     * The cached result of converting this GuacamoleInstruction to the format
+     * used by the Guacamole protocol.
+     */
+    private String protocolForm = null;
 
     /**
      * Creates a new GuacamoleInstruction having the given Operation and
@@ -91,31 +97,41 @@ public class GuacamoleInstruction {
      * Returns this GuacamoleInstruction in the form it would be sent over the
      * Guacamole protocol.
      *
-     * @return This GuacamoleInstruction in the form it would be sent over the
-     *         Guacamole protocol.
+     * @return
+     *     This GuacamoleInstruction in the form it would be sent over the
+     *     Guacamole protocol.
      */
     @Override
     public String toString() {
 
-        StringBuilder buff = new StringBuilder();
+        // Avoid rebuilding Guacamole protocol form of instruction if already
+        // known
+        if (protocolForm == null) {
 
-        // Write opcode
-        buff.append(opcode.length());
-        buff.append('.');
-        buff.append(opcode);
+            StringBuilder buff = new StringBuilder();
 
-        // Write argument values
-        for (String value : args) {
-            buff.append(',');
-            buff.append(value.length());
+            // Write opcode
+            buff.append(opcode.length());
             buff.append('.');
-            buff.append(value);
+            buff.append(opcode);
+
+            // Write argument values
+            for (String value : args) {
+                buff.append(',');
+                buff.append(value.length());
+                buff.append('.');
+                buff.append(value);
+            }
+
+            // Write terminator
+            buff.append(';');
+
+            // Cache result for future calls
+            protocolForm = buff.toString();
+
         }
 
-        // Write terminator
-        buff.append(';');
-
-        return buff.toString();
+        return protocolForm;
 
     }