GUACAMOLE-1253: Merge batching of large SQL queries.

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-base/src/main/java/org/apache/guacamole/auth/jdbc/JDBCEnvironment.java

@@ -67,6 +67,19 @@ public abstract class JDBCEnvironment extends DelegatingEnvironment {
      */
     public abstract int getAbsoluteMaxConnections() throws GuacamoleException;
 
+    /**
+     * Returns the maximum number of identifiers/parameters to be 
+     * included in a single batch when executing SQL statements.
+     *
+     * @return
+     *     The maximum number of identifiers/parameters to be included 
+     *     in a single batch.
+     *
+     * @throws GuacamoleException
+     *     If an error occurs while retrieving the property.
+     */
+    public abstract int getBatchSize() throws GuacamoleException;
+
     /**
      * Returns the default maximum number of concurrent connections to allow to
      * any one connection, unless specified differently on an individual

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-base/src/main/java/org/apache/guacamole/auth/jdbc/base/ModeledDirectoryObjectService.java

@@ -19,13 +19,18 @@
 
 package org.apache.guacamole.auth.jdbc.base;
 
+import com.google.common.collect.Lists;
+import com.google.inject.Inject;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 import org.apache.guacamole.auth.jdbc.user.ModeledAuthenticatedUser;
 import org.apache.guacamole.GuacamoleException;
 import org.apache.guacamole.GuacamoleSecurityException;
+import org.apache.guacamole.auth.jdbc.JDBCEnvironment;
 import org.apache.guacamole.auth.jdbc.permission.ObjectPermissionMapper;
 import org.apache.guacamole.auth.jdbc.permission.ObjectPermissionModel;
 import org.apache.guacamole.auth.jdbc.user.UserModel;
@@ -66,6 +71,12 @@ public abstract class ModeledDirectoryObjectService<InternalType extends Modeled
         ObjectPermission.Type.ADMINISTER
     };
     
+    /**
+     * The environment of the Guacamole server.
+     */
+    @Inject
+    private JDBCEnvironment environment;
+    
     /**
      * Returns an instance of a mapper for the type of object used by this
      * service.
@@ -347,10 +358,10 @@ public abstract class ModeledDirectoryObjectService<InternalType extends Modeled
      *     A new collection containing only the strings within the provided
      *     collection which are valid identifiers.
      */
-    protected Collection<String> filterIdentifiers(Collection<String> identifiers) {
+    protected List<String> filterIdentifiers(Collection<String> identifiers) {
 
         // Obtain enough space for a full copy of the given identifiers
-        Collection<String> validIdentifiers = new ArrayList<String>(identifiers.size());
+        List<String> validIdentifiers = new ArrayList<>(identifiers.size());
 
         // Add only valid identifiers to the copy
         for (String identifier : identifiers) {
@@ -387,26 +398,36 @@ public abstract class ModeledDirectoryObjectService<InternalType extends Modeled
             Collection<String> identifiers) throws GuacamoleException {
 
         // Ignore invalid identifiers
-        identifiers = filterIdentifiers(identifiers);
+        List<String> filteredIdentifiers = filterIdentifiers(identifiers);
 
         // Do not query if no identifiers given
-        if (identifiers.isEmpty())
+        if (filteredIdentifiers.isEmpty())
             return Collections.<InternalType>emptyList();
 
-        Collection<ModelType> objects;
+        int batchSize = environment.getBatchSize();
 
-        // Bypass permission checks if the user is privileged
+        boolean userIsPrivileged = user.isPrivileged();
-        if (user.isPrivileged())
-            objects = getObjectMapper().select(identifiers);
 
-        // Otherwise only return explicitly readable identifiers
+        // Process the filteredIdentifiers in batches using Lists.partition() and flatMap
-        else
+        Collection<ModelType> allObjects = Lists.partition(filteredIdentifiers, batchSize).stream()
-            objects = getObjectMapper().selectReadable(user.getUser().getModel(),
+                .flatMap(chunk -> {
-                    identifiers, user.getEffectiveUserGroups());
+                    Collection<ModelType> objects;
+
+                    // Bypass permission checks if the user is privileged
+                    if (userIsPrivileged)
+                        objects = getObjectMapper().select(chunk);
+
+                    // Otherwise only return explicitly readable identifiers
+                    else
+                        objects = getObjectMapper().selectReadable(user.getUser().getModel(),
+                                chunk, user.getEffectiveUserGroups());
+
+                    return objects.stream();
+                })
+                .collect(Collectors.toList());
 
         // Return collection of requested objects
-        return getObjectInstances(user, objects);
+        return getObjectInstances(user, allObjects);
-        
     }
 
     /**

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-mysql/src/main/java/org/apache/guacamole/auth/mysql/conf/MySQLEnvironment.java

@@ -105,6 +105,20 @@ public class MySQLEnvironment extends JDBCEnvironment {
      */
     private final MySQLSSLMode DEFAULT_SSL_MODE = MySQLSSLMode.PREFERRED;
     
+    /**
+     * The default maximum number of identifiers/parameters to be included in a 
+     * single batch when executing SQL statements for MySQL and MariaDB.
+     * 
+     * MySQL and MariaDB impose a limit on the maximum size of a query, 
+     * determined by the max_allowed_packet configuration variable. A value of 
+     * 1000 is chosen to accommodate the max_allowed_packet limit without 
+     * exceeding it.
+     *
+     * @see https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet
+     * @see https://mariadb.com/kb/en/server-system-variables/#max_allowed_packet
+     */
+    private static final int DEFAULT_BATCH_SIZE = 1000;
+
     /**
      * Constructs a new MySQLEnvironment, providing access to MySQL-specific
      * configuration options.
@@ -135,6 +149,13 @@ public class MySQLEnvironment extends JDBCEnvironment {
         );
     }
     
+    @Override
+    public int getBatchSize() throws GuacamoleException {
+        return getProperty(MySQLGuacamoleProperties.MYSQL_BATCH_SIZE,
+            DEFAULT_BATCH_SIZE
+        );
+    }
+
     @Override
     public int getDefaultMaxConnections() throws GuacamoleException {
         return getProperty(

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-mysql/src/main/java/org/apache/guacamole/auth/mysql/conf/MySQLGuacamoleProperties.java

@@ -291,4 +291,16 @@ public class MySQLGuacamoleProperties {
 
     };
 
+    /**
+     * The maximum number of identifiers/parameters to be included in a single batch when
+     * executing SQL statements.
+     */
+    public static final IntegerGuacamoleProperty MYSQL_BATCH_SIZE =
+            new IntegerGuacamoleProperty() {
+
+        @Override
+        public String getName() { return "mysql-batch-size"; }
+
+    };    
+    
 }

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-postgresql/src/main/java/org/apache/guacamole/auth/postgresql/conf/PostgreSQLEnvironment.java

@@ -116,6 +116,18 @@ public class PostgreSQLEnvironment extends JDBCEnvironment {
      */
     private final PostgreSQLSSLMode DEFAULT_SSL_MODE = PostgreSQLSSLMode.PREFER;
     
+    /**
+     * The default maximum number of identifiers/parameters to be included in a 
+     * single batch when executing SQL statements for PostgreSQL.
+     * 
+     * PostgreSQL has a maximum limit of 65535 parameters per prepared statement.
+     * A value of 5000 is chosen to avoid potential performance issues or query
+     * execution errors while staying well below the maximum limit.
+     *
+     * @see https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAX-PREPARED-STATEMENT-ARGS
+     */
+    private static final int DEFAULT_BATCH_SIZE = 5000;
+
     /**
      * Constructs a new PostgreSQLEnvironment, providing access to PostgreSQL-specific
      * configuration options.
@@ -146,6 +158,13 @@ public class PostgreSQLEnvironment extends JDBCEnvironment {
         );
     }
 
+    @Override
+    public int getBatchSize() throws GuacamoleException {
+        return getProperty(PostgreSQLGuacamoleProperties.POSTGRESQL_BATCH_SIZE,
+            DEFAULT_BATCH_SIZE
+        );
+    }    
+    
     @Override
     public int getDefaultMaxConnections() throws GuacamoleException {
         return getProperty(

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-postgresql/src/main/java/org/apache/guacamole/auth/postgresql/conf/PostgreSQLGuacamoleProperties.java

@@ -303,4 +303,16 @@ public class PostgreSQLGuacamoleProperties {
 
     };
     
+    /**
+     * The maximum number of identifiers/parameters to be included in a single batch when
+     * executing SQL statements.
+     */
+    public static final IntegerGuacamoleProperty POSTGRESQL_BATCH_SIZE =
+            new IntegerGuacamoleProperty() {
+
+        @Override
+        public String getName() { return "postgresql-batch-size"; }
+
+    };
+
 }

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-sqlserver/src/main/java/org/apache/guacamole/auth/sqlserver/conf/SQLServerEnvironment.java

@@ -88,6 +88,17 @@ public class SQLServerEnvironment extends JDBCEnvironment {
      */
     public static final SQLServerDriver SQLSERVER_DEFAULT_DRIVER = SQLServerDriver.MICROSOFT_2005;
 
+    /**
+     * The default maximum number of identifiers/parameters to be included in a 
+     * single batch when executing SQL statements for SQL Server.
+     * 
+     * SQL Server supports a maximum of 2100 parameters per query. A value of
+     * 1000 is chosen to stay within this limit and avoid query execution errors.
+     *
+     * @see https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server
+     */
+    private static final int DEFAULT_BATCH_SIZE = 1000;
+    
     /**
      * Constructs a new SQLServerEnvironment, providing access to SQLServer-specific
      * configuration options.
@@ -118,6 +129,13 @@ public class SQLServerEnvironment extends JDBCEnvironment {
         );
     }
     
+    @Override
+    public int getBatchSize() throws GuacamoleException {
+        return getProperty(SQLServerGuacamoleProperties.SQLSERVER_BATCH_SIZE,
+            DEFAULT_BATCH_SIZE
+        );
+    }    
+
     @Override
     public int getDefaultMaxConnections() throws GuacamoleException {
         return getProperty(

extensions/guacamole-auth-jdbc/modules/guacamole-auth-jdbc-sqlserver/src/main/java/org/apache/guacamole/auth/sqlserver/conf/SQLServerGuacamoleProperties.java

@@ -233,4 +233,16 @@ public class SQLServerGuacamoleProperties {
 
     };
     
+    /**
+     * The maximum number of identifiers/parameters to be included in a single batch when
+     * executing SQL statements.
+     */
+    public static final IntegerGuacamoleProperty SQLSERVER_BATCH_SIZE =
+            new IntegerGuacamoleProperty() {
+
+        @Override
+        public String getName() { return "sqlserver-batch-size"; }
+
+    };
+
 }