safebox/guacamole-client
4
0
Fork 0
mirror of https://github.com/gyurix1968/guacamole-client.git synced 2025-09-06 05:07:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

GUACAMOLE-1876: Merge support for displaying a "points of interest" histogram alongside session recordings.

Browse Source
This commit is contained in:
Mike Jumper
2023-11-02 16:05:07 -07:00
committed by GitHub
parent ac6e501eff 6bd19b4714
commit d03da4a385
13 changed files with 534 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
doc/licenses/d3-path-3.1.0/LICENSE Normal file
View File

@@ -0,0 +1,13 @@
Copyright 2015-2022 Mike Bostock
Permission to use, copy, modify, and/or distribute this software for any purpose
with or without fee is hereby granted, provided that the above copyright notice
and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
THIS SOFTWARE.

8
doc/licenses/d3-path-3.1.0/README Normal file
View File

@@ -0,0 +1,8 @@
d3-path (https://github.com/d3/d3-path)
----------------------------------------------------------
Version: 3.1.0
From: 'Mike Bostock'
License(s):
BSD 0-clause (bundled/d3-path-3.1.0/LICENSE)

1
doc/licenses/d3-path-3.1.0/dep-coordinates.txt Normal file
View File

@@ -0,0 +1 @@
d3-path:3.1.0

13
doc/licenses/d3-shape-3.2.0/LICENSE Normal file
View File

@@ -0,0 +1,13 @@
Copyright 2010-2022 Mike Bostock
Permission to use, copy, modify, and/or distribute this software for any purpose
with or without fee is hereby granted, provided that the above copyright notice
and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
THIS SOFTWARE.

8
doc/licenses/d3-shape-3.2.0/README Normal file
View File

@@ -0,0 +1,8 @@
d3-path (https://github.com/d3/d3-shape)
----------------------------------------------------------
Version: 3.2.0
From: 'Mike Bostock'
License(s):
BSD 0-clause (bundled/d3-shape-3.2.0/LICENSE)

1
doc/licenses/d3-shape-3.2.0/dep-coordinates.txt Normal file
View File

@@ -0,0 +1 @@
d3-shape:3.2.0

34
guacamole/src/main/frontend/package-lock.json generated
View File

@@ -14,6 +14,8 @@
"angular-translate-loader-static-files": "^2.19.0",
"blob-polyfill": ">=7.0.20220408",
"csv": "^6.2.5",
"d3-path": "^3.1.0",
"d3-shape": "^3.2.0",
"datalist-polyfill": "^1.25.1",
"file-saver": "^2.0.5",
"fuzzysort": "^2.0.4",
@@ -4651,6 +4653,25 @@
"resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
"integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A=="
},
"node_modules/d3-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-shape": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
"dependencies": {
"d3-path": "^3.1.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/datalist-polyfill": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/datalist-polyfill/-/datalist-polyfill-1.25.1.tgz",
@@ -14937,6 +14958,19 @@
"resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
"integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A=="
},
"d3-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ=="
},
"d3-shape": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
"requires": {
"d3-path": "^3.1.0"
}
},
"datalist-polyfill": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/datalist-polyfill/-/datalist-polyfill-1.25.1.tgz",

2
guacamole/src/main/frontend/package.json
View File

@@ -13,6 +13,8 @@
"angular-translate-loader-static-files": "^2.19.0",
"blob-polyfill": ">=7.0.20220408",
"csv": "^6.2.5",
"d3-path": "^3.1.0",
"d3-shape": "^3.2.0",
"datalist-polyfill": "^1.25.1",
"file-saver": "^2.0.5",
"fuzzysort": "^2.0.4",

97
guacamole/src/main/frontend/src/app/player/directives/player.js
View File

@@ -81,6 +81,7 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
// Required services
const keyEventDisplayService = $injector.get('keyEventDisplayService');
const playerHeatmapService = $injector.get('playerHeatmapService');
const playerTimeService = $injector.get('playerTimeService');
/**
@@ -187,6 +188,65 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
*/
$scope.showKeyLog = false;
/**
* The height, in pixels, of the SVG heatmap paths. Note that this is not
* necessarily the actual rendered height, just the initial size of the
* SVG path before any styling is applied.
*
* @type {!number}
*/
$scope.HEATMAP_HEIGHT = 100;
/**
* The width, in pixels, of the SVG heatmap paths. Note that this is not
* necessarily the actual rendered width, just the initial size of the
* SVG path before any styling is applied.
*
* @type {!number}
*/
$scope.HEATMAP_WIDTH = 1000;
/**
* The maximum number of key events per millisecond to display in the
* key event heatmap. Any key event rates exceeding this value will be
* capped at this rate to ensure that unsually large spikes don't make
* swamp the rest of the data.
*
* Note: This is 6 keys per second (events include both presses and
* releases) - equivalent to ~88 words per minute typed.
*
* @type {!number}
*/
const KEY_EVENT_RATE_CAP = 12 / 1000;
/**
* The maximum number of frames per millisecond to display in the
* frame heatmap. Any frame rates exceeding this value will be
* capped at this rate to ensure that unsually large spikes don't make
* swamp the rest of the data.
*
* @type {!number}
*/
const FRAME_RATE_CAP = 10 / 1000;
/**
* An SVG path describing a smoothed curve that visualizes the relative
* number of frames rendered throughout the recording - i.e. a heatmap
* of screen updates.
*
* @type {!string}
*/
$scope.frameHeatmap = '';
/**
* An SVG path describing a smoothed curve that visualizes the relative
* number of key events recorded throughout the recording - i.e. a
* heatmap of key events.
*
* @type {!string}
*/
$scope.keyHeatmap = '';
/**
* Whether a seek request is currently in progress. A seek request is
* in progress if the user is attempting to change the current playback
@@ -213,6 +273,22 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
*/
var mouseActivityTimer = null;
/**
* The recording-relative timestamp of each frame of the recording that
* has been processed so far.
*
* @type {!number[]}
*/
var frameTimestamps = [];
/**
* The recording-relative timestamp of each text event that has been
* processed so far.
*
* @type {!number[]}
*/
var keyTimestamps = [];
/**
* Return true if any batches of key event logs are available for this
* recording, or false otherwise.
@@ -355,11 +431,25 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
// Begin downloading the recording
$scope.recording.connect();
// Notify listeners when the recording is completely loaded
// Notify listeners and set any heatmap paths
// when the recording is completely loaded
$scope.recording.onload = function recordingLoaded() {
$scope.operationMessage = null;
$scope.$emit('guacPlayerLoaded');
$scope.$evalAsync();
const recordingDuration = $scope.recording.getDuration();
// Generate heat maps for rendered frames and typed text
$scope.frameHeatmap = (
playerHeatmapService.generateHeatmapPath(
frameTimestamps, recordingDuration, FRAME_RATE_CAP,
$scope.HEATMAP_HEIGHT, $scope.HEATMAP_WIDTH));
$scope.keyHeatmap = (
playerHeatmapService.generateHeatmapPath(
keyTimestamps, recordingDuration, KEY_EVENT_RATE_CAP,
$scope.HEATMAP_HEIGHT, $scope.HEATMAP_WIDTH));
};
// Notify listeners if an error occurs
@@ -375,6 +465,9 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
$scope.operationProgress = src.size ? current / src.size : 0;
$scope.$emit('guacPlayerProgress', duration, current);
$scope.$evalAsync();
// Store the timestamp of the just-received frame
frameTimestamps.push(duration);
};
// Notify listeners when playback has started/resumed
@@ -396,6 +489,8 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
$scope.textBatches = (
keyEventDisplayService.parseEvents(events));
keyTimestamps = events.map(event => event.timestamp);
};
// Notify listeners when current position within the recording

264
guacamole/src/main/frontend/src/app/player/services/playerHeatmapService.js Normal file
View File

@@ -0,0 +1,264 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* 'License'); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { curveCatmullRom } from 'd3-shape';
import { path } from 'd3-path';
/**
* A service for generating heat maps of activity levels per time interval,
* for session recording playback.
*/
angular.module('player').factory('playerHeatmapService', [() => {
/**
* A default, relatively-gentle Gaussian smoothing kernel. This kernel
* should help heatmaps look a bit less jagged, while not reducing fidelity
* very much.
*
* @type {!number[]}
*/
const GAUSSIAN_KERNEL = [0.0013, 0.1573, 0.6827, 0.1573, 0.0013];
/**
* The number of buckets that a series of activity timestamps should be
* divided into.
*
* @type {!number}
*/
const NUM_BUCKETS = 100;
/**
* Given a list of values to smooth out, produce a smoothed data set with
* the same length as the original provided list.
*
* @param {!number[]} values
* The list of histogram values to smooth out.
*
* @returns {!number[]}
* The smoothed value array.
*/
function smooth(values) {
// The starting offset into the values array for each calculation
const lookBack = Math.floor(GAUSSIAN_KERNEL.length / 2);
// Apply the smoothing kernel to each value in the provided array
return _.map(values, (value, index) => {
// Total up the weighted values for each position in the kernel
return _.reduce(GAUSSIAN_KERNEL, (total, weight, kernelIndex) => {
// The offset into the original values array for the kernel
const valuesOffset = kernelIndex - lookBack;
// The position inside the original values array to be included
const valuesIndex = index + valuesOffset;
// If the contribution to the final smoothed value would be outside
// the bounds of the array, just use the original value instead
const contribution = ((valuesIndex >= 0) && valuesIndex < values.length)
? values[valuesIndex] : value;
// Use the provided weight from the kernel and add to the total
return total + (contribution * weight);
}, 0);
});
}
/**
* Given an array of values, with each value representing an activity count
* during a bucket of time, generate a smooth curve, scaled to PATH_HEIGHT
* height, and PATH_WIDTH width.
*
* @param {!number[]} bucketizedData
* The bucketized counts to create an SVG path from.
*
* @param {!number} maxBucketValue
* The size of the largest value in the bucketized data.
*
* @param {!number} height
* The target height, in pixels, of the highest point in the heatmap.
*
* @param {!number} width
* The target width, in pixels, of the heatmap.
*
* @returns {!string}
* An SVG path representing a smooth curve, passing through all points
* in the provided data.
*/
function createPath(bucketizedData, maxBucketValue, height, width) {
// Calculate scaling factor to ensure that paths are all the same heigh
const yScalingFactor = height / maxBucketValue;
// Scale a given Y value appropriately
const scaleYValue = yValue => height - (yValue * yScalingFactor);
// Calculate scaling factor to ensure that paths are all the same width
const xScalingFactor = width / bucketizedData.length;
// Construct a continuous curved path
const curvedPath = path();
const curve = curveCatmullRom(curvedPath);
curve.lineStart();
// Add all the data points
for (let i = 0; i < bucketizedData.length; i++) {
// Scale up the x value to hit the target width
const x = xScalingFactor * i;
// Scale and invert the height for display
const y = scaleYValue(bucketizedData[i]);
// Add the scaled point
curve.point(x, y);
}
// Move back to 0 to complete the path
curve.lineEnd();
curvedPath.lineTo(width, scaleYValue(0));
// Generate the SVG path for this curve
const rawPathText = curvedPath.toString();
// The SVG path as generated by D3 starts with a move to the first data
// point. This means that when the path ends and the subpath is closed,
// it returns to the position of the first data point instead of the
// origin. To fix this, the initial move command is removed, and the
// path is amended to start at the origin. TODO: Find a better way to
// handle this.
const startAtOrigin = (
// Start at origin
'M0,' + scaleYValue(0) +
// Line to the first point in the curve, to close the shape
'L0,' + scaleYValue(bucketizedData[0])
);
// Strip off the first move command from the path
const strippedPathText = _.replace(rawPathText, /^[^C]*/, '');
return startAtOrigin + strippedPathText;
}
const service = {};
/**
* Given a raw array of timestamps indicating when events of a certain type
* occured during a record, generate and return a smoothed SVG path
* indicating how many events occured during each equal-length bucket.
*
* @param {!number[]} timestamps
* A raw array of timestamps, one for every relevant event. These
* must be monotonically increasing.
*
* @param {!number} duration
* The duration over which the heatmap should apply. This value may
* be greater than the maximum timestamp value, in which case the path
* will drop to 0 after the last timestamp in the provided array.
*
* @param {number} maxRate
* The maximum number of events per millisecond that should be displayed
* in the final path. Any rates over this amount will just be capped at
* this value.
*
* @param {!number} height
* The target height, in pixels, of the highest point in the heatmap.
*
* @param {!number} width
* The target width, in pixels, of the heatmap.
*
* @returns {!string}
* A smoothed, graphable SVG path representing levels of activity over
* time, as extracted from the provided timestamps.
*/
service.generateHeatmapPath = (timestamps, duration, maxRate, height, width) => {
// The height and width must both be valid in order to create the path
if (!height || !width) {
console.warn("Heatmap height and width must be positive.");
return '';
}
// If no timestamps are available, no path can be created
if (!timestamps || !timestamps.length)
return '';
// An initially empty array containing no activity in any bucket
const buckets = new Array(NUM_BUCKETS).fill(0);
// If no events occured, return the an empty path
if (!timestamps.length)
return '';
// Determine the bucket granularity
const bucketDuration = duration / NUM_BUCKETS;
// The rate-limited maximum number of events that any bucket can have,
const maxPossibleBucketValue = Math.floor(bucketDuration * maxRate);
// If the duration is invalid, return the still-empty array
if (duration <= 0)
return '';
let maxBucketValue = 0;
// Partition the events into a count of events per bucket
let currentBucketIndex = 0;
timestamps.forEach(timestamp => {
// If the current timestamp has passed the end of the current
// bucket, move to the appropriate bucket
if (timestamp >= (currentBucketIndex + 1) * bucketDuration)
currentBucketIndex = Math.min(
Math.floor((timestamp / bucketDuration)), NUM_BUCKETS - 1);
// Do not record events that exceed the maximum allowable rate
if (buckets[currentBucketIndex] >= maxPossibleBucketValue)
buckets[currentBucketIndex] = maxPossibleBucketValue;
else
// Increment the count for the current bucket
buckets[currentBucketIndex]++;
// Keep track of the maximum value seen so far
maxBucketValue = Math.max(
maxBucketValue, buckets[currentBucketIndex]);
});
// Smooth the data for better aesthetics before creating the path
const smoothed = smooth(buckets);
// Create an SVG path based on the smoothed data
return createPath(smoothed, maxBucketValue, height, width);
}
return service;
}]);

15
guacamole/src/main/frontend/src/app/player/templates/player.html
View File

@@ -28,6 +28,21 @@
ng-model="playbackPosition"
ng-on-change="commitSeekRequest()">
<div class="heat-map">
<svg class="frame-events" ng-attr-view_box="0 0 {{ HEATMAP_WIDTH }} {{ HEATMAP_HEIGHT }}" preserveAspectRatio="none">
<path ng-attr-d="{{ frameHeatmap }}"/>
</svg>
<svg class="key-events" ng-attr-view_box="0 0 {{ HEATMAP_WIDTH }} {{ HEATMAP_HEIGHT }}" preserveAspectRatio="none">
<path ng-attr-d="{{ keyHeatmap }}"/>
</svg>
<div class="legend">
<span class="frame-events">{{ 'PLAYER.INFO_FRAME_EVENTS_LEGEND' | translate }}</span>
<span class="key-events">{{ 'PLAYER.INFO_KEY_EVENTS_LEGEND' | translate }}</span>
</div>
</div>
<div class="guac-player-buttons">
<!-- Play button -->

73
guacamole/src/main/frontend/src/app/settings/styles/history-player.css
View File

@@ -116,3 +116,76 @@
.settings.connectionHistoryPlayer guac-player.recent-mouse-movement .guac-player-controls.playing {
opacity: 1;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map {
position: absolute;
width: 100%;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map svg {
position: absolute;
bottom: 7px;
height: 50px;
width: 100%;
z-index: 100;
pointer-events: none;
opacity: 0;
-webkit-transition: opacity 0.1s linear 0.1s;
-moz-transition: opacity 0.1s linear 0.1s;
-o-transition: opacity 0.1s linear 0.1s;
transition: opacity 0.1s linear 0.1s;
}
.settings.connectionHistoryPlayer .guac-player-controls:hover .heat-map svg {
opacity: 0.5;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend {
position: absolute;
display: flex;
flex-direction: column;
align-items: end;
bottom: 65px;
right: 10px;
z-index: 100;
opacity: 0;
-webkit-transition: opacity 0.1s linear 0.1s;
-moz-transition: opacity 0.1s linear 0.1s;
-o-transition: opacity 0.1s linear 0.1s;
transition: opacity 0.1s linear 0.1s;
}
.settings.connectionHistoryPlayer .guac-player-controls:hover .heat-map .legend {
opacity: 1;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .key-events::after,
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .frame-events::after {
display: inline-block;
content: '';
width: 25px;
height: 10px;
margin-left: 3px;
margin-right: 10px;
border-radius: 3px;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .key-events::after {
background-color: #5BA300;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .frame-events::after {
background-color: #FFFFFF;
}
.settings.connectionHistoryPlayer .heat-map svg.key-events {
/* Convert to #5BA300 color */
filter: invert(69%) sepia(80%) saturate(5092%) hue-rotate(54deg) brightness(96%) contrast(101%);
}
.settings.connectionHistoryPlayer .heat-map svg.frame-events {
/* Convert to #FFFFFF color */
filter: invert(100%) sepia(0%) saturate(0%) hue-rotate(93deg) brightness(103%) contrast(103%);
}

10
guacamole/src/main/frontend/src/translations/en.json
View File

@@ -483,10 +483,12 @@
"ACTION_PLAY" : "@:APP.ACTION_PLAY",
"ACTION_SHOW_KEY_LOG" : "Keystroke Log",
"INFO_LOADING_RECORDING" : "Your recording is now being loaded. Please wait...",
"INFO_NO_KEY_LOG" : "Keystroke Log Unavailable",
"INFO_NUMBER_OF_RESULTS" : "{RESULTS} {RESULTS, plural, one{Match} other{Matches}}",
"INFO_SEEK_IN_PROGRESS" : "Seeking to the requested position. Please wait...",
"INFO_FRAME_EVENTS_LEGEND" : "On-screen Activity",
"INFO_KEY_EVENTS_LEGEND" : "Keyboard Activity",
"INFO_LOADING_RECORDING" : "Your recording is now being loaded. Please wait...",
"INFO_NO_KEY_LOG" : "Keystroke Log Unavailable",
"INFO_NUMBER_OF_RESULTS" : "{RESULTS} {RESULTS, plural, one{Match} other{Matches}}",
"INFO_SEEK_IN_PROGRESS" : "Seeking to the requested position. Please wait...",
"FIELD_PLACEHOLDER_TEXT_BATCH_FILTER" : "@:APP.FIELD_PLACEHOLDER_FILTER"