GUACAMOLE-1876: Display points of interest heatmap in history recording player.

This commit is contained in:
James Muehlner
2023-10-25 22:36:48 +00:00
parent ac6e501eff
commit 6bd19b4714
13 changed files with 534 additions and 5 deletions

View File

@@ -14,6 +14,8 @@
"angular-translate-loader-static-files": "^2.19.0",
"blob-polyfill": ">=7.0.20220408",
"csv": "^6.2.5",
"d3-path": "^3.1.0",
"d3-shape": "^3.2.0",
"datalist-polyfill": "^1.25.1",
"file-saver": "^2.0.5",
"fuzzysort": "^2.0.4",
@@ -4651,6 +4653,25 @@
"resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
"integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A=="
},
"node_modules/d3-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-shape": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
"dependencies": {
"d3-path": "^3.1.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/datalist-polyfill": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/datalist-polyfill/-/datalist-polyfill-1.25.1.tgz",
@@ -14937,6 +14958,19 @@
"resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
"integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A=="
},
"d3-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ=="
},
"d3-shape": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
"requires": {
"d3-path": "^3.1.0"
}
},
"datalist-polyfill": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/datalist-polyfill/-/datalist-polyfill-1.25.1.tgz",

View File

@@ -13,6 +13,8 @@
"angular-translate-loader-static-files": "^2.19.0",
"blob-polyfill": ">=7.0.20220408",
"csv": "^6.2.5",
"d3-path": "^3.1.0",
"d3-shape": "^3.2.0",
"datalist-polyfill": "^1.25.1",
"file-saver": "^2.0.5",
"fuzzysort": "^2.0.4",

View File

@@ -81,6 +81,7 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
// Required services
const keyEventDisplayService = $injector.get('keyEventDisplayService');
const playerHeatmapService = $injector.get('playerHeatmapService');
const playerTimeService = $injector.get('playerTimeService');
/**
@@ -187,6 +188,65 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
*/
$scope.showKeyLog = false;
/**
* The height, in pixels, of the SVG heatmap paths. Note that this is not
* necessarily the actual rendered height, just the initial size of the
* SVG path before any styling is applied.
*
* @type {!number}
*/
$scope.HEATMAP_HEIGHT = 100;
/**
* The width, in pixels, of the SVG heatmap paths. Note that this is not
* necessarily the actual rendered width, just the initial size of the
* SVG path before any styling is applied.
*
* @type {!number}
*/
$scope.HEATMAP_WIDTH = 1000;
/**
* The maximum number of key events per millisecond to display in the
* key event heatmap. Any key event rates exceeding this value will be
* capped at this rate to ensure that unsually large spikes don't make
* swamp the rest of the data.
*
* Note: This is 6 keys per second (events include both presses and
* releases) - equivalent to ~88 words per minute typed.
*
* @type {!number}
*/
const KEY_EVENT_RATE_CAP = 12 / 1000;
/**
* The maximum number of frames per millisecond to display in the
* frame heatmap. Any frame rates exceeding this value will be
* capped at this rate to ensure that unsually large spikes don't make
* swamp the rest of the data.
*
* @type {!number}
*/
const FRAME_RATE_CAP = 10 / 1000;
/**
* An SVG path describing a smoothed curve that visualizes the relative
* number of frames rendered throughout the recording - i.e. a heatmap
* of screen updates.
*
* @type {!string}
*/
$scope.frameHeatmap = '';
/**
* An SVG path describing a smoothed curve that visualizes the relative
* number of key events recorded throughout the recording - i.e. a
* heatmap of key events.
*
* @type {!string}
*/
$scope.keyHeatmap = '';
/**
* Whether a seek request is currently in progress. A seek request is
* in progress if the user is attempting to change the current playback
@@ -213,6 +273,22 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
*/
var mouseActivityTimer = null;
/**
* The recording-relative timestamp of each frame of the recording that
* has been processed so far.
*
* @type {!number[]}
*/
var frameTimestamps = [];
/**
* The recording-relative timestamp of each text event that has been
* processed so far.
*
* @type {!number[]}
*/
var keyTimestamps = [];
/**
* Return true if any batches of key event logs are available for this
* recording, or false otherwise.
@@ -355,11 +431,25 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
// Begin downloading the recording
$scope.recording.connect();
// Notify listeners when the recording is completely loaded
// Notify listeners and set any heatmap paths
// when the recording is completely loaded
$scope.recording.onload = function recordingLoaded() {
$scope.operationMessage = null;
$scope.$emit('guacPlayerLoaded');
$scope.$evalAsync();
const recordingDuration = $scope.recording.getDuration();
// Generate heat maps for rendered frames and typed text
$scope.frameHeatmap = (
playerHeatmapService.generateHeatmapPath(
frameTimestamps, recordingDuration, FRAME_RATE_CAP,
$scope.HEATMAP_HEIGHT, $scope.HEATMAP_WIDTH));
$scope.keyHeatmap = (
playerHeatmapService.generateHeatmapPath(
keyTimestamps, recordingDuration, KEY_EVENT_RATE_CAP,
$scope.HEATMAP_HEIGHT, $scope.HEATMAP_WIDTH));
};
// Notify listeners if an error occurs
@@ -375,6 +465,9 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
$scope.operationProgress = src.size ? current / src.size : 0;
$scope.$emit('guacPlayerProgress', duration, current);
$scope.$evalAsync();
// Store the timestamp of the just-received frame
frameTimestamps.push(duration);
};
// Notify listeners when playback has started/resumed
@@ -396,6 +489,8 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
$scope.textBatches = (
keyEventDisplayService.parseEvents(events));
keyTimestamps = events.map(event => event.timestamp);
};
// Notify listeners when current position within the recording

View File

@@ -0,0 +1,264 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* 'License'); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { curveCatmullRom } from 'd3-shape';
import { path } from 'd3-path';
/**
* A service for generating heat maps of activity levels per time interval,
* for session recording playback.
*/
angular.module('player').factory('playerHeatmapService', [() => {
/**
* A default, relatively-gentle Gaussian smoothing kernel. This kernel
* should help heatmaps look a bit less jagged, while not reducing fidelity
* very much.
*
* @type {!number[]}
*/
const GAUSSIAN_KERNEL = [0.0013, 0.1573, 0.6827, 0.1573, 0.0013];
/**
* The number of buckets that a series of activity timestamps should be
* divided into.
*
* @type {!number}
*/
const NUM_BUCKETS = 100;
/**
* Given a list of values to smooth out, produce a smoothed data set with
* the same length as the original provided list.
*
* @param {!number[]} values
* The list of histogram values to smooth out.
*
* @returns {!number[]}
* The smoothed value array.
*/
function smooth(values) {
// The starting offset into the values array for each calculation
const lookBack = Math.floor(GAUSSIAN_KERNEL.length / 2);
// Apply the smoothing kernel to each value in the provided array
return _.map(values, (value, index) => {
// Total up the weighted values for each position in the kernel
return _.reduce(GAUSSIAN_KERNEL, (total, weight, kernelIndex) => {
// The offset into the original values array for the kernel
const valuesOffset = kernelIndex - lookBack;
// The position inside the original values array to be included
const valuesIndex = index + valuesOffset;
// If the contribution to the final smoothed value would be outside
// the bounds of the array, just use the original value instead
const contribution = ((valuesIndex >= 0) && valuesIndex < values.length)
? values[valuesIndex] : value;
// Use the provided weight from the kernel and add to the total
return total + (contribution * weight);
}, 0);
});
}
/**
* Given an array of values, with each value representing an activity count
* during a bucket of time, generate a smooth curve, scaled to PATH_HEIGHT
* height, and PATH_WIDTH width.
*
* @param {!number[]} bucketizedData
* The bucketized counts to create an SVG path from.
*
* @param {!number} maxBucketValue
* The size of the largest value in the bucketized data.
*
* @param {!number} height
* The target height, in pixels, of the highest point in the heatmap.
*
* @param {!number} width
* The target width, in pixels, of the heatmap.
*
* @returns {!string}
* An SVG path representing a smooth curve, passing through all points
* in the provided data.
*/
function createPath(bucketizedData, maxBucketValue, height, width) {
// Calculate scaling factor to ensure that paths are all the same heigh
const yScalingFactor = height / maxBucketValue;
// Scale a given Y value appropriately
const scaleYValue = yValue => height - (yValue * yScalingFactor);
// Calculate scaling factor to ensure that paths are all the same width
const xScalingFactor = width / bucketizedData.length;
// Construct a continuous curved path
const curvedPath = path();
const curve = curveCatmullRom(curvedPath);
curve.lineStart();
// Add all the data points
for (let i = 0; i < bucketizedData.length; i++) {
// Scale up the x value to hit the target width
const x = xScalingFactor * i;
// Scale and invert the height for display
const y = scaleYValue(bucketizedData[i]);
// Add the scaled point
curve.point(x, y);
}
// Move back to 0 to complete the path
curve.lineEnd();
curvedPath.lineTo(width, scaleYValue(0));
// Generate the SVG path for this curve
const rawPathText = curvedPath.toString();
// The SVG path as generated by D3 starts with a move to the first data
// point. This means that when the path ends and the subpath is closed,
// it returns to the position of the first data point instead of the
// origin. To fix this, the initial move command is removed, and the
// path is amended to start at the origin. TODO: Find a better way to
// handle this.
const startAtOrigin = (
// Start at origin
'M0,' + scaleYValue(0) +
// Line to the first point in the curve, to close the shape
'L0,' + scaleYValue(bucketizedData[0])
);
// Strip off the first move command from the path
const strippedPathText = _.replace(rawPathText, /^[^C]*/, '');
return startAtOrigin + strippedPathText;
}
const service = {};
/**
* Given a raw array of timestamps indicating when events of a certain type
* occured during a record, generate and return a smoothed SVG path
* indicating how many events occured during each equal-length bucket.
*
* @param {!number[]} timestamps
* A raw array of timestamps, one for every relevant event. These
* must be monotonically increasing.
*
* @param {!number} duration
* The duration over which the heatmap should apply. This value may
* be greater than the maximum timestamp value, in which case the path
* will drop to 0 after the last timestamp in the provided array.
*
* @param {number} maxRate
* The maximum number of events per millisecond that should be displayed
* in the final path. Any rates over this amount will just be capped at
* this value.
*
* @param {!number} height
* The target height, in pixels, of the highest point in the heatmap.
*
* @param {!number} width
* The target width, in pixels, of the heatmap.
*
* @returns {!string}
* A smoothed, graphable SVG path representing levels of activity over
* time, as extracted from the provided timestamps.
*/
service.generateHeatmapPath = (timestamps, duration, maxRate, height, width) => {
// The height and width must both be valid in order to create the path
if (!height || !width) {
console.warn("Heatmap height and width must be positive.");
return '';
}
// If no timestamps are available, no path can be created
if (!timestamps || !timestamps.length)
return '';
// An initially empty array containing no activity in any bucket
const buckets = new Array(NUM_BUCKETS).fill(0);
// If no events occured, return the an empty path
if (!timestamps.length)
return '';
// Determine the bucket granularity
const bucketDuration = duration / NUM_BUCKETS;
// The rate-limited maximum number of events that any bucket can have,
const maxPossibleBucketValue = Math.floor(bucketDuration * maxRate);
// If the duration is invalid, return the still-empty array
if (duration <= 0)
return '';
let maxBucketValue = 0;
// Partition the events into a count of events per bucket
let currentBucketIndex = 0;
timestamps.forEach(timestamp => {
// If the current timestamp has passed the end of the current
// bucket, move to the appropriate bucket
if (timestamp >= (currentBucketIndex + 1) * bucketDuration)
currentBucketIndex = Math.min(
Math.floor((timestamp / bucketDuration)), NUM_BUCKETS - 1);
// Do not record events that exceed the maximum allowable rate
if (buckets[currentBucketIndex] >= maxPossibleBucketValue)
buckets[currentBucketIndex] = maxPossibleBucketValue;
else
// Increment the count for the current bucket
buckets[currentBucketIndex]++;
// Keep track of the maximum value seen so far
maxBucketValue = Math.max(
maxBucketValue, buckets[currentBucketIndex]);
});
// Smooth the data for better aesthetics before creating the path
const smoothed = smooth(buckets);
// Create an SVG path based on the smoothed data
return createPath(smoothed, maxBucketValue, height, width);
}
return service;
}]);

View File

@@ -28,6 +28,21 @@
ng-model="playbackPosition"
ng-on-change="commitSeekRequest()">
<div class="heat-map">
<svg class="frame-events" ng-attr-view_box="0 0 {{ HEATMAP_WIDTH }} {{ HEATMAP_HEIGHT }}" preserveAspectRatio="none">
<path ng-attr-d="{{ frameHeatmap }}"/>
</svg>
<svg class="key-events" ng-attr-view_box="0 0 {{ HEATMAP_WIDTH }} {{ HEATMAP_HEIGHT }}" preserveAspectRatio="none">
<path ng-attr-d="{{ keyHeatmap }}"/>
</svg>
<div class="legend">
<span class="frame-events">{{ 'PLAYER.INFO_FRAME_EVENTS_LEGEND' | translate }}</span>
<span class="key-events">{{ 'PLAYER.INFO_KEY_EVENTS_LEGEND' | translate }}</span>
</div>
</div>
<div class="guac-player-buttons">
<!-- Play button -->

View File

@@ -116,3 +116,76 @@
.settings.connectionHistoryPlayer guac-player.recent-mouse-movement .guac-player-controls.playing {
opacity: 1;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map {
position: absolute;
width: 100%;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map svg {
position: absolute;
bottom: 7px;
height: 50px;
width: 100%;
z-index: 100;
pointer-events: none;
opacity: 0;
-webkit-transition: opacity 0.1s linear 0.1s;
-moz-transition: opacity 0.1s linear 0.1s;
-o-transition: opacity 0.1s linear 0.1s;
transition: opacity 0.1s linear 0.1s;
}
.settings.connectionHistoryPlayer .guac-player-controls:hover .heat-map svg {
opacity: 0.5;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend {
position: absolute;
display: flex;
flex-direction: column;
align-items: end;
bottom: 65px;
right: 10px;
z-index: 100;
opacity: 0;
-webkit-transition: opacity 0.1s linear 0.1s;
-moz-transition: opacity 0.1s linear 0.1s;
-o-transition: opacity 0.1s linear 0.1s;
transition: opacity 0.1s linear 0.1s;
}
.settings.connectionHistoryPlayer .guac-player-controls:hover .heat-map .legend {
opacity: 1;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .key-events::after,
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .frame-events::after {
display: inline-block;
content: '';
width: 25px;
height: 10px;
margin-left: 3px;
margin-right: 10px;
border-radius: 3px;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .key-events::after {
background-color: #5BA300;
}
.settings.connectionHistoryPlayer .guac-player-controls .heat-map .legend .frame-events::after {
background-color: #FFFFFF;
}
.settings.connectionHistoryPlayer .heat-map svg.key-events {
/* Convert to #5BA300 color */
filter: invert(69%) sepia(80%) saturate(5092%) hue-rotate(54deg) brightness(96%) contrast(101%);
}
.settings.connectionHistoryPlayer .heat-map svg.frame-events {
/* Convert to #FFFFFF color */
filter: invert(100%) sepia(0%) saturate(0%) hue-rotate(93deg) brightness(103%) contrast(103%);
}

View File

@@ -483,10 +483,12 @@
"ACTION_PLAY" : "@:APP.ACTION_PLAY",
"ACTION_SHOW_KEY_LOG" : "Keystroke Log",
"INFO_LOADING_RECORDING" : "Your recording is now being loaded. Please wait...",
"INFO_NO_KEY_LOG" : "Keystroke Log Unavailable",
"INFO_NUMBER_OF_RESULTS" : "{RESULTS} {RESULTS, plural, one{Match} other{Matches}}",
"INFO_SEEK_IN_PROGRESS" : "Seeking to the requested position. Please wait...",
"INFO_FRAME_EVENTS_LEGEND" : "On-screen Activity",
"INFO_KEY_EVENTS_LEGEND" : "Keyboard Activity",
"INFO_LOADING_RECORDING" : "Your recording is now being loaded. Please wait...",
"INFO_NO_KEY_LOG" : "Keystroke Log Unavailable",
"INFO_NUMBER_OF_RESULTS" : "{RESULTS} {RESULTS, plural, one{Match} other{Matches}}",
"INFO_SEEK_IN_PROGRESS" : "Seeking to the requested position. Please wait...",
"FIELD_PLACEHOLDER_TEXT_BATCH_FILTER" : "@:APP.FIELD_PLACEHOLDER_FILTER"