deploy: current vibn theia state

Made-with: Cursor

scripts/performance/.gitignore

@@ -0,0 +1,6 @@
+profiles
+workspace
+*.csv
+*.json
+!base-package.json
+!electron-trace-config.json

scripts/performance/README.md

@@ -0,0 +1,102 @@
+# Performance measurements
+
+This directory contains scripts that measure the start-up performance of the Theia frontend in both the browser and the Electron examples.
+
+The frontend's start-up time is measured using the timestamp of the last recorded `Largest contentful paint (LCP)` candidate metric.
+
+## Running the browser start-up script
+
+### Quick Start
+
+Execute `npm run performance:startup:browser` in the root directory to startup the backend and execute the script.
+
+### Prerequisites
+
+To run the script the Theia backend needs to be started.
+This can either be done with the `Launch Browser Backend` launch config or by running `npm run start` in the `examples/browser-app` directory.
+
+### Executing the script
+
+The script can be executed using `node browser-performance.js` in this directory.
+
+The script accepts the following optional parameters:
+
+- `--name`: Specify a name for the current measurement (default: `Browser Frontend Startup`)
+- `--url`: Point Theia to a url for example for specifying a specific workspace (default: `http://localhost:3000/#/<pathToMeasurementScript>/workspace`)
+- `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `browser`)
+- `--runs`: Number of runs for the measurement (default: `10`)
+- `--headless`: Boolean, if the tests should be run in headless mode (default: `true`)
+
+_**Note**: When multiple runs are specified the script will calculate the mean and the standard deviation of all values._
+
+## Running the Electron start-up script
+
+### Quick Start
+
+Execute `yarn run performance:startup:electron` in the root directory to execute the script.
+
+### Prerequisites
+
+To run the script the Theia Electron example needs to be built. In the root directory:
+
+```console
+npm install
+npm run build:electron
+```
+
+### Executing the script
+
+The script can be executed using `node electron-performance.js` in this directory.
+
+The script accepts the following optional parameters:
+
+- `--name`: Specify a name for the current measurement (default: `Electron Frontend Startup`)
+- `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `electron`)
+- `--workspace`: Absolute path to a Theia workspace to open (default: an empty workspace folder)
+- `--runs`: Number of runs for the measurement (default: `10`)
+- `--debug`: Whether to log debug information to the console. Currently, this is only the standard error of the Electron app, which ordinarily is suppressed because the child process is detached
+
+_**Note**: When multiple runs are specified the script will calculate the mean and the standard deviation of all values, except for any runs that failed to capture a measurement due to an exception._
+
+It can happen that the Electron app does not start normally because the native browser modules are not properly built for the Electron target.
+The symptom for this is usually an error about a module not self-registering; when this condition is detected, the script stops rather than print out an inevitable series of failures to measure the performance.
+
+## Measure impact on startup performance of extensions
+
+To measure the startup performance impact that extensions have on the application, another script is available, which uses the measurements from the `browser-performance.js` or `electron-performance.js` script.
+The `extension-impact.js` script runs the measurement for a defined base application (`base-package.json` in this directory) and then measures the startup time when one of the defined extensions is added to the base application.
+The script will then print a table (in CSV format) to the console (and store it in a file) which contains the mean, standard deviation (Std Dev) and coefficient of variation (CV) for each extensions run.
+Additionally, each extensions entry will contain the difference to the base application time.
+
+Example Table:
+
+| Extension Name    | Mean (10 runs) (in s) | Std Dev (in s) | CV (%) | Delta (in s) |
+| ----------------- | --------------------- | -------------- | ------ | ------------ |
+| Base Theia        | 2.027                 | 0.084          | 4.144  | -            |
+| @theia/git:1.19.0 | 2.103                 | 0.041          | 1.950  | 0.076        |
+
+### Script usage
+
+The script can be executed by running `node extension-impact.js` in this directory.
+
+The following parameters are available:
+
+- `--app`: The example app in which to measure performance, either `browser` or `electron` (default: `browser`)
+- `--runs`: Specify the number of measurements for each extension (default: `10`)
+- `--base-time`: Provide an existing measurement (mean) for the base Theia application. If none is provided it will be measured.
+- `--extensions`: Provide a list of extensions (need to be locally installed) that shall be tested (default: all extensions in packages folder)
+
+    _**Note**: Each entry should:_
+
+  - _have the format {name}:{version}_
+  - _not contain whitespaces_
+  - _and be separated by whitespaces_
+
+    _For example: `--extensions @theia/git:1.19.0 @theia/keymaps:1.19.0`_
+
+- `--yarn`: Flag to trigger a full build at script startup (e.g. to build changes to extensions)
+- `--url`: Specify a URL that Theia should be launched with (can be used to specify the workspace to be opened). _Applies only to the `browser` app_ (default: `http://localhost:3000/#/<GIT_ROOT>/scripts/performance/workspace`)
+- `--workspace`: Specify a workspace on which to launch Theia. _Applies only to the `electron` app_ (default: `/<GIT_ROOT>/scripts/performance/workspace`)
+- `--file`: Relative path to the output file (default: `./script.csv`)
+
+_**Note**: If no extensions are provided all extensions from the `packages` folder will be measured._

scripts/performance/base-package.json

@@ -0,0 +1,37 @@
+{
+  "private": true,
+  "name": "@theia/example-{{app}}",
+  "version": "{{version}}",
+  "license": "EPL-2.0 OR GPL-2.0-with-classpath-exception",
+  "theia": {
+    "target": "{{app}}",
+    "frontend": {
+      "config": {
+        "applicationName": "Theia {{app}} Example",
+        "preferences": {
+          "files.enableTrash": false
+        }
+      }
+    },
+    "backend": {
+      "config": {
+        "resolveSystemPlugins": false
+      }
+    }
+  },
+  "dependencies": {
+    "@theia/core": "{{version}}",
+    "@theia/plugin-ext": "{{version}}"
+  },
+  "scripts": {
+    "clean": "theia clean",
+    "build": "npm run -s compile && npm run -s bundle",
+    "bundle": "theia build --mode development",
+    "compile": "tsc -b",
+    "rebuild": "theia rebuild:{{app}} --cacheRoot ../..",
+    "start": "THEIA_CONFIG_DIR=./theia-config-dir theia start --plugins=local-dir:../../noPlugins --log-level=fatal"
+  },
+  "devDependencies": {
+    "@theia/cli": "{{version}}"
+  }
+}

scripts/performance/browser-performance.js

@@ -0,0 +1,147 @@
+// *****************************************************************************
+// Copyright (C) 2021 STMicroelectronics and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+// @ts-check
+const puppeteer = require('puppeteer');
+const fsx = require('fs-extra');
+const { resolve } = require('path');
+const { delay, githubReporting, isLCP, lcp, measure } = require('./common-performance');
+
+const workspacePath = resolve('./workspace');
+const profilesPath = './profiles/';
+
+let name = 'Browser Frontend Startup';
+let url = 'http://localhost:3000/#' + workspacePath;
+let folder = 'browser';
+let headless = true;
+let runs = 10;
+
+(async () => {
+    let defaultUrl = true;
+    const yargs = require('yargs');
+    const args = yargs(process.argv.slice(2)).option('name', {
+        alias: 'n',
+        desc: 'A name for the test suite',
+        type: 'string',
+        default: name
+    }).option('folder', {
+        alias: 'f',
+        desc: 'Name of a folder within the "profiles" folder in which to collect trace logs',
+        type: 'string',
+        default: folder
+    }).option('runs', {
+        alias: 'r',
+        desc: 'The number of times to run the test',
+        type: 'number',
+        default: runs
+    }).option('url', {
+        alias: 'u',
+        desc: 'URL on which to open Theia in the browser (e.g., to specify a workspace)',
+        type: 'string',
+        default: url
+    }).option('headless', {
+        desc: 'Run in headless mode (do not open a browser)',
+        type: 'boolean',
+        default: headless
+    }).wrap(Math.min(120, yargs.terminalWidth())).argv;
+
+    if (args.name) {
+        name = args.name.toString();
+    }
+    if (args.url) {
+        url = args.url.toString();
+        defaultUrl = false;
+    }
+    if (args.folder) {
+        folder = args.folder.toString();
+    }
+    if (args.runs) {
+        runs = parseInt(args.runs.toString());
+    }
+    if (args.headless !== undefined && args.headless.toString() === 'false') {
+        headless = false;
+    }
+    if (process.env.GITHUB_ACTIONS) {
+        githubReporting.enabled = true;
+    }
+
+    // Verify that the application exists
+    const indexHTML = resolve(__dirname, '../../examples/browser/src-gen/frontend/index.html');
+    if (!fsx.existsSync(indexHTML)) {
+        console.error('Browser example app does not exist. Please build it before running this script.');
+        process.exit(1);
+    }
+
+    if (defaultUrl) { fsx.ensureDirSync(workspacePath); }
+    fsx.ensureDirSync(profilesPath);
+    const folderPath = profilesPath + folder;
+    fsx.ensureDirSync(folderPath);
+
+    const deployed = await waitForDeployed(url, 10, 500);
+    if (deployed == false) {
+        console.error('Could not connect to application.')
+    } else {
+        await measurePerformance(name, url, folderPath, headless, runs);
+    }
+})();
+
+async function measurePerformance(name, url, folder, headless, runs) {
+
+    /** @type import('./common-performance').TestFunction */
+    const testScenario = async (runNr) => {
+        const browser = await puppeteer.launch({ headless: headless });
+        const page = await browser.newPage();
+
+        const file = folder + '/' + runNr + '.json';
+        await page.tracing.start({ path: file, screenshots: true });
+        await page.goto(url);
+        // This selector is for the theia application, which is exposed when the loading indicator vanishes
+        await page.waitForSelector('.theia-ApplicationShell', { visible: true });
+        // Prevent tracing from stopping too soon and skipping a LCP candidate
+        await delay(1000);
+
+        await page.tracing.stop();
+
+        await browser.close();
+
+        return file;
+    };
+
+    measure(name, lcp, runs, testScenario, isStart, isLCP);
+}
+
+function isStart(x) {
+    return x.name === 'TracingStartedInBrowser';
+}
+
+async function waitForDeployed(url, maxTries, ms) {
+    let deployed = true;
+    const browser = await puppeteer.launch({ headless: true });
+    const page = await browser.newPage();
+    try {
+        await page.goto(url);
+    } catch (e) {
+        await delay(ms);
+        let newTries = maxTries - 1;
+        if (newTries > 0) {
+            deployed = await waitForDeployed(url, newTries, ms);
+        } else {
+            browser.close();
+            return false;
+        }
+    }
+    browser.close();
+    return deployed;
+}

scripts/performance/common-performance.js

@@ -0,0 +1,281 @@
+// *****************************************************************************
+// Copyright (C) 2021 STMicroelectronics and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+// @ts-check
+
+/**
+ * An event in the performance trace (from the Chrome performance API).
+ * @typedef TraceEvent
+ * @property {string} name the event name
+ * @property {number} ts the timestamp, in microseconds since some time after host system start
+ */
+
+/**
+ * A call-back that selects an event from the performance trace.
+ * 
+ * @callback EventPredicate
+ * @param {TraceEvent} event an event to test
+ * @returns {boolean} whether the predicate selects the `event`
+ */
+
+/**
+ * A call-back that runs the test scenario to be analyzed.
+ * 
+ * @async
+ * @callback TestFunction
+ * @param {number} runNr the current run index of the multiple runs being executed
+ * @returns {PromiseLike<string>} the path to the recorded performance profiling trace file
+ */
+
+const fs = require('fs');
+const { resolve } = require('path');
+
+const performanceTag = braceText('Performance');
+const lcp = 'Largest Contentful Paint (LCP)';
+
+/**
+ * A GitHub performance results record.
+ * 
+ * @typedef PerformanceResult
+ * @property {string} name The performance measurement name
+ * @property {string} unit The performance unit of measure
+ * @property {number} value The performance measurement
+ * @property {number} [range] The standard deviation (the GitHub action calls it a "range") of the measurement
+ */
+
+/**
+ * Configuration of reporting of performance test results in a GitHub build.
+ * 
+ * @property {boolean} enabled whether GitHub result reporting is enabled (`false` by default)
+ * @property {Array<PerformanceResult>} results the performance results, if reporting is enabled
+ */
+var githubReporting = { enabled: false, results: [] };
+
+/**
+ * Measure the performance of a `test` function implementing some `scenario` of interest.
+ * 
+ * @param {string} name the application name to measure
+ * @param {string} scenario a label for the scenario being measured
+ * @param {number} runs the number of times to run the `test` scenario
+ * @param {TestFunction} test a function that executes the `scenario` to be measured, returning the file 
+ *        that records the performance profile trace
+ * @param {EventPredicate} isStartEvent a predicate matching the trace event that marks the start of the measured scenario
+ * @param {EventPredicate} isEndEvent a predicate matching the trace event that marks the end of the measured scenario
+ */
+async function measure(name, scenario, runs, test, isStartEvent, isEndEvent) {
+    const durations = [];
+    for (let i = 0; i < runs; i++) {
+        const runNr = i + 1;
+
+        const file = await test(runNr);
+        let time;
+
+        try {
+            time = await analyzeTrace(file, isStartEvent, isEndEvent);
+
+            durations.push(time);
+            logDuration(name, runNr, scenario, time, runs > 1);
+        } catch (e) {
+            logException(name, runNr, scenario, e, runs > 1);
+        }
+    }
+
+    logSummary(name, scenario, durations);
+}
+
+
+/**
+ * Log a summary of the given measured `durations`.
+ * 
+ * @param {string} name the performance script name
+ * @param {string} scenario the scenario that was measured
+ * @param {number[]} durations the measurements captured for the `scenario`
+ */
+function logSummary(name, scenario, durations) {
+    if (durations.length > 1) {
+        const mean = calculateMean(durations);
+        const stdev = calculateStandardDeviation(mean, durations);
+        logDuration(name, 'MEAN', scenario, mean);
+        logDuration(name, 'STDEV', scenario, stdev);
+        if (githubReporting.enabled) {
+            githubResult({ name, unit: 'seconds', value: prec(mean), range: prec(stdev) });
+        }
+    } else if (githubReporting.enabled) {
+        // Only one duration
+        githubResult({ name, unit: 'seconds', value: prec(durations[0]) });
+    }
+}
+
+function prec(value, precision = 3) {
+    return Number.parseFloat(value.toPrecision(precision));
+}
+
+/**
+ * Report the performance result for GitHub to pick up.
+ * 
+ * @param {PerformanceResult} result the performance result to report
+ */
+function githubResult(result) {
+    const resultsFile = resolve('../..', 'performance-result.json');
+
+    // We append to any previous results that there may have been from another script
+    const previousResults = fs.existsSync(resultsFile) ? JSON.parse(fs.readFileSync(resultsFile, 'utf-8')) : [];
+    githubReporting.results.push(...previousResults);
+
+    githubReporting.results.push(result);
+    fs.writeFileSync(resultsFile, JSON.stringify(githubReporting.results, undefined, 2), 'utf-8');
+}
+
+/**
+ * Analyze a performance trace file.
+ * 
+ * @param {string} profilePath the profiling trace file path
+ * @param {EventPredicate} isStartEvent a predicate matching the trace event that marks the start of the measured scenario
+ * @param {EventPredicate} isEndEvent a predicate matching the trace event that marks the end of the measured scenario
+ */
+async function analyzeTrace(profilePath, isStartEvent, isEndEvent) {
+    let startEvent;
+    const tracing = JSON.parse(fs.readFileSync(profilePath, 'utf8'));
+    const endEvents = tracing.traceEvents.filter(e => {
+        if (startEvent === undefined && isStartEvent(e)) {
+            startEvent = e;
+            return false;
+        }
+        return isEndEvent(e);
+    });
+
+    if (startEvent !== undefined && endEvents.length > 0) {
+        return duration(endEvents[endEvents.length - 1], startEvent);
+    }
+
+    throw new Error('Could not analyze performance trace');
+}
+
+/**
+ * Query whether a trace `event` is a candidate for the Largest Contentful Paint.
+ * 
+ * @param {TraceEvent} event an event in the performance trace
+ * @returns whether the `event` is an LCP candidate
+ */
+function isLCP(event) {
+    return event.name === 'largestContentfulPaint::Candidate';
+}
+
+/**
+ * Compute the duration, in seconds, to an `event` from a start event.
+ * 
+ * @param {TraceEvent} event the duration end event
+ * @param {TraceEvent} startEvent the duration start event
+ * @returns the duration, in seconds
+ */
+function duration(event, startEvent) {
+    return (event.ts - startEvent.ts) / 1_000_000;
+}
+
+/**
+ * Log a `duration` measured for some scenario.
+ * 
+ * @param {string} name the performance script name
+ * @param {number|string} run the run index number, or some kind of aggregate like 'Total' or 'Avg'
+ * @param {string} metric the scenario that was measured
+ * @param {number} duration the duration, in seconds, of the measured scenario
+ * @param {boolean} [multipleRuns=true] whether the `run` logged is one of many being logged (default: `true`)
+ */
+function logDuration(name, run, metric, duration, multipleRuns = true) {
+    let runText = '';
+    if (multipleRuns) {
+        runText = braceText(run);
+    }
+    console.log(performanceTag + braceText(name) + runText + ' ' + metric + ': ' + duration.toFixed(3) + ' seconds');
+}
+
+/**
+ * Log an `exception` in measurement of some scenario.
+ * 
+ * @param {string} name the performance script name
+ * @param {number|string} run the run index number, or some kind of aggregate like 'Total' or 'Avg'
+ * @param {string} metric the scenario that was measured
+ * @param {Error} exception the duration, in seconds, of the measured scenario
+ * @param {boolean} [multipleRuns=true] whether the `run` logged is one of many being logged (default: `true`)
+ */
+function logException(name, run, metric, exception, multipleRuns = true) {
+    let runText = '';
+    if (multipleRuns) {
+        runText = braceText(run);
+    }
+    console.log(performanceTag + braceText(name) + runText + ' ' + metric + ' failed to obtain a measurement: ' + exception.message);
+    console.error(`Failed to obtain a measurement. The most likely cause is that the performance trace file was incomplete because the script did not wait long enough for "${metric}".`);
+    console.error(exception);
+}
+
+/**
+ * Compute the arithmetic mean of an `array` of numbers.
+ * 
+ * @param {number[]} array an array of numbers to average
+ * @returns the average of the `array`
+ */
+function calculateMean(array) {
+    let sum = 0;
+    array.forEach(x => {
+        sum += x;
+    });
+    return (sum / array.length);
+};
+
+/**
+ * Compute the standard deviation from the mean of an `array` of numbers.
+ * 
+ * @param {number[]} array an array of numbers
+ * @returns the standard deviation of the `array` from its mean
+ */
+function calculateStandardDeviation(mean, array) {
+    let sumOfDiffsSquared = 0;
+    array.forEach(time => {
+        sumOfDiffsSquared += Math.pow((time - mean), 2)
+    });
+    const variance = sumOfDiffsSquared / array.length;
+    return Math.sqrt(variance);
+}
+
+/**
+ * Surround a string of `text` in square braces.
+ * 
+ * @param {string|number} text a string of text or a number that can be rendered as text
+ * @returns the `text` in braces
+ */
+function braceText(text) {
+    return '[' + text + ']';
+}
+
+/**
+ * Obtain a promise that resolves after some delay.
+ * 
+ * @param {number} time a delay, in milliseconds
+ * @returns a promise that will resolve after the given number of milliseconds
+ */
+function delay(time) {
+    return new Promise(function (resolve) {
+        setTimeout(resolve, time)
+    });
+}
+
+module.exports = {
+    githubReporting,
+    measure, analyzeTrace,
+    calculateMean, calculateStandardDeviation,
+    duration, logDuration, logSummary,
+    braceText, delay,
+    lcp, isLCP
+};

scripts/performance/electron-performance.js

@@ -0,0 +1,197 @@
+// *****************************************************************************
+// Copyright (C) 2021 STMicroelectronics and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+// @ts-check
+const fsx = require('fs-extra');
+const { resolve } = require('path');
+const { spawn, ChildProcess } = require('child_process');
+const { delay, githubReporting, isLCP, lcp, measure } = require('./common-performance');
+const traceConfigTemplate = require('./electron-trace-config.json');
+const { exit } = require('process');
+
+const basePath = resolve(__dirname, '../..');
+const profilesPath = resolve(__dirname, './profiles/');
+const electronExample = resolve(basePath, 'examples/electron');
+const theia = resolve(electronExample, 'node_modules/.bin/theia');
+
+let name = 'Electron Frontend Startup';
+let folder = 'electron';
+let runs = 10;
+let workspace = resolve('./workspace');
+let debugging = false;
+
+(async () => {
+    let defaultWorkspace = true;
+
+    const yargs = require('yargs');
+    const args = yargs(process.argv.slice(2)).option('name', {
+        alias: 'n',
+        desc: 'A name for the test suite',
+        type: 'string',
+        default: name
+    }).option('folder', {
+        alias: 'f',
+        desc: 'Name of a folder within the "profiles" folder in which to collect trace logs',
+        type: 'string',
+        default: folder
+    }).option('runs', {
+        alias: 'r',
+        desc: 'The number of times to run the test',
+        type: 'number',
+        default: runs
+    }).option('workspace', {
+        alias: 'w',
+        desc: 'Path to a Theia workspace to open',
+        type: 'string',
+        default: workspace
+    }).option('debug', {
+        alias: 'X',
+        desc: 'Whether to log debug information',
+        boolean: true
+    }).wrap(Math.min(120, yargs.terminalWidth())).argv;
+
+    if (args.name) {
+        name = args.name.toString();
+    }
+    if (args.folder) {
+        folder = args.folder.toString();
+    }
+    if (args.workspace) {
+        workspace = args.workspace.toString();
+        if (resolve(workspace) !== workspace) {
+            console.log('Workspace path must be an absolute path:', workspace);
+            exit(1);
+        }
+        defaultWorkspace = false;
+    }
+    if (args.runs) {
+        runs = parseInt(args.runs.toString());
+    }
+    debugging = args.debug;
+    if (process.env.GITHUB_ACTIONS) {
+        githubReporting.enabled = true;
+    }
+
+    // Verify that the application exists
+    const indexHTML = resolve(electronExample, 'src-gen/frontend/index.html');
+    if (!fsx.existsSync(indexHTML)) {
+        console.error('Electron example app does not exist. Please build it before running this script.');
+        process.exit(1);
+    }
+
+    if (defaultWorkspace) {
+        // Ensure that it exists
+        fsx.ensureDirSync(workspace);
+    }
+
+    await measurePerformance();
+})();
+
+async function measurePerformance() {
+    fsx.emptyDirSync(resolve(profilesPath, folder));
+    const traceConfigPath = resolve(profilesPath, folder, 'trace-config.json');
+
+    /**
+     * Generate trace config from the template.
+     * @param {number} runNr
+     * @returns {string} the output trace file path
+     */
+    const traceConfigGenerator = (runNr) => {
+        const traceConfig = { ...traceConfigTemplate };
+        const traceFilePath = resolve(profilesPath, folder, `${runNr}.json`);
+        traceConfig.result_file = traceFilePath
+        fsx.writeFileSync(traceConfigPath, JSON.stringify(traceConfig, undefined, 2), 'utf-8');
+        return traceFilePath;
+    };
+
+    const exitHandler = (andExit = false) => {
+        return () => {
+            if (electron && !electron.killed) {
+                process.kill(-electron.pid, 'SIGINT');
+            }
+            if (andExit) {
+                process.exit();
+            }
+        }
+    };
+
+    // Be sure not to leave a detached Electron child process
+    process.on('exit', exitHandler());
+    process.on('SIGINT', exitHandler(true));
+    process.on('SIGTERM', exitHandler(true));
+
+    let electron;
+
+    /** @type import('./common-performance').TestFunction */
+    const testScenario = async (runNr) => {
+        const traceFile = traceConfigGenerator(runNr);
+        electron = await launchElectron(traceConfigPath);
+
+        electron.stderr.on('data', data => analyzeStderr(data.toString()));
+
+        // Wait long enough to be sure that tracing has finished. Kill the process group
+        // because the 'theia' child process was detached
+        await delay(traceConfigTemplate.startup_duration * 1_000 * 3 / 2)
+            .then(() => electron.exitCode !== null || process.kill(-electron.pid, 'SIGINT'));
+        electron = undefined;
+        return traceFile;
+    };
+
+    measure(name, lcp, runs, testScenario, hasNonzeroTimestamp, isLCP);
+}
+
+/**
+ * Launch the Electron app as a detached child process with tracing configured to start
+ * immediately upon launch. The child process is detached because otherwise the attempt
+ * to signal it to terminate when the test run is complete will not terminate the entire
+ * process tree but only the root `theia` process, leaving the electron app instance
+ * running until eventually this script itself exits.
+ * 
+ * @param {string} traceConfigPath the path to the tracing configuration file with which to initiate tracing
+ * @returns {Promise<ChildProcess>} the Electron child process, if successfully launched
+ */
+async function launchElectron(traceConfigPath) {
+    const args = ['start', workspace, '--plugins=local-dir:../../plugins', `--trace-config-file=${traceConfigPath}`];
+    if (process.platform === 'linux') {
+        args.push('--headless');
+    }
+    return spawn(theia, args, { cwd: electronExample, detached: true });
+}
+
+function hasNonzeroTimestamp(traceEvent) {
+    return traceEvent.hasOwnProperty('ts') // The traces don't have explicit nulls or undefineds
+        && traceEvent.ts > 0;
+}
+
+/**
+ * Analyze a `chunk` of text on the standard error stream of the child process.
+ * If running in debug mode, this will always at least print out the `chunk` to the console.
+ * In addition, the text is analyzed to look for known conditions that will invalidate the
+ * test procedure and cause the script to bail. These include:
+ * 
+ * - the native browser modules not being built correctly for Electron
+ * 
+ * @param {string} chunk a chunk of standard error text from the child process
+ */
+function analyzeStderr(chunk) {
+    if (debugging) {
+        console.error('>', chunk.trimEnd());
+    }
+
+    if (chunk.includes('Error: Module did not self-register')) {
+        console.error('Native browser modules are not built properly. Please rebuild the workspace and try again.');
+        exit(1);
+    }
+}

scripts/performance/electron-trace-config.json

@@ -0,0 +1,16 @@
+{
+  "startup_duration": 10,
+  "result_file": "{{placeholder}}.json",
+  "trace_config": {
+    "enable_argument_filter": false,
+    "enable_systrace": false,
+    "included_categories": [
+      "blink",
+      "loading",
+      "disabled-by-default-devtools.timeline"
+    ],
+    "excluded_categories": [
+      "*"
+    ]
+  }
+}

scripts/performance/extension-impact.js

@@ -0,0 +1,312 @@
+// *****************************************************************************
+// Copyright (C) 2021 STMicroelectronics and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+// @ts-check
+const { execSync, exec } = require('child_process');
+const { EOL } = require('os');
+const { copyFileSync, readdirSync, writeFileSync, appendFileSync, unlinkSync, readFileSync, rmdirSync } = require('fs');
+const { ensureFileSync } = require('fs-extra');
+const mkdirp = require('mkdirp');
+const path = require('path');
+const env = Object.assign({}, process.env);
+env.PATH = path.resolve("../../node_modules/.bin") + path.delimiter + env.PATH;
+let basePackage;
+const { exit } = require('process');
+let runs = 10;
+let baseTime;
+let extensions = [];
+let yarn = false;
+let url;
+let workspace;
+let file = path.resolve('./script.csv');
+let hostApp = 'browser';
+
+async function sigintHandler() {
+    process.exit();
+}
+
+async function exitHandler() {
+    cleanWorkspace();
+    printFile();
+}
+
+(async () => {
+    process.on('SIGINT', sigintHandler);
+    process.on('exit', exitHandler);
+
+    const yargs = require('yargs');
+    const args = yargs(process.argv.slice(2))
+        .option('base-time', {
+            alias: 'b',
+            desc: 'Pass an existing mean of the base application',
+            type: 'number'
+        })
+        .option('runs', {
+            alias: 'r',
+            desc: 'The number of runs to measure',
+            type: 'number',
+            default: 10
+        })
+        .option('extensions', {
+            alias: 'e',
+            desc: `An array of extensions to test (defaults to the extensions in the packages folder).
+                - Each entry must have this format: {name}:{version}
+                - Entries are separated by whitespaces
+                - Example: --extensions @theia/git:1.18.0 @theia/keymaps:1.18.0`,
+            type: 'array'
+        })
+        .option('yarn', {
+            alias: 'y',
+            desc: 'Build all typescript sources on script start',
+            type: 'boolean',
+            default: false
+        }).option('url', {
+            alias: 'u',
+            desc: 'Specify a custom URL at which to launch Theia in the browser (e.g. with a specific workspace)',
+            type: 'string'
+        }).option('workspace', {
+            alias: 'w',
+            desc: 'Specify an absolute path to a workspace on which to launch Theia in Electron',
+            type: 'string'
+        }).option('file', {
+            alias: 'f',
+            desc: 'Specify the relative path to a CSV file which stores the result',
+            type: 'string',
+            default: file
+        }).option('app', {
+            alias: 'a',
+            desc: 'Specify in which application to run the tests',
+            type: 'string',
+            choices: ['browser', 'electron'],
+            default: 'browser'
+        }).wrap(Math.min(120, yargs.terminalWidth())).argv;
+    if (args.baseTime) {
+        baseTime = parseFloat(args.baseTime.toString()).toFixed(3);
+    }
+    if (args.extensions) {
+        extensions = args.extensions;
+    }
+    if (args.runs) {
+        runs = parseInt(args.runs.toString());
+        if (runs < 2) {
+            console.error('--runs must be at least 2');
+            return;
+        }
+    }
+    if (args.yarn) {
+        yarn = true;
+    }
+    if (args.url) {
+        url = args.url;
+    }
+    if (args.workspace) {
+        workspace = args.workspace;
+    }
+    if (args.file) {
+        file = path.resolve(args.file);
+        if (!file.endsWith('.csv')) {
+            console.error('--file must end with .csv');
+            return;
+        }
+    }
+    if (args.app) {
+        hostApp = args.app;
+    }
+
+    preparePackageTemplate();
+    prepareWorkspace();
+    if (yarn) {
+        execSync('npm run build', { cwd: '../../', stdio: 'pipe' });
+    }
+    await extensionImpact(extensions);
+})();
+
+async function extensionImpact(extensions) {
+    logToFile(`Extension Name, Mean (${runs} runs) (in s), Std Dev (in s), CV (%), Delta (in s)`);
+    if (baseTime === undefined) {
+        await calculateExtension(undefined);
+    } else {
+        log(`Base Theia (provided), ${baseTime}, -, -, -`);
+    }
+
+    if (extensions.length < 1) {
+        extensions = await getExtensionsFromPackagesDir();
+    }
+
+    for (const e of extensions) {
+        await calculateExtension(e);
+    }
+}
+
+function preparePackageTemplate() {
+    const core = require('../../packages/core/package.json');
+    const version = core.version;
+    const content = readFileSync(path.resolve(__dirname, './base-package.json'), 'utf-8')
+        .replace(/\{\{app\}\}/g, hostApp)
+        .replace(/\{\{version\}\}/g, version);
+    basePackage = JSON.parse(content);
+    if (hostApp === 'electron') {
+        basePackage.dependencies['@theia/electron'] = version;
+    }
+    return basePackage;
+}
+
+function prepareWorkspace() {
+    copyFileSync(`../../examples/${hostApp}/package.json`, './backup-package.json');
+    mkdirp('../../noPlugins', function (err) {
+        if (err) {
+            console.error(err);
+        }
+    });
+    mkdirp('./theia-config-dir', function (err) {
+        if (err) {
+            console.error(err);
+        }
+    });
+    ensureFileSync(file);
+    writeFileSync(file, '');
+}
+
+function cleanWorkspace() {
+    copyFileSync('./backup-package.json', `../../examples/${hostApp}/package.json`);
+    unlinkSync('./backup-package.json');
+    rmdirSync('../../noPlugins');
+    rmdirSync('./theia-config-dir');
+}
+
+async function getExtensionsFromPackagesDir() {
+    const directories = readdirSync('../../packages', { withFileTypes: true })
+        .filter(dir => dir.isDirectory() && dir.name !== 'core')
+        .map(dir => dir.name);
+
+    return directories.map(directory => {
+        const name = `"${require(`../../packages/${directory}/package.json`).name}"`;
+        const version = `"${require(`../../packages/${directory}/package.json`).version}"`;
+        return name + ': ' + version;
+    });
+}
+
+async function calculateExtension(extensionQualifier) {
+    const basePackageCopy = { ...basePackage };
+    basePackageCopy.dependencies = { ...basePackageCopy.dependencies };
+    if (extensionQualifier !== undefined) {
+        const qualifier = extensionQualifier.replace(/"/g, '');
+        const name = qualifier.substring(0, qualifier.lastIndexOf(':'));
+        const version = qualifier.substring(qualifier.lastIndexOf(':') + 1);
+        basePackageCopy.dependencies[name] = version;
+    } else {
+        extensionQualifier = "Base Theia";
+    }
+    logToConsole(`Building the ${hostApp} example with ${extensionQualifier}.`);
+    writeFileSync(`../../examples/${hostApp}/package.json`, JSON.stringify(basePackageCopy, null, 2));
+    try {
+        execSync(`npm run build:${hostApp}`, { cwd: '../../', stdio: 'pipe' });
+
+        // Rebuild native modules if necessary
+        execSync(`npm run rebuild:${hostApp}`, { cwd: '../../', stdio: 'pipe' });
+    } catch (error) {
+        log(`${extensionQualifier}, Error while building the package.json, -, -, -`);
+        return;
+    }
+
+    logToConsole(`Measuring the startup time with ${extensionQualifier} ${runs} times. This may take a while.`);
+    const appCommand = (app) => {
+        let command;
+        let cwd;
+        switch (app) {
+            case 'browser':
+                command = `concurrently --success first -k -r "cd scripts/performance && node browser-performance.js --name Browser --folder browser --runs ${runs}${url ? ' --url ' + url : ''}" `
+                    + `"npm run start:browser | grep -v '.*'"`
+                cwd = path.resolve(__dirname, '../../');
+                break;
+            case 'electron':
+                command = `node electron-performance.js  --name Electron --folder electron --runs ${runs}${workspace ? ' --workspace "' + workspace + '"' : ''}`
+                cwd = __dirname;
+                break;
+            default:
+                console.log('Unknown host app:', hostApp);
+                exit(1);
+                break; // Unreachable
+        }
+        return [command, cwd];
+    };
+    const [command, cwd] = appCommand(hostApp);
+    const output = await execCommand(command, { env: env, cwd: cwd, shell: true });
+
+    const mean = parseFloat(getMeasurement(output, '[MEAN] Largest Contentful Paint (LCP):'));
+    const stdev = parseFloat(getMeasurement(output, '[STDEV] Largest Contentful Paint (LCP):'));
+
+    if (isNaN(mean) || isNaN(stdev)) {
+        log(`${extensionQualifier}, Error while measuring with this extension, -, -, -`);
+    } else {
+        const cv = ((stdev / mean) * 100).toFixed(3);
+        let diff;
+        if (baseTime === undefined) {
+            diff = '-';
+            baseTime = mean;
+        } else {
+            diff = (mean - baseTime).toFixed(3);
+        }
+        log(`${extensionQualifier}, ${mean.toFixed(3)}, ${stdev.toFixed(3)}, ${cv}, ${diff}`);
+    }
+}
+
+async function execCommand(command, args) {
+    return new Promise((resolve) => {
+        let output = '';
+        const childProcess = exec(command, args);
+        childProcess.stdout.on('data', function (out) {
+            output += out.toString();
+            console.log(out.toString().trim());
+        });
+
+        childProcess.stderr.on('data', function (error) {
+            console.log(error.toString());
+        });
+
+        childProcess.on('close', function () {
+            resolve(output);
+        });
+
+        childProcess.on('exit', function () {
+            resolve(output);
+        })
+    });
+}
+
+function getMeasurement(output, identifier) {
+    const firstIndex = output.lastIndexOf(identifier) + identifier.length + 1;
+    const lastIndex = output.indexOf("seconds", firstIndex) - 1;
+    return output.toString().substring(firstIndex, lastIndex);
+}
+
+function printFile() {
+    console.log();
+    const content = readFileSync(file).toString();
+    console.log(content);
+}
+
+function log(text) {
+    logToConsole(text);
+    logToFile(text);
+}
+
+function logToConsole(text) {
+    console.log(text);
+}
+
+function logToFile(text) {
+    appendFileSync(file, text + EOL);
+}