deploy: current vibn theia state

Made-with: Cursor

dev-packages/cli/src/check-dependencies.ts

@@ -0,0 +1,328 @@
+// *****************************************************************************
+// Copyright (C) 2022 STMicroelectronics and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { glob } from 'glob';
+import { create as logUpdater } from 'log-update';
+import * as chalk from 'chalk';
+
+const NODE_MODULES = 'node_modules';
+const PACKAGE_JSON = 'package.json';
+
+const logUpdate = logUpdater(process.stdout);
+
+interface CheckDependenciesOptions {
+    workspaces: string[] | undefined,
+    include: string[],
+    exclude: string[],
+    skipHoisted: boolean,
+    skipUniqueness: boolean,
+    skipSingleTheiaVersion: boolean,
+    onlyTheiaExtensions: boolean,
+    suppress: boolean
+}
+
+/** NPM package */
+interface Package {
+    /** Name of the package, e.g. `@theia/core`. */
+    name: string,
+    /** Actual resolved version of the package, e.g. `1.27.0`. */
+    version: string,
+    /** Path of the package relative to the workspace, e.g. `node_modules/@theia/core`. */
+    path: string,
+    /** Whether the package is hoisted or not, i.e., whether it is contained in the root `node_modules`. */
+    hoisted: boolean,
+    /** Workspace location in which the package was found. */
+    dependent: string | undefined,
+    /** Whether the package is a Theia extension or not */
+    isTheiaExtension?: boolean,
+}
+
+/** Issue found with a specific package. */
+interface DependencyIssue {
+    /** Type of the issue. */
+    issueType: 'not-hoisted' | 'multiple-versions' | 'theia-version-mix',
+    /** Package with issue. */
+    package: Package,
+    /** Packages related to this issue. */
+    relatedPackages: Package[],
+    /** Severity */
+    severity: 'warning' | 'error'
+}
+
+export default function checkDependencies(options: CheckDependenciesOptions): void {
+    const workspaces = deriveWorkspaces(options);
+    logUpdate(`✅ Found ${workspaces.length} workspaces`);
+
+    console.log('🔍 Collecting dependencies...');
+    const dependencies = findAllDependencies(workspaces, options);
+    logUpdate(`✅ Found ${dependencies.length} dependencies`);
+
+    console.log('🔍 Analyzing dependencies...');
+    const issues = analyzeDependencies(dependencies, options);
+    if (issues.length <= 0) {
+        logUpdate('✅ No issues were found');
+        process.exit(0);
+    }
+
+    logUpdate('🟠 Found ' + issues.length + ' issues');
+    printIssues(issues);
+    printHints(issues);
+    process.exit(options.suppress ? 0 : 1);
+}
+
+function deriveWorkspaces(options: CheckDependenciesOptions): string[] {
+    const wsGlobs = options.workspaces ?? readWorkspaceGlobsFromPackageJson();
+    const workspaces: string[] = [];
+    for (const wsGlob of wsGlobs) {
+        workspaces.push(...glob.sync(wsGlob + '/'));
+    }
+    return workspaces;
+}
+
+function readWorkspaceGlobsFromPackageJson(): string[] {
+    const rootPackageJson = path.join(process.cwd(), PACKAGE_JSON);
+    if (!fs.existsSync(rootPackageJson)) {
+        console.error('Directory does not contain a package.json with defined workspaces');
+        console.info('Run in the root of a Theia project or specify them via --workspaces');
+        process.exit(1);
+    }
+    return require(rootPackageJson).workspaces ?? [];
+}
+
+function findAllDependencies(workspaces: string[], options: CheckDependenciesOptions): Package[] {
+    const dependencies: Package[] = [];
+    dependencies.push(...findDependencies('.', options));
+    for (const workspace of workspaces) {
+        dependencies.push(...findDependencies(workspace, options));
+    }
+    return dependencies;
+}
+
+function findDependencies(workspace: string, options: CheckDependenciesOptions): Package[] {
+    const dependent = getPackageName(path.join(process.cwd(), workspace, PACKAGE_JSON));
+    const nodeModulesDir = path.join(workspace, NODE_MODULES);
+    const matchingPackageJsons: Package[] = [];
+    options.include.forEach(include =>
+        glob.sync(`${include}/${PACKAGE_JSON}`, {
+            cwd: nodeModulesDir,
+            ignore: [
+                `**/${NODE_MODULES}/**`, // node_modules folders within dependencies
+                `[^@]*/*/**/${PACKAGE_JSON}`, // package.json that isn't at the package root (and not in an @org)
+                `@*/*/*/**/${PACKAGE_JSON}`, // package.json that isn't at the package root (and in an @org)
+                ...options.exclude] // user-specified exclude patterns
+        }).forEach(packageJsonPath => {
+            const dependency = toDependency(packageJsonPath, nodeModulesDir, dependent);
+            if (!options.onlyTheiaExtensions || dependency.isTheiaExtension) {
+                matchingPackageJsons.push(dependency);
+            }
+            const childNodeModules: string = path.join(nodeModulesDir, packageJsonPath, '..');
+            matchingPackageJsons.push(...findDependencies(childNodeModules, options));
+        })
+    );
+    return matchingPackageJsons;
+}
+
+function toDependency(packageJsonPath: string, nodeModulesDir: string, dependent?: string): Package {
+    const fullPackageJsonPath = path.join(process.cwd(), nodeModulesDir, packageJsonPath);
+    const name = getPackageName(fullPackageJsonPath);
+    const version = getPackageVersion(fullPackageJsonPath);
+    return {
+        name: name ?? packageJsonPath.replace('/' + PACKAGE_JSON, ''),
+        version: version ?? 'unknown',
+        path: path.relative(process.cwd(), fullPackageJsonPath),
+        hoisted: nodeModulesDir === NODE_MODULES,
+        dependent: dependent,
+        isTheiaExtension: isTheiaExtension(fullPackageJsonPath)
+    };
+}
+
+function getPackageVersion(fullPackageJsonPath: string): string | undefined {
+    try {
+        return require(fullPackageJsonPath).version;
+    } catch (error) {
+        return undefined;
+    }
+}
+
+function getPackageName(fullPackageJsonPath: string): string | undefined {
+    try {
+        return require(fullPackageJsonPath).name;
+    } catch (error) {
+        return undefined;
+    }
+}
+
+function isTheiaExtension(fullPackageJsonPath: string): boolean {
+    try {
+        const theiaExtension = require(fullPackageJsonPath).theiaExtensions;
+        return theiaExtension ? true : false;
+    } catch (error) {
+        return false;
+    }
+}
+
+function analyzeDependencies(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] {
+    const issues: DependencyIssue[] = [];
+    if (!options.skipHoisted) {
+        issues.push(...findNotHoistedDependencies(packages, options));
+    }
+    if (!options.skipUniqueness) {
+        issues.push(...findDuplicateDependencies(packages, options));
+    }
+    if (!options.skipSingleTheiaVersion) {
+        issues.push(...findTheiaVersionMix(packages, options));
+    }
+    return issues;
+}
+
+function findNotHoistedDependencies(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] {
+    const issues: DependencyIssue[] = [];
+    const nonHoistedPackages = packages.filter(p => p.hoisted === false);
+    for (const nonHoistedPackage of nonHoistedPackages) {
+        issues.push(createNonHoistedPackageIssue(nonHoistedPackage, options));
+    }
+    return issues;
+}
+
+function createNonHoistedPackageIssue(nonHoistedPackage: Package, options: CheckDependenciesOptions): DependencyIssue {
+    return {
+        issueType: 'not-hoisted',
+        package: nonHoistedPackage,
+        relatedPackages: [getHoistedPackageByName(nonHoistedPackage.name)],
+        severity: options.suppress ? 'warning' : 'error'
+    };
+}
+
+function getHoistedPackageByName(name: string): Package {
+    return toDependency(path.join(name, PACKAGE_JSON), NODE_MODULES);
+}
+
+function findDuplicateDependencies(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] {
+    const duplicates: string[] = [];
+    const packagesGroupedByName = new Map<string, Package[]>();
+    for (const currentPackage of packages) {
+        const name = currentPackage.name;
+        if (!packagesGroupedByName.has(name)) {
+            packagesGroupedByName.set(name, []);
+        }
+        const currentPackages = packagesGroupedByName.get(name)!;
+        currentPackages.push(currentPackage);
+        if (currentPackages.length > 1 && duplicates.indexOf(name) === -1) {
+            duplicates.push(name);
+        }
+    }
+
+    duplicates.sort();
+    const issues: DependencyIssue[] = [];
+    for (const duplicate of duplicates) {
+        const duplicatePackages = packagesGroupedByName.get(duplicate);
+        if (duplicatePackages && duplicatePackages.length > 0) {
+            issues.push({
+                issueType: 'multiple-versions',
+                package: duplicatePackages.pop()!,
+                relatedPackages: duplicatePackages,
+                severity: options.suppress ? 'warning' : 'error'
+            });
+        }
+    }
+
+    return issues;
+}
+
+function findTheiaVersionMix(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] {
+    // @theia/monaco-editor-core is following the versions of Monaco so it can't be part of this check
+    const theiaPackages = packages.filter(p => p.name.startsWith('@theia/') && !p.name.startsWith('@theia/monaco-editor-core'));
+    let theiaVersion = undefined;
+    let referenceTheiaPackage = undefined;
+    const packagesWithOtherVersion: Package[] = [];
+    for (const theiaPackage of theiaPackages) {
+        if (!theiaVersion && theiaPackage.version) {
+            theiaVersion = theiaPackage.version;
+            referenceTheiaPackage = theiaPackage;
+        } else if (theiaVersion !== theiaPackage.version) {
+            packagesWithOtherVersion.push(theiaPackage);
+        }
+    }
+
+    if (referenceTheiaPackage && packagesWithOtherVersion.length > 0) {
+        return [{
+            issueType: 'theia-version-mix',
+            package: referenceTheiaPackage,
+            relatedPackages: packagesWithOtherVersion,
+            severity: 'error'
+        }];
+    }
+    return [];
+}
+
+function printIssues(issues: DependencyIssue[]): void {
+    console.log();
+    const indent = issues.length.toString().length;
+    issues.forEach((issue, index) => {
+        printIssue(issue, index + 1, indent);
+    });
+}
+
+function printIssue(issue: DependencyIssue, issueNumber: number, indent: number): void {
+    const remainingIndent = indent - issueNumber.toString().length;
+    const indentString = ' '.repeat(remainingIndent + 1);
+    console.log(issueTitle(issue, issueNumber, indentString));
+    console.log(issueDetails(issue, '   ' + ' '.repeat(indent)));
+    console.log();
+}
+
+function issueTitle(issue: DependencyIssue, issueNumber: number, indent: string): string {
+    const dependent = issue.package.dependent ? ` in ${chalk.blueBright(issue.package.dependent ?? 'unknown')}` : '';
+    return chalk.bgWhiteBright.bold.black(`#${issueNumber}${indent}`) + ' ' + chalk.cyanBright(issue.package.name)
+        + dependent + chalk.dim(` [${issue.issueType}]`);
+}
+
+function issueDetails(issue: DependencyIssue, indent: string): string {
+    return indent + severity(issue) + ' ' + issueMessage(issue) + '\n'
+        + indent + versionLine(issue.package) + '\n'
+        + issue.relatedPackages.map(p => indent + versionLine(p)).join('\n');
+}
+
+function issueMessage(issue: DependencyIssue): string {
+    if (issue.issueType === 'multiple-versions') {
+        return `Multiple versions of dependency ${chalk.bold(issue.package.name)} found.`;
+    } else if (issue.issueType === 'theia-version-mix') {
+        return `Mix of ${chalk.bold('@theia/*')} versions found.`;
+    } else {
+        return `Dependency ${chalk.bold(issue.package.name)} is not hoisted.`;
+    }
+}
+
+function severity(issue: DependencyIssue): string {
+    return issue.severity === 'error' ? chalk.red('error') : chalk.yellow('warning');
+}
+
+function versionLine(pckg: Package): string {
+    return chalk.bold(pckg.version) + ' in ' + pckg.path;
+}
+
+function printHints(issues: DependencyIssue[]): void {
+    console.log();
+    if (issues.find(i => i.issueType === 'theia-version-mix')) {
+        console.log('⛔ A mix of Theia versions is very likely leading to a broken application.');
+    }
+    console.log(`ℹ️  Use ${chalk.bold('npm ls <package-name>')} to find out why those multiple versions of a package are pulled.`);
+    console.log('ℹ️  Try to resolve those issues by finding package versions along the dependency chain that depend on compatible versions.');
+    console.log(`ℹ️  Use ${chalk.bold('overrides')} in your root package.json to force specific versions as a last resort.`);
+    console.log();
+}

dev-packages/cli/src/download-plugins.ts

@@ -0,0 +1,421 @@
+// *****************************************************************************
+// Copyright (C) 2020 Ericsson and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { OVSXApiFilterImpl, OVSXClient, VSXTargetPlatform } from '@theia/ovsx-client';
+import * as chalk from 'chalk';
+import * as decompress from 'decompress';
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import * as temp from 'temp';
+import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api';
+import { RequestContext, RequestService } from '@theia/request';
+import { RateLimiter } from 'limiter';
+import escapeStringRegexp = require('escape-string-regexp');
+
+temp.track();
+
+/**
+ * Available options when downloading.
+ */
+export interface DownloadPluginsOptions {
+    /**
+     * Determines if a plugin should be unpacked.
+     * Defaults to `false`.
+     */
+    packed?: boolean;
+
+    /**
+     * Determines if failures while downloading plugins should be ignored.
+     * Defaults to `false`.
+     */
+    ignoreErrors?: boolean;
+
+    /**
+     * The supported vscode API version.
+     * Used to determine extension compatibility.
+     */
+    apiVersion?: string;
+
+    /**
+     * Fetch plugins in parallel
+     */
+    parallel?: boolean;
+}
+
+interface PluginDownload {
+    id: string,
+    downloadUrl: string,
+    version?: string | undefined
+}
+
+export default async function downloadPlugins(
+    ovsxClient: OVSXClient,
+    rateLimiter: RateLimiter,
+    requestService: RequestService,
+    options: DownloadPluginsOptions = {}
+): Promise<void> {
+    const {
+        packed = false,
+        ignoreErrors = false,
+        apiVersion = DEFAULT_SUPPORTED_API_VERSION,
+        parallel = true
+    } = options;
+
+    const apiFilter = new OVSXApiFilterImpl(ovsxClient, apiVersion);
+
+    // Collect the list of failures to be appended at the end of the script.
+    const failures: string[] = [];
+
+    // Resolve the `package.json` at the current working directory.
+    const pck = JSON.parse(await fs.readFile(path.resolve('package.json'), 'utf8'));
+
+    // Resolve the directory for which to download the plugins.
+    const pluginsDir = pck.theiaPluginsDir || 'plugins';
+
+    // Excluded extension ids.
+    const excludedIds = new Set<string>(pck.theiaPluginsExcludeIds || []);
+
+    const parallelOrSequence = async (tasks: (() => unknown)[]) => {
+        if (parallel) {
+            await Promise.all(tasks.map(task => task()));
+        } else {
+            for (const task of tasks) {
+                await task();
+            }
+        }
+    };
+
+    // Downloader wrapper
+    const downloadPlugin = async (plugin: PluginDownload): Promise<void> => {
+        await downloadPluginAsync(requestService, rateLimiter, failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, excludedIds, plugin.version);
+    };
+
+    const downloader = async (plugins: PluginDownload[]) => {
+        await parallelOrSequence(plugins.map(plugin => () => downloadPlugin(plugin)));
+    };
+
+    await fs.mkdir(pluginsDir, { recursive: true });
+
+    if (!pck.theiaPlugins) {
+        console.log(chalk.red('error: missing mandatory \'theiaPlugins\' property.'));
+        return;
+    }
+    try {
+        console.warn('--- downloading plugins ---');
+        // Download the raw plugins defined by the `theiaPlugins` property.
+        // This will include both "normal" plugins as well as "extension packs".
+        const pluginsToDownload = Object.entries(pck.theiaPlugins)
+            .filter((entry: [string, unknown]): entry is [string, string] => typeof entry[1] === 'string')
+            .map(([id, url]) => ({ id, downloadUrl: resolveDownloadUrlPlaceholders(url) }));
+        await downloader(pluginsToDownload);
+
+        const handleDependencyList = async (dependencies: (string | string[])[]) => {
+            // De-duplicate extension ids to only download each once:
+            const ids = new Set<string>(dependencies.flat());
+            await parallelOrSequence(Array.from(ids, id => async () => {
+                try {
+                    await rateLimiter.removeTokens(1);
+                    const extension = await apiFilter.findLatestCompatibleExtension({
+                        extensionId: id,
+                        includeAllVersions: true,
+                        targetPlatform
+                    });
+                    const version = extension?.version;
+                    const downloadUrl = extension?.files.download;
+                    if (downloadUrl) {
+                        await rateLimiter.removeTokens(1);
+                        await downloadPlugin({ id, downloadUrl, version });
+                    } else {
+                        failures.push(`No download url for extension pack ${id} (${version})`);
+                    }
+                } catch (err) {
+                    console.error(err);
+                    failures.push(err.message);
+                }
+            }));
+        };
+
+        console.warn('--- collecting extension-packs ---');
+        const extensionPacks = await collectExtensionPacks(pluginsDir, excludedIds);
+        if (extensionPacks.size > 0) {
+            console.warn(`--- resolving ${extensionPacks.size} extension-packs ---`);
+            await handleDependencyList(Array.from(extensionPacks.values()));
+        }
+
+        console.warn('--- collecting extension dependencies ---');
+        const pluginDependencies = await collectPluginDependencies(pluginsDir, excludedIds);
+        if (pluginDependencies.length > 0) {
+            console.warn(`--- resolving ${pluginDependencies.length} extension dependencies ---`);
+            await handleDependencyList(pluginDependencies);
+        }
+
+    } finally {
+        temp.cleanupSync();
+    }
+    for (const failure of failures) {
+        console.error(failure);
+    }
+    if (!ignoreErrors && failures.length > 0) {
+        throw new Error('Errors downloading some plugins. To make these errors non fatal, re-run with --ignore-errors');
+    }
+}
+
+const targetPlatform = `${process.platform}-${process.arch}` as VSXTargetPlatform;
+
+const placeholders: Record<string, string> = {
+    targetPlatform
+};
+function resolveDownloadUrlPlaceholders(url: string): string {
+    for (const [name, value] of Object.entries(placeholders)) {
+        url = url.replace(new RegExp(escapeStringRegexp(`\${${name}}`), 'g'), value);
+    }
+    return url;
+}
+
+/**
+ * Downloads a plugin, will make multiple attempts before actually failing.
+ * @param requestService
+ * @param failures reference to an array storing all failures.
+ * @param plugin plugin short name.
+ * @param pluginUrl url to download the plugin at.
+ * @param target where to download the plugin in.
+ * @param packed whether to decompress or not.
+ */
+async function downloadPluginAsync(
+    requestService: RequestService,
+    rateLimiter: RateLimiter,
+    failures: string[],
+    plugin: string,
+    pluginUrl: string,
+    pluginsDir: string,
+    packed: boolean,
+    excludedIds: Set<string>,
+    version?: string
+): Promise<void> {
+    if (!plugin) {
+        return;
+    }
+    let fileExt: string;
+    if (pluginUrl.endsWith('tar.gz')) {
+        fileExt = '.tar.gz';
+    } else if (pluginUrl.endsWith('vsix')) {
+        fileExt = '.vsix';
+    } else if (pluginUrl.endsWith('theia')) {
+        fileExt = '.theia'; // theia plugins.
+    } else {
+        failures.push(chalk.red(`error: '${plugin}' has an unsupported file type: '${pluginUrl}'`));
+        return;
+    }
+    const targetPath = path.resolve(pluginsDir, `${plugin}${packed === true ? fileExt : ''}`);
+
+    // Skip plugins which have previously been downloaded.
+    if (await isDownloaded(targetPath)) {
+        console.warn('- ' + plugin + ': already downloaded - skipping');
+        return;
+    }
+
+    const maxAttempts = 5;
+    const retryDelay = 2000;
+
+    let attempts: number;
+    let lastError: Error | undefined;
+    let response: RequestContext | undefined;
+
+    for (attempts = 0; attempts < maxAttempts; attempts++) {
+        if (attempts > 0) {
+            await new Promise(resolve => setTimeout(resolve, retryDelay));
+        }
+        lastError = undefined;
+        try {
+            await rateLimiter.removeTokens(1);
+            response = await requestService.request({
+                url: pluginUrl
+            });
+        } catch (error) {
+            lastError = error;
+            continue;
+        }
+        const status = response.res.statusCode;
+        const retry = status && (status === 429 || status === 439 || status >= 500);
+        if (!retry) {
+            break;
+        }
+    }
+    if (lastError) {
+        failures.push(chalk.red(`x ${plugin}: failed to download, last error:\n ${lastError}`));
+        return;
+    }
+    if (typeof response === 'undefined') {
+        failures.push(chalk.red(`x ${plugin}: failed to download (unknown reason)`));
+        return;
+    }
+    if (response.res.statusCode !== 200) {
+        failures.push(chalk.red(`x ${plugin}: failed to download with: ${response.res.statusCode}`));
+        return;
+    }
+
+    if ((fileExt === '.vsix' || fileExt === '.theia')) {
+        if (packed) {
+            // Download .vsix without decompressing.
+            await fs.writeFile(targetPath, response.buffer);
+        } else {
+            await decompressVsix(targetPath, response.buffer);
+        }
+        console.warn(chalk.green(`+ ${plugin}${version ? `@${version}` : ''}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`));
+    } else if (fileExt === '.tar.gz') {
+        // Write the downloaded tar.gz to a temporary file for decompression.
+        const tempFile = temp.path('theia-plugin-download');
+        await fs.writeFile(tempFile, response.buffer);
+        // Decompress to inspect archive contents and determine handling strategy.
+        const files = await decompress(tempFile);
+        // Check if the archive is a bundle containing only .vsix files.
+        const allVsix = files.length > 0 && files.every(file => file.path.endsWith('.vsix'));
+
+        if (allVsix) {
+            // Handle pure vsix bundle: process each vsix individually.
+            for (const file of files) {
+                const vsixName = path.basename(file.path);
+                const pluginId = vsixName.replace(/\.vsix$/, '');
+                if (excludedIds.has(pluginId)) {
+                    console.log(chalk.yellow(`'${pluginId}' referred to by '${plugin}' (tar.gz) is excluded because of 'theiaPluginsExcludeIds'`));
+                    continue;
+                }
+                const vsixTargetPath = packed
+                    ? path.join(pluginsDir, vsixName)
+                    : path.join(pluginsDir, pluginId);
+                if (await isDownloaded(vsixTargetPath)) {
+                    console.warn('- ' + pluginId + ': already downloaded - skipping');
+                    continue;
+                }
+                if (packed) {
+                    // Download .vsix without decompressing.
+                    await fs.writeFile(vsixTargetPath, file.data);
+                } else {
+                    await decompressVsix(vsixTargetPath, file.data);
+                }
+
+                console.warn(chalk.green(`+ ${pluginId}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`));
+            }
+        } else {
+            // Handle regular tar.gz: decompress directly to target directory.
+            await fs.mkdir(targetPath, { recursive: true });
+            await decompress(tempFile, targetPath);
+            console.warn(chalk.green(`+ ${plugin}${version ? `@${version}` : ''}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`));
+        }
+        await fs.unlink(tempFile);
+    }
+}
+
+/**
+ * Decompresses a VSIX plugin archive to a target directory.
+ *
+ * Creates the target directory if it doesn't exist, writes the buffer content
+ * to a temporary file, and then extracts the archive contents to the target path.
+ *
+ * @param targetPath the directory path where the VSIX contents will be extracted.
+ * @param buffer the VSIX file content as a binary buffer or string.
+ */
+async function decompressVsix(targetPath: string, buffer: Uint8Array | string): Promise<void> {
+    await fs.mkdir(targetPath, { recursive: true });
+    const tempFile = temp.path('theia-plugin-download');
+    await fs.writeFile(tempFile, buffer);
+    await decompress(tempFile, targetPath);
+    await fs.unlink(tempFile);
+}
+
+/**
+ * Determine if the resource for the given path is already downloaded.
+ * @param filePath the resource path.
+ *
+ * @returns `true` if the resource is already downloaded, else `false`.
+ */
+async function isDownloaded(filePath: string): Promise<boolean> {
+    return fs.stat(filePath).then(() => true, () => false);
+}
+
+/**
+ * Walk the plugin directory and collect available extension paths.
+ * @param pluginDir the plugin directory.
+ * @returns the list of all available extension paths.
+ */
+async function collectPackageJsonPaths(pluginDir: string): Promise<string[]> {
+    const packageJsonPathList: string[] = [];
+    const files = await fs.readdir(pluginDir);
+    // Recursively fetch the list of extension `package.json` files.
+    for (const file of files) {
+        const filePath = path.join(pluginDir, file);
+        if ((await fs.stat(filePath)).isDirectory()) {
+            packageJsonPathList.push(...await collectPackageJsonPaths(filePath));
+        } else if (path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules')) {
+            packageJsonPathList.push(filePath);
+        }
+    }
+    return packageJsonPathList;
+}
+
+/**
+ * Get the mapping of extension-pack paths and their included plugin ids.
+ * - If an extension-pack references an explicitly excluded `id` the `id` will be omitted.
+ * @param pluginDir the plugin directory.
+ * @param excludedIds the list of plugin ids to exclude.
+ * @returns the mapping of extension-pack paths and their included plugin ids.
+ */
+async function collectExtensionPacks(pluginDir: string, excludedIds: Set<string>): Promise<Map<string, string[]>> {
+    const extensionPackPaths = new Map<string, string[]>();
+    const packageJsonPaths = await collectPackageJsonPaths(pluginDir);
+    await Promise.all(packageJsonPaths.map(async packageJsonPath => {
+        const json = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+        const extensionPack: unknown = json.extensionPack;
+        if (Array.isArray(extensionPack)) {
+            extensionPackPaths.set(packageJsonPath, extensionPack.filter(id => {
+                if (excludedIds.has(id)) {
+                    console.log(chalk.yellow(`'${id}' referred to by '${json.name}' (ext pack) is excluded because of 'theiaPluginsExcludeIds'`));
+                    return false; // remove
+                }
+                return true; // keep
+            }));
+        }
+    }));
+    return extensionPackPaths;
+}
+
+/**
+ * Get the mapping of  paths and their included plugin ids.
+ * - If an extension-pack references an explicitly excluded `id` the `id` will be omitted.
+ * @param pluginDir the plugin directory.
+ * @param excludedIds the list of plugin ids to exclude.
+ * @returns the mapping of extension-pack paths and their included plugin ids.
+ */
+async function collectPluginDependencies(pluginDir: string, excludedIds: Set<string>): Promise<string[]> {
+    const dependencyIds: string[] = [];
+    const packageJsonPaths = await collectPackageJsonPaths(pluginDir);
+    await Promise.all(packageJsonPaths.map(async packageJsonPath => {
+        const json = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+        const extensionDependencies: unknown = json.extensionDependencies;
+        if (Array.isArray(extensionDependencies)) {
+            for (const dependency of extensionDependencies) {
+                if (excludedIds.has(dependency)) {
+                    console.log(chalk.yellow(`'${dependency}' referred to by '${json.name}' is excluded because of 'theiaPluginsExcludeIds'`));
+                } else {
+                    dependencyIds.push(dependency);
+                }
+            }
+        }
+    }));
+    return dependencyIds;
+}

dev-packages/cli/src/run-test.ts

@@ -0,0 +1,88 @@
+// *****************************************************************************
+// Copyright (C) 2020 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import * as net from 'net';
+import * as puppeteer from 'puppeteer-core';
+import newTestPage, { TestFileOptions } from './test-page';
+
+export interface TestOptions {
+    start: () => Promise<net.AddressInfo>
+    launch?: puppeteer.PuppeteerLaunchOptions
+    files?: Partial<TestFileOptions>
+    coverage?: boolean
+}
+
+export default async function runTest(options: TestOptions): Promise<void> {
+    const { start, launch } = options;
+    const exit = !(launch && launch.devtools);
+
+    const testPage = await newTestPage({
+        files: options.files,
+        matchAppUrl: () => true, // all urls are application urls
+        newPage: async () => {
+            const browser = await puppeteer.launch(launch);
+            // re-use empty tab
+            const [tab] = await browser.pages();
+            return tab;
+        },
+        onWillRun: async () => {
+            const promises = [];
+            if (options.coverage) {
+                promises.push(testPage.coverage.startJSCoverage());
+                promises.push(testPage.coverage.startCSSCoverage());
+            }
+            // When launching in non-headless mode (with a UI and dev-tools open), make sure
+            // the app has focus, to avoid failures of tests that query the UI's state.
+            if (launch && launch.devtools) {
+                promises.push(testPage.waitForSelector('#theia-app-shell.lm-Widget.theia-ApplicationShell')
+                    .then(e => {
+                        // eslint-disable-next-line no-null/no-null
+                        if (e !== null) {
+                            e.click();
+                        }
+                    }));
+            }
+
+            // Clear application's local storage to avoid reusing previous state
+            promises.push(testPage.evaluate(() => localStorage.clear()));
+            await Promise.all(promises);
+        },
+        onDidRun: async failures => {
+            if (options.coverage) {
+                console.log('collecting test coverage...');
+                const [jsCoverage, cssCoverage] = await Promise.all([
+                    testPage.coverage.stopJSCoverage(),
+                    testPage.coverage.stopCSSCoverage(),
+                ]);
+                require('puppeteer-to-istanbul').write([...jsCoverage, ...cssCoverage]);
+            }
+            if (exit) {
+                // allow a bit of time to finish printing-out test results
+                await new Promise(resolve => setTimeout(resolve, 1000));
+                await testPage.close();
+                process.exit(failures > 0 ? 1 : 0);
+            }
+        }
+    });
+    const { address, port } = await start();
+    const url = net.isIPv6(address)
+        ? `http://[${address}]:${port}`
+        : `http://${address}:${port}`;
+    await testPage.goto(url);
+    await testPage.bringToFront();
+}

dev-packages/cli/src/test-page.ts

@@ -0,0 +1,138 @@
+// *****************************************************************************
+// Copyright (C) 2020 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import * as puppeteer from 'puppeteer-core';
+const collectFiles: (options: TestFileOptions) => { files: string[] } = require('mocha/lib/cli/collect-files');
+
+export interface TestFileOptions {
+    ignore: string[]
+    extension: string[]
+    file: string[]
+    recursive: boolean
+    sort: boolean
+    spec: string[]
+}
+
+export interface TestPageOptions {
+    files?: Partial<TestFileOptions>
+    newPage: () => Promise<puppeteer.Page>
+    matchAppUrl?: (url: string) => boolean
+    onWillRun?: () => Promise<void>
+    onDidRun?: (failures: number) => Promise<void>
+}
+
+export default async function newTestPage(options: TestPageOptions): Promise<puppeteer.Page> {
+    const { newPage, matchAppUrl, onWillRun, onDidRun } = options;
+
+    const fileOptions: TestFileOptions = {
+        ignore: options.files && options.files.ignore || [],
+        extension: options.files && options.files.extension || [],
+        file: options.files && options.files.file || [],
+        spec: options.files && options.files.spec || [],
+        recursive: options.files && options.files.recursive || false,
+        sort: options.files && options.files.sort || false
+    };
+
+    // quick check whether test files exist
+    const files = collectFiles(fileOptions);
+
+    const page = await newPage();
+    page.on('dialog', dialog => dialog.dismiss());
+    page.on('pageerror', console.error);
+
+    let theiaLoaded = false;
+    page.exposeFunction('fireDidUnloadTheia', () => theiaLoaded = false);
+    const preLoad = (frame: puppeteer.Frame) => {
+        const frameUrl = frame.url();
+        if (matchAppUrl && !matchAppUrl(frameUrl)) {
+            return;
+        }
+        if (theiaLoaded) {
+            return;
+        }
+        console.log('loading chai...');
+        theiaLoaded = true;
+        page.addScriptTag({ path: require.resolve('chai/chai.js') });
+        page.evaluate(() =>
+            window.addEventListener('beforeunload', () => (window as any)['fireDidUnloadTheia']())
+        );
+    };
+    page.on('frameattached', preLoad);
+    page.on('framenavigated', preLoad);
+
+    page.on('load', async () => {
+        if (matchAppUrl && !matchAppUrl(page.url())) {
+            return;
+        }
+        console.log('loading mocha...');
+        // replace console.log by theia logger for mocha
+        await page.waitForFunction(() => !!(window as any)['theia']?.['@theia/core/lib/common/logger']?.logger, {
+            timeout: 30 * 1000
+        });
+        await page.addScriptTag({ path: require.resolve('mocha/mocha.js') });
+        await page.waitForFunction(() => !!(window as any)['chai'] && !!(window as any)['mocha'] && !!(window as any)['theia'].container, { timeout: 30 * 1000 });
+
+        console.log('loading Theia...');
+        await page.evaluate(() => {
+            const { FrontendApplicationStateService } = (window as any)['theia']['@theia/core/lib/browser/frontend-application-state'];
+            const { PreferenceService } = (window as any)['theia']['@theia/core/lib/common/preferences/preference-service'];
+            const { WorkspaceService } = (window as any)['theia']['@theia/workspace/lib/browser/workspace-service'];
+
+            const container = (window as any)['theia'].container;
+            const frontendApplicationState = container.get(FrontendApplicationStateService);
+            const preferenceService = container.get(PreferenceService);
+            const workspaceService = container.get(WorkspaceService);
+
+            return Promise.all([
+                frontendApplicationState.reachedState('ready'),
+                preferenceService.ready,
+                workspaceService.roots
+            ]);
+        });
+
+        console.log('loading test files...');
+        await page.evaluate(() => {
+            // replace require to load modules from theia namespace
+            (window as any)['require'] = (moduleName: string) => (window as any)['theia'][moduleName];
+            mocha.setup({
+                reporter: 'spec',
+                ui: 'bdd',
+                color: true,
+                retries: 0,
+                timeout: 10000
+            });
+        });
+
+        if (onWillRun) {
+            await onWillRun();
+        }
+
+        for (const file of files.files) {
+            await page.addScriptTag({ path: file });
+        }
+
+        console.log('running test files...');
+        const failures = await page.evaluate(() =>
+            new Promise<number>(resolve => mocha.run(resolve))
+        );
+        if (onDidRun) {
+            await onDidRun(failures);
+        }
+    });
+    return page;
+}

dev-packages/cli/src/theia.ts

@@ -0,0 +1,704 @@
+// *****************************************************************************
+// Copyright (C) 2017 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as temp from 'temp';
+import * as yargs from 'yargs';
+import yargsFactory = require('yargs/yargs');
+import { ApplicationPackageManager, rebuild } from '@theia/application-manager';
+import { ApplicationProps, DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package';
+import checkDependencies from './check-dependencies';
+import downloadPlugins from './download-plugins';
+import runTest from './run-test';
+import { RateLimiter } from 'limiter';
+import { LocalizationManager, extract } from '@theia/localization-manager';
+import { NodeRequestService } from '@theia/request/lib/node-request-service';
+import { ExtensionIdMatchesFilterFactory, OVSX_RATE_LIMIT, OVSXClient, OVSXHttpClient, OVSXRouterClient, RequestContainsFilterFactory } from '@theia/ovsx-client';
+
+const { executablePath } = require('puppeteer');
+
+process.on('unhandledRejection', (reason, promise) => {
+    throw reason;
+});
+process.on('uncaughtException', error => {
+    if (error) {
+        console.error('Uncaught Exception: ', error.toString());
+        if (error.stack) {
+            console.error(error.stack);
+        }
+    }
+    process.exit(1);
+});
+theiaCli();
+
+function toStringArray(argv: (string | number)[]): string[];
+function toStringArray(argv?: (string | number)[]): string[] | undefined;
+function toStringArray(argv?: (string | number)[]): string[] | undefined {
+    return argv?.map(arg => String(arg));
+}
+
+function rebuildCommand(command: string, target: ApplicationProps.Target): yargs.CommandModule<unknown, {
+    modules: string[]
+    cacheRoot?: string
+    forceAbi?: number,
+}> {
+    return {
+        command,
+        describe: `Rebuild/revert native node modules for "${target}"`,
+        builder: {
+            'cacheRoot': {
+                type: 'string',
+                describe: 'Root folder where to store the .browser_modules cache'
+            },
+            'modules': {
+                alias: 'm',
+                type: 'array', // === `--modules/-m` can be specified multiple times
+                describe: 'List of modules to rebuild/revert'
+            },
+            'forceAbi': {
+                type: 'number',
+                describe: 'The Node ABI version to rebuild for'
+            }
+        },
+        handler: ({ cacheRoot, modules, forceAbi }) => {
+            // Note: `modules` is actually `string[] | undefined`.
+            if (modules) {
+                // It is ergonomic to pass arguments as --modules="a,b,c,..."
+                // but yargs doesn't parse it this way by default.
+                const flattened: string[] = [];
+                for (const value of modules) {
+                    if (value.includes(',')) {
+                        flattened.push(...value.split(',').map(mod => mod.trim()));
+                    } else {
+                        flattened.push(value);
+                    }
+                }
+                modules = flattened;
+            }
+            rebuild(target, { cacheRoot, modules, forceAbi });
+        }
+    };
+}
+
+function defineCommonOptions<T>(cli: yargs.Argv<T>): yargs.Argv<T & {
+    appTarget?: 'browser' | 'electron' | 'browser-only'
+}> {
+    return cli
+        .option('app-target', {
+            description: 'The target application type. Overrides `theia.target` in the application\'s package.json',
+            choices: ['browser', 'electron', 'browser-only'] as const,
+        });
+}
+
+async function theiaCli(): Promise<void> {
+    const { version } = await fs.promises.readFile(path.join(__dirname, '../package.json'), 'utf8').then(JSON.parse);
+    yargs.scriptName('theia').version(version);
+    const projectPath = process.cwd();
+    // Create a sub `yargs` parser to read `app-target` without
+    // affecting the global `yargs` instance used by the CLI.
+    const { appTarget } = defineCommonOptions(yargsFactory()).help(false).parse();
+    const manager = new ApplicationPackageManager({ projectPath, appTarget });
+    const localizationManager = new LocalizationManager();
+    const { target } = manager.pck;
+    defineCommonOptions(yargs)
+        .command<{
+            theiaArgs?: (string | number)[]
+        }>({
+            command: 'start [theia-args...]',
+            describe: `Start the ${target} backend`,
+            // Disable this command's `--help` option so that it is forwarded to Theia's CLI
+            builder: cli => cli.help(false) as yargs.Argv,
+            handler: async ({ theiaArgs }) => {
+                manager.start(toStringArray(theiaArgs));
+            }
+        })
+        .command({
+            command: 'clean',
+            describe: `Clean for the ${target} target`,
+            handler: async () => {
+                await manager.clean();
+            }
+        })
+        .command({
+            command: 'copy',
+            describe: 'Copy various files from `src-gen` to `lib`',
+            handler: async () => {
+                await manager.copy();
+            }
+        })
+        .command<{
+            mode: 'development' | 'production',
+            splitFrontend?: boolean
+        }>({
+            command: 'generate',
+            describe: `Generate various files for the ${target} target`,
+            builder: cli => ApplicationPackageManager.defineGeneratorOptions(cli),
+            handler: async ({ mode, splitFrontend }) => {
+                await manager.generate({ mode, splitFrontend });
+            }
+        })
+        .command<{
+            mode: 'development' | 'production',
+            webpackHelp: boolean
+            splitFrontend?: boolean
+            webpackArgs?: (string | number)[]
+        }>({
+            command: 'build [webpack-args...]',
+            describe: `Generate and bundle the ${target} frontend using webpack`,
+            builder: cli => ApplicationPackageManager.defineGeneratorOptions(cli)
+                .option('webpack-help' as 'webpackHelp', {
+                    boolean: true,
+                    description: 'Display Webpack\'s help',
+                    default: false
+                }),
+            handler: async ({ mode, splitFrontend, webpackHelp, webpackArgs = [] }) => {
+                await manager.build(
+                    webpackHelp
+                        ? ['--help']
+                        : [
+                            // Forward the `mode` argument to Webpack too:
+                            '--mode', mode,
+                            ...toStringArray(webpackArgs)
+                        ],
+                    { mode, splitFrontend }
+                );
+            }
+        })
+        .command(rebuildCommand('rebuild', target))
+        .command(rebuildCommand('rebuild:browser', 'browser'))
+        .command(rebuildCommand('rebuild:electron', 'electron'))
+        .command<{
+            suppress: boolean
+        }>({
+            command: 'check:hoisted',
+            describe: 'Check that all dependencies are hoisted',
+            builder: {
+                'suppress': {
+                    alias: 's',
+                    describe: 'Suppress exiting with failure code',
+                    boolean: true,
+                    default: false
+                }
+            },
+            handler: ({ suppress }) => {
+                checkDependencies({
+                    workspaces: ['packages/*'],
+                    include: ['**'],
+                    exclude: ['.bin/**', '.cache/**'],
+                    skipHoisted: false,
+                    skipUniqueness: true,
+                    skipSingleTheiaVersion: true,
+                    onlyTheiaExtensions: false,
+                    suppress
+                });
+            }
+        })
+        .command<{
+            suppress: boolean
+        }>({
+            command: 'check:theia-version',
+            describe: 'Check that all dependencies have been resolved to the same Theia version',
+            builder: {
+                'suppress': {
+                    alias: 's',
+                    describe: 'Suppress exiting with failure code',
+                    boolean: true,
+                    default: false
+                }
+            },
+            handler: ({ suppress }) => {
+                checkDependencies({
+                    workspaces: undefined,
+                    include: ['@theia/**'],
+                    exclude: [],
+                    skipHoisted: true,
+                    skipUniqueness: false,
+                    skipSingleTheiaVersion: false,
+                    onlyTheiaExtensions: false,
+                    suppress
+                });
+            }
+        })
+        .command<{
+            suppress: boolean
+        }>({
+            command: 'check:theia-extensions',
+            describe: 'Check uniqueness of Theia extension versions or whether they are hoisted',
+            builder: {
+                'suppress': {
+                    alias: 's',
+                    describe: 'Suppress exiting with failure code',
+                    boolean: true,
+                    default: false
+                }
+            },
+            handler: ({ suppress }) => {
+                checkDependencies({
+                    workspaces: undefined,
+                    include: ['**'],
+                    exclude: [],
+                    skipHoisted: true,
+                    skipUniqueness: false,
+                    skipSingleTheiaVersion: true,
+                    onlyTheiaExtensions: true,
+                    suppress
+                });
+            }
+        })
+        .command<{
+            workspaces: string[] | undefined,
+            include: string[],
+            exclude: string[],
+            skipHoisted: boolean,
+            skipUniqueness: boolean,
+            skipSingleTheiaVersion: boolean,
+            onlyTheiaExtensions: boolean,
+            suppress: boolean
+        }>({
+            command: 'check:dependencies',
+            describe: 'Check uniqueness of dependency versions or whether they are hoisted',
+            builder: {
+                'workspaces': {
+                    alias: 'w',
+                    describe: 'Glob patterns of workspaces to analyze, relative to `cwd`',
+                    array: true,
+                    defaultDescription: 'All glob patterns listed in the package.json\'s workspaces',
+                    demandOption: false
+                },
+                'include': {
+                    alias: 'i',
+                    describe: 'Glob pattern of dependencies\' package names to be included, e.g. -i "@theia/**"',
+                    array: true,
+                    default: ['**']
+                },
+                'exclude': {
+                    alias: 'e',
+                    describe: 'Glob pattern of dependencies\' package names to be excluded',
+                    array: true,
+                    defaultDescription: 'None',
+                    default: []
+                },
+                'skip-hoisted': {
+                    alias: 'h',
+                    describe: 'Skip checking whether dependencies are hoisted',
+                    boolean: true,
+                    default: false
+                },
+                'skip-uniqueness': {
+                    alias: 'u',
+                    describe: 'Skip checking whether all dependencies are resolved to a unique version',
+                    boolean: true,
+                    default: false
+                },
+                'skip-single-theia-version': {
+                    alias: 't',
+                    describe: 'Skip checking whether all @theia/* dependencies are resolved to a single version',
+                    boolean: true,
+                    default: false
+                },
+                'only-theia-extensions': {
+                    alias: 'o',
+                    describe: 'Only check dependencies which are Theia extensions',
+                    boolean: true,
+                    default: false
+                },
+                'suppress': {
+                    alias: 's',
+                    describe: 'Suppress exiting with failure code',
+                    boolean: true,
+                    default: false
+                }
+            },
+            handler: ({
+                workspaces,
+                include,
+                exclude,
+                skipHoisted,
+                skipUniqueness,
+                skipSingleTheiaVersion,
+                onlyTheiaExtensions,
+                suppress
+            }) => {
+                checkDependencies({
+                    workspaces,
+                    include,
+                    exclude,
+                    skipHoisted,
+                    skipUniqueness,
+                    skipSingleTheiaVersion,
+                    onlyTheiaExtensions,
+                    suppress
+                });
+            }
+        })
+        .command<{
+            packed: boolean
+            ignoreErrors: boolean
+            apiVersion: string
+            apiUrl: string
+            parallel: boolean
+            proxyUrl?: string
+            proxyAuthorization?: string
+            strictSsl: boolean
+            rateLimit: number
+            ovsxRouterConfig?: string
+        }>({
+            command: 'download:plugins',
+            describe: 'Download defined external plugins',
+            builder: {
+                'packed': {
+                    alias: 'p',
+                    describe: 'Controls whether to pack or unpack plugins',
+                    boolean: true,
+                    default: false,
+                },
+                'ignore-errors': {
+                    alias: 'i',
+                    describe: 'Ignore errors while downloading plugins',
+                    boolean: true,
+                    default: false,
+                },
+                'api-version': {
+                    alias: 'v',
+                    describe: 'Supported API version for plugins',
+                    default: DEFAULT_SUPPORTED_API_VERSION
+                },
+                'api-url': {
+                    alias: 'u',
+                    describe: 'Open-VSX Registry API URL',
+                    default: 'https://open-vsx.org/api'
+                },
+                'parallel': {
+                    describe: 'Download in parallel',
+                    boolean: true,
+                    default: true
+                },
+                'rate-limit': {
+                    describe: 'Amount of maximum open-vsx requests per second',
+                    number: true,
+                    default: OVSX_RATE_LIMIT
+                },
+                'proxy-url': {
+                    describe: 'Proxy URL'
+                },
+                'proxy-authorization': {
+                    describe: 'Proxy authorization information'
+                },
+                'strict-ssl': {
+                    describe: 'Whether to enable strict SSL mode',
+                    boolean: true,
+                    default: false
+                },
+                'ovsx-router-config': {
+                    describe: 'JSON configuration file for the OVSX router client',
+                    type: 'string'
+                }
+            },
+            handler: async ({ apiUrl, proxyUrl, proxyAuthorization, strictSsl, ovsxRouterConfig, ...options }) => {
+                const requestService = new NodeRequestService();
+                await requestService.configure({
+                    proxyUrl,
+                    proxyAuthorization,
+                    strictSSL: strictSsl
+                });
+                let client: OVSXClient | undefined;
+                const rateLimiter = new RateLimiter({ tokensPerInterval: options.rateLimit, interval: 'second' });
+                if (ovsxRouterConfig) {
+                    const routerConfig = await fs.promises.readFile(ovsxRouterConfig, 'utf8').then(JSON.parse, error => {
+                        console.error(error);
+                    });
+                    if (routerConfig) {
+                        client = await OVSXRouterClient.FromConfig(
+                            routerConfig,
+                            OVSXHttpClient.createClientFactory(requestService, rateLimiter),
+                            [RequestContainsFilterFactory, ExtensionIdMatchesFilterFactory]
+                        );
+                    }
+                }
+                if (!client) {
+                    client = new OVSXHttpClient(apiUrl, requestService, rateLimiter);
+                }
+                try {
+                    await downloadPlugins(client, rateLimiter, requestService, options);
+                } catch (error) {
+                    console.error(error);
+                    process.exit(1);
+                }
+                process.exit(0);
+            },
+        })
+        .command<{
+            freeApi?: boolean,
+            deeplKey: string,
+            file: string,
+            languages: string[],
+            sourceLanguage?: string
+        }>({
+            command: 'nls-localize [languages...]',
+            describe: 'Localize json files using the DeepL API',
+            builder: {
+                'file': {
+                    alias: 'f',
+                    describe: 'The source file which should be translated',
+                    demandOption: true
+                },
+                'deepl-key': {
+                    alias: 'k',
+                    describe: 'DeepL key used for API access. See https://www.deepl.com/docs-api for more information',
+                    demandOption: true
+                },
+                'free-api': {
+                    describe: 'Indicates whether the specified DeepL API key belongs to the free API',
+                    boolean: true,
+                    default: false,
+                },
+                'source-language': {
+                    alias: 's',
+                    describe: 'The source language of the translation file'
+                }
+            },
+            handler: async ({ freeApi, deeplKey, file, sourceLanguage, languages = [] }) => {
+                const success = await localizationManager.localize({
+                    sourceFile: file,
+                    freeApi: freeApi ?? true,
+                    authKey: deeplKey,
+                    targetLanguages: languages,
+                    sourceLanguage
+                });
+                if (!success) {
+                    process.exit(1);
+                }
+            }
+        })
+        .command<{
+            root: string,
+            output: string,
+            merge: boolean,
+            exclude?: string,
+            logs?: string,
+            files?: string[],
+            quiet: boolean
+        }>({
+            command: 'nls-extract',
+            describe: 'Extract translation key/value pairs from source code',
+            builder: {
+                'output': {
+                    alias: 'o',
+                    describe: 'Output file for the extracted translations',
+                    demandOption: true
+                },
+                'root': {
+                    alias: 'r',
+                    describe: 'The directory which contains the source code',
+                    default: '.'
+                },
+                'merge': {
+                    alias: 'm',
+                    describe: 'Whether to merge new with existing translation values',
+                    boolean: true,
+                    default: false
+                },
+                'exclude': {
+                    alias: 'e',
+                    describe: 'Allows to exclude translation keys starting with this value'
+                },
+                'files': {
+                    alias: 'f',
+                    describe: 'Glob pattern matching the files to extract from (starting from --root).',
+                    array: true
+                },
+                'logs': {
+                    alias: 'l',
+                    describe: 'File path to a log file'
+                },
+                'quiet': {
+                    alias: 'q',
+                    describe: 'Prevents errors from being logged to console',
+                    boolean: true,
+                    default: false
+                }
+            },
+            handler: async options => {
+                await extract(options);
+            }
+        })
+        .command<{
+            testInspect: boolean,
+            testExtension: string[],
+            testFile: string[],
+            testIgnore: string[],
+            testRecursive: boolean,
+            testSort: boolean,
+            testSpec: string[],
+            testCoverage: boolean
+            theiaArgs?: (string | number)[]
+        }>({
+            command: 'test [theia-args...]',
+            builder: {
+                'test-inspect': {
+                    describe: 'Whether to auto-open a DevTools panel for test page.',
+                    boolean: true,
+                    default: false
+                },
+                'test-extension': {
+                    describe: 'Test file extension(s) to load',
+                    array: true,
+                    default: ['js']
+                },
+                'test-file': {
+                    describe: 'Specify test file(s) to be loaded prior to root suite execution',
+                    array: true,
+                    default: []
+                },
+                'test-ignore': {
+                    describe: 'Ignore test file(s) or glob pattern(s)',
+                    array: true,
+                    default: []
+                },
+                'test-recursive': {
+                    describe: 'Look for tests in subdirectories',
+                    boolean: true,
+                    default: false
+                },
+                'test-sort': {
+                    describe: 'Sort test files',
+                    boolean: true,
+                    default: false
+                },
+                'test-spec': {
+                    describe: 'One or more test files, directories, or globs to test',
+                    array: true,
+                    default: ['test']
+                },
+                'test-coverage': {
+                    describe: 'Report test coverage consumable by istanbul',
+                    boolean: true,
+                    default: false
+                }
+            },
+            handler: async ({ testInspect, testExtension, testFile, testIgnore, testRecursive, testSort, testSpec, testCoverage, theiaArgs }) => {
+                if (!process.env.THEIA_CONFIG_DIR) {
+                    process.env.THEIA_CONFIG_DIR = temp.track().mkdirSync('theia-test-config-dir');
+                }
+                const args = ['--no-sandbox'];
+                if (!testInspect) {
+                    args.push('--headless=old');
+                }
+                await runTest({
+                    start: () => new Promise((resolve, reject) => {
+                        const serverProcess = manager.start(toStringArray(theiaArgs));
+                        serverProcess.on('message', resolve);
+                        serverProcess.on('error', reject);
+                        serverProcess.on('close', (code, signal) => reject(`Server process exited unexpectedly: ${code ?? signal}`));
+                    }),
+                    launch: {
+                        args: args,
+                        // eslint-disable-next-line no-null/no-null
+                        defaultViewport: null, // view port can take available space instead of 800x600 default
+                        devtools: testInspect,
+                        headless: testInspect ? false : 'shell',
+                        executablePath: executablePath(),
+                        protocolTimeout: 600000,
+                        timeout: 60000
+                    },
+                    files: {
+                        extension: testExtension,
+                        file: testFile,
+                        ignore: testIgnore,
+                        recursive: testRecursive,
+                        sort: testSort,
+                        spec: testSpec
+                    },
+                    coverage: testCoverage
+                });
+            }
+        })
+        .command<{
+            electronVersion?: string
+            electronDist?: string
+            ffmpegPath?: string
+            platform?: NodeJS.Platform
+        }>({
+            command: 'ffmpeg:replace [ffmpeg-path]',
+            describe: '',
+            builder: {
+                'electronDist': {
+                    description: 'Electron distribution location.',
+                },
+                'electronVersion': {
+                    description: 'Electron version for which to pull the "clean" ffmpeg library.',
+                },
+                'ffmpegPath': {
+                    description: 'Absolute path to the ffmpeg shared library.',
+                },
+                'platform': {
+                    description: 'Dictates where the library is located within the Electron distribution.',
+                    choices: ['darwin', 'linux', 'win32'] as NodeJS.Platform[],
+                },
+            },
+            handler: async options => {
+                const ffmpeg = await import('@theia/ffmpeg');
+                await ffmpeg.replaceFfmpeg(options);
+            },
+        })
+        .command<{
+            electronDist?: string
+            ffmpegPath?: string
+            json?: boolean
+            platform?: NodeJS.Platform
+        }>({
+            command: 'ffmpeg:check [ffmpeg-path]',
+            describe: '(electron-only) Check that ffmpeg doesn\'t contain proprietary codecs',
+            builder: {
+                'electronDist': {
+                    description: 'Electron distribution location',
+                },
+                'ffmpegPath': {
+                    describe: 'Absolute path to the ffmpeg shared library',
+                },
+                'json': {
+                    description: 'Output the found codecs as JSON on stdout',
+                    boolean: true,
+                },
+                'platform': {
+                    description: 'Dictates where the library is located within the Electron distribution',
+                    choices: ['darwin', 'linux', 'win32'] as NodeJS.Platform[],
+                },
+            },
+            handler: async options => {
+                const ffmpeg = await import('@theia/ffmpeg');
+                await ffmpeg.checkFfmpeg(options);
+            },
+        })
+        .parserConfiguration({
+            'unknown-options-as-args': true,
+        })
+        .strictCommands()
+        .demandCommand(1, 'Please run a command')
+        .fail((msg, err, cli) => {
+            process.exitCode = 1;
+            if (err) {
+                // One of the handlers threw an error:
+                console.error(err);
+            } else {
+                // Yargs detected a problem with commands and/or arguments while parsing:
+                cli.showHelp();
+                console.error(msg);
+            }
+        })
+        .parse();
+}