deploy: current vibn theia state

Made-with: Cursor

dev-packages/localization-manager/.eslintrc.js

@@ -0,0 +1,13 @@
+/** @type {import('eslint').Linter.Config} */
+module.exports = {
+    extends: [
+        '../../configs/build.eslintrc.json'
+    ],
+    parserOptions: {
+        tsconfigRootDir: __dirname,
+        project: 'tsconfig.json'
+    },
+    rules: {
+        'import/no-dynamic-require': 'off'
+    }
+};

dev-packages/localization-manager/README.md

@@ -0,0 +1,67 @@
+<div align='center'>
+
+<br />
+
+<img src='https://raw.githubusercontent.com/eclipse-theia/theia/master/logo/theia.svg?sanitize=true' alt='theia-ext-logo' width='100px' />
+
+<h2>ECLIPSE THEIA - LOCALIZATION-MANAGER</h2>
+
+<hr />
+
+</div>
+
+## Description
+
+The `@theia/localization-manager` package is used easily create localizations of Theia and Theia extensions for different languages. It has two main use cases.
+
+First, it allows to extract localization keys and default values from `nls.localize` calls within the codebase using the `nls-extract` Theia-CLI command. Take this code for example:
+
+```ts
+const hi = nls.localize('greetings/hi', 'Hello');
+const bye = nls.localize('greetings/bye', 'Bye');
+```
+
+It will be converted into this JSON file (`nls.json`):
+
+```json
+{
+  "greetings": {
+    "hi": "Hello",
+    "bye": "Bye"
+  }
+}
+```
+
+Afterwards, any manual or automatic translation approach can be used to translate this file into other languages. These JSON files are supposed to be picked up by `LocalizationContribution`s.
+
+Additionally, Theia provides a simple way to translate the generated JSON files out of the box using the [DeepL API](https://www.deepl.com/docs-api). For this, a [DeepL free or pro account](https://www.deepl.com/pro) is needed. Using the `nls-localize` command of the Theia-CLI, a target file can be translated into different languages. For example, when calling the command using the previous JSON file with the `fr` (french) language, the following `nls.fr.json` file will be created in the same directory as the translation source:
+
+```json
+{
+  "greetings": {
+    "hi": "Bonjour",
+    "bye": "Au revoir"
+  }
+}
+```
+
+Only JSON entries without corresponding translations are translated using DeepL. This ensures that manual changes to the translated files aren't overwritten and only new translation entries are actually sent to DeepL.
+
+Use `theia nls-localize --help` for more information on how to use the command and supply DeepL API keys.
+
+For more information, see the [internationalization documentation](https://theia-ide.org/docs/i18n/).
+
+## Additional Information
+
+- [Theia - GitHub](https://github.com/eclipse-theia/theia)
+- [Theia - Website](https://theia-ide.org/)
+
+## License
+
+- [Eclipse Public License 2.0](http://www.eclipse.org/legal/epl-2.0/)
+- [一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception](https://projects.eclipse.org/license/secondary-gpl-2.0-cp)
+
+## Trademark
+
+"Theia" is a trademark of the Eclipse Foundation
+<https://www.eclipse.org/theia>

dev-packages/localization-manager/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@theia/localization-manager",
+  "version": "1.68.0",
+  "description": "Theia localization manager API.",
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/eclipse-theia/theia.git"
+  },
+  "bugs": {
+    "url": "https://github.com/eclipse-theia/theia/issues"
+  },
+  "homepage": "https://github.com/eclipse-theia/theia",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "scripts": {
+    "build": "theiaext build",
+    "clean": "theiaext clean",
+    "compile": "theiaext compile",
+    "lint": "theiaext lint",
+    "test": "theiaext test",
+    "watch": "theiaext watch"
+  },
+  "dependencies": {
+    "@types/bent": "^7.0.1",
+    "@types/fs-extra": "^4.0.2",
+    "bent": "^7.1.0",
+    "chalk": "4.0.0",
+    "deepmerge": "^4.2.2",
+    "fs-extra": "^4.0.2",
+    "glob": "^7.2.0",
+    "limiter": "^2.1.0",
+    "tslib": "^2.6.2",
+    "typescript": "~5.9.3"
+  },
+  "devDependencies": {
+    "@theia/ext-scripts": "1.68.0"
+  },
+  "nyc": {
+    "extends": "../../configs/nyc.json"
+  },
+  "gitHead": "21358137e41342742707f660b8e222f940a27652"
+}

dev-packages/localization-manager/src/common.ts

@@ -0,0 +1,27 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+export interface Localization {
+    [key: string]: string | Localization
+}
+
+export function sortLocalization(localization: Localization): Localization {
+    return Object.keys(localization).sort().reduce((result: Localization, key: string) => {
+        const value = localization[key];
+        result[key] = typeof value === 'string' ? value : sortLocalization(value);
+        return result;
+    }, {});
+}

dev-packages/localization-manager/src/deepl-api.ts

@@ -0,0 +1,190 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as bent from 'bent';
+import { RateLimiter } from 'limiter';
+
+const post = bent('POST', 'json', 200);
+// 50 is the maximum amount of translations per request
+const deeplLimit = 50;
+const rateLimiter = new RateLimiter({
+    tokensPerInterval: 10,
+    interval: 'second',
+    fireImmediately: true
+});
+
+export async function deepl(
+    parameters: DeeplParameters
+): Promise<DeeplResponse> {
+    coerceLanguage(parameters);
+    const sub_domain = parameters.free_api ? 'api-free' : 'api';
+    const textChunks: string[][] = [];
+    const textArray = [...parameters.text];
+    while (textArray.length > 0) {
+        textChunks.push(textArray.splice(0, deeplLimit));
+    }
+    const responses: DeeplResponse[] = await Promise.all(textChunks.map(async chunk => {
+        const parameterCopy: DeeplParameters = { ...parameters, text: chunk };
+        const url = `https://${sub_domain}.deepl.com/v2/translate`;
+        const buffer = Buffer.from(toFormData(parameterCopy));
+        return postWithRetry(url, parameters.auth_key, buffer, 1);
+    }));
+    const mergedResponse: DeeplResponse = { translations: [] };
+    for (const response of responses) {
+        mergedResponse.translations.push(...response.translations);
+    }
+    for (const translation of mergedResponse.translations) {
+        translation.text = coerceTranslation(translation.text);
+    }
+    return mergedResponse;
+}
+
+async function postWithRetry(url: string, key: string, buffer: Buffer, attempt: number): Promise<DeeplResponse> {
+    try {
+        await rateLimiter.removeTokens(Math.min(attempt, 10));
+        const response = await post(url, buffer, {
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'User-Agent': 'Theia-Localization-Manager',
+            'Authorization': 'DeepL-Auth-Key ' + key
+        });
+        return response;
+    } catch (e) {
+        if ('message' in e && typeof e.message === 'string' && e.message.includes('Too Many Requests')) {
+            return postWithRetry(url, key, buffer, attempt + 1);
+        }
+        throw e;
+    }
+}
+
+/**
+ * Coerces the target language into a form expected by Deepl.
+ *
+ * Currently only replaces `ZH-CN` with `ZH`
+ */
+function coerceLanguage(parameters: DeeplParameters): void {
+    if (parameters.target_lang === 'ZH-CN') {
+        parameters.target_lang = 'ZH-HANS';
+    } else if (parameters.target_lang === 'ZH-TW') {
+        parameters.target_lang = 'ZH-HANT';
+    }
+}
+
+/**
+ * Coerces translated text into a form expected by VSCode/Theia.
+ *
+ * Replaces certain full-width characters with their ascii counter-part.
+ */
+function coerceTranslation(text: string): string {
+    return text
+        .replace(/\uff08/g, '(')
+        .replace(/\uff09/g, ')')
+        .replace(/\uff0c/g, ',')
+        .replace(/\uff1a/g, ':')
+        .replace(/\uff1b/g, ';')
+        .replace(/\uff1f/g, '?');
+}
+
+function toFormData(parameters: DeeplParameters): string {
+    const str: string[] = [];
+    for (const [key, value] of Object.entries(parameters)) {
+        if (typeof value === 'string') {
+            str.push(encodeURIComponent(key) + '=' + encodeURIComponent(value.toString()));
+        } else if (Array.isArray(value)) {
+            for (const item of value) {
+                str.push(encodeURIComponent(key) + '=' + encodeURIComponent(item.toString()));
+            }
+        }
+    }
+    return str.join('&');
+}
+
+export type DeeplLanguage =
+    | 'BG'
+    | 'CS'
+    | 'DA'
+    | 'DE'
+    | 'EL'
+    | 'EN-GB'
+    | 'EN-US'
+    | 'EN'
+    | 'ES'
+    | 'ET'
+    | 'FI'
+    | 'FR'
+    | 'HU'
+    | 'ID'
+    | 'IT'
+    | 'JA'
+    | 'KO'
+    | 'LT'
+    | 'LV'
+    | 'NB'
+    | 'NL'
+    | 'PL'
+    | 'PT-PT'
+    | 'PT-BR'
+    | 'PT'
+    | 'RO'
+    | 'RU'
+    | 'SK'
+    | 'SL'
+    | 'SV'
+    | 'TR'
+    | 'UK'
+    | 'ZH-CN'
+    | 'ZH-TW'
+    | 'ZH-HANS'
+    | 'ZH-HANT'
+    | 'ZH';
+
+export const supportedLanguages = [
+    'BG', 'CS', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT',
+    'JA', 'KO', 'LT', 'LV', 'NL', 'PL', 'PT-PT', 'PT-BR', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH-CN', 'ZH-TW'
+];
+
+// From https://code.visualstudio.com/docs/getstarted/locales#_available-locales
+export const defaultLanguages = [
+    'ZH-CN', 'ZH-TW', 'FR', 'DE', 'IT', 'ES', 'JA', 'KO', 'RU', 'PT-BR', 'TR', 'PL', 'CS', 'HU'
+] as const;
+
+export function isSupportedLanguage(language: string): language is DeeplLanguage {
+    return supportedLanguages.includes(language.toUpperCase());
+}
+
+export interface DeeplParameters {
+    free_api: Boolean
+    auth_key: string
+    text: string[]
+    source_lang?: DeeplLanguage
+    target_lang: DeeplLanguage
+    split_sentences?: '0' | '1' | 'nonewlines'
+    preserve_formatting?: '0' | '1'
+    formality?: 'default' | 'more' | 'less'
+    tag_handling?: string[]
+    non_splitting_tags?: string[]
+    outline_detection?: string
+    splitting_tags?: string[]
+    ignore_tags?: string[]
+}
+
+export interface DeeplResponse {
+    translations: DeeplTranslation[]
+}
+
+export interface DeeplTranslation {
+    detected_source_language: string
+    text: string
+}

dev-packages/localization-manager/src/index.ts

@@ -0,0 +1,19 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+export * from './common';
+export * from './localization-extractor';
+export * from './localization-manager';

dev-packages/localization-manager/src/localization-extractor.spec.ts

@@ -0,0 +1,151 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as assert from 'assert';
+import { extractFromFile, ExtractionOptions } from './localization-extractor';
+
+const TEST_FILE = 'test.ts';
+const quiet: ExtractionOptions = { quiet: true };
+
+describe('correctly extracts from file content', () => {
+
+    it('should extract from simple nls.localize() call', async () => {
+        const content = 'nls.localize("key", "value")';
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content), {
+            'key': 'value'
+        });
+    });
+
+    it('should extract from nested nls.localize() call', async () => {
+        const content = 'nls.localize("nested/key", "value")';
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content), {
+            'nested': {
+                'key': 'value'
+            }
+        });
+    });
+
+    it('should extract IDs from Command.toLocalizedCommand() call', async () => {
+        const content = `
+        Command.toLocalizedCommand({
+            id: 'command-id1',
+            label: 'command-label1'
+        });
+        Command.toLocalizedCommand({
+            id: 'command-id2',
+            label: 'command-label2'
+        }, 'command-key');
+        `;
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content), {
+            'command-id1': 'command-label1',
+            'command-key': 'command-label2'
+        });
+    });
+
+    it('should extract category from Command.toLocalizedCommand() call', async () => {
+        const content = `
+        Command.toLocalizedCommand({
+            id: 'id',
+            label: 'label',
+            category: 'category'
+        }, undefined, 'category-key');`;
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content), {
+            'id': 'label',
+            'category-key': 'category'
+        });
+    });
+
+    it('should merge different nls.localize() calls', async () => {
+        const content = `
+        nls.localize('nested/key1', 'value1');
+        nls.localize('nested/key2', 'value2');
+        `;
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content), {
+            'nested': {
+                'key1': 'value1',
+                'key2': 'value2'
+            }
+        });
+    });
+
+    it('should be able to resolve local references', async () => {
+        const content = `
+        const a = 'key';
+        nls.localize(a, 'value');
+        `;
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content), {
+            'key': 'value'
+        });
+    });
+
+    it('should return an error when resolving is not successful', async () => {
+        const content = "nls.localize(a, 'value')";
+        const errors: string[] = [];
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {});
+        assert.deepStrictEqual(errors, [
+            "test.ts(1,14): Could not resolve reference to 'a'"
+        ]);
+    });
+
+    it('should return an error when resolving from an expression', async () => {
+        const content = "nls.localize(test.value, 'value');";
+        const errors: string[] = [];
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {});
+        assert.deepStrictEqual(errors, [
+            "test.ts(1,14): 'test.value' is not a string constant"
+        ]);
+    });
+
+    it('should show error when trying to merge an object and a string', async () => {
+        const content = `
+        nls.localize('key', 'value');
+        nls.localize('key/nested', 'value');
+        `.trim();
+        const errors: string[] = [];
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {
+            'key': 'value'
+        });
+        assert.deepStrictEqual(errors, [
+            "test.ts(2,35): String entry already exists at 'key'"
+        ]);
+    });
+
+    it('should show error when trying to merge a string into an object', async () => {
+        const content = `
+        nls.localize('key/nested', 'value');
+        nls.localize('key', 'value');
+        `.trim();
+        const errors: string[] = [];
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {
+            'key': {
+                'nested': 'value'
+            }
+        });
+        assert.deepStrictEqual(errors, [
+            "test.ts(2,28): Multiple translation keys already exist at 'key'"
+        ]);
+    });
+
+    it('should show error for template literals', async () => {
+        const content = 'nls.localize("key", `template literal value`)';
+        const errors: string[] = [];
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {});
+        assert.deepStrictEqual(errors, [
+            "test.ts(1,20): Template literals are not supported for localization. Please use the additional arguments of the 'nls.localize' function to format strings"
+        ]);
+    });
+
+});

dev-packages/localization-manager/src/localization-extractor.ts

@@ -0,0 +1,431 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as fs from 'fs-extra';
+import * as ts from 'typescript';
+import * as os from 'os';
+import * as path from 'path';
+import { glob } from 'glob';
+import { promisify } from 'util';
+import deepmerge = require('deepmerge');
+import { Localization, sortLocalization } from './common';
+
+const globPromise = promisify(glob);
+
+export interface ExtractionOptions {
+    root?: string
+    output?: string
+    exclude?: string
+    logs?: string
+    /** List of globs matching the files to extract from. */
+    files?: string[]
+    merge?: boolean
+    quiet?: boolean
+}
+
+class SingleFileServiceHost implements ts.LanguageServiceHost {
+
+    private file: ts.IScriptSnapshot;
+    private lib: ts.IScriptSnapshot;
+
+    constructor(private options: ts.CompilerOptions, private filename: string, contents: string) {
+        this.file = ts.ScriptSnapshot.fromString(contents);
+        this.lib = ts.ScriptSnapshot.fromString('');
+    }
+
+    getCompilationSettings = () => this.options;
+    getScriptFileNames = () => [this.filename];
+    getScriptVersion = () => '1';
+    getScriptSnapshot = (name: string) => name === this.filename ? this.file : this.lib;
+    getCurrentDirectory = () => '';
+    getDefaultLibFileName = () => 'lib.d.ts';
+    readFile(file: string, encoding?: string | undefined): string | undefined {
+        if (file === this.filename) {
+            return this.file.getText(0, this.file.getLength());
+        }
+    }
+    fileExists(file: string): boolean {
+        return this.filename === file;
+    }
+}
+
+class TypeScriptError extends Error {
+    constructor(message: string, node: ts.Node) {
+        super(buildErrorMessage(message, node));
+    }
+}
+
+function buildErrorMessage(message: string, node: ts.Node): string {
+    const source = node.getSourceFile();
+    const sourcePath = source.fileName;
+    const pos = source.getLineAndCharacterOfPosition(node.pos);
+    return `${sourcePath}(${pos.line + 1},${pos.character + 1}): ${message}`;
+}
+
+const tsOptions: ts.CompilerOptions = {
+    allowJs: true
+};
+
+export async function extract(options: ExtractionOptions): Promise<void> {
+    const cwd = path.resolve(process.env.INIT_CWD || process.cwd(), options.root ?? '');
+    const files: string[] = [];
+    await Promise.all((options.files ?? ['**/src/**/*.{ts,tsx}']).map(
+        async pattern => files.push(...await globPromise(pattern, { cwd }))
+    ));
+    let localization: Localization = {};
+    const errors: string[] = [];
+    for (const file of files) {
+        const filePath = path.resolve(cwd, file);
+        const fileName = path.relative(cwd, file).split(path.sep).join('/');
+        const content = await fs.readFile(filePath, 'utf8');
+        const fileLocalization = await extractFromFile(fileName, content, errors, options);
+        localization = deepmerge(localization, fileLocalization);
+    }
+    if (errors.length > 0 && options.logs) {
+        await fs.writeFile(options.logs, errors.join(os.EOL));
+    }
+    const out = path.resolve(process.env.INIT_CWD || process.cwd(), options.output ?? '');
+    if (options.merge && await fs.pathExists(out)) {
+        const existing = await fs.readJson(out);
+        localization = deepmerge(existing, localization);
+    }
+    localization = sortLocalization(localization);
+    await fs.mkdirs(path.dirname(out));
+    await fs.writeJson(out, localization, {
+        spaces: 2
+    });
+}
+
+export async function extractFromFile(file: string, content: string, errors?: string[], options?: ExtractionOptions): Promise<Localization> {
+    const serviceHost = new SingleFileServiceHost(tsOptions, file, content);
+    const service = ts.createLanguageService(serviceHost);
+    const sourceFile = service.getProgram()!.getSourceFile(file)!;
+    const localization: Localization = {};
+    const localizationCalls = collect(sourceFile, node => isLocalizeCall(node));
+    for (const call of localizationCalls) {
+        try {
+            const extracted = extractFromLocalizeCall(call, options);
+            if (extracted) {
+                insert(localization, extracted);
+            }
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
+        }
+    }
+    const localizedCommands = collect(sourceFile, node => isCommandLocalizeUtility(node));
+    for (const command of localizedCommands) {
+        try {
+            const extracted = extractFromLocalizedCommandCall(command, errors, options);
+            const label = extracted.label;
+            const category = extracted.category;
+            if (!isExcluded(options, label[0])) {
+                insert(localization, label);
+            }
+            if (category && !isExcluded(options, category[0])) {
+                insert(localization, category);
+            }
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
+        }
+    }
+    return localization;
+}
+
+function isExcluded(options: ExtractionOptions | undefined, key: string): boolean {
+    return !!options?.exclude && key.startsWith(options.exclude);
+}
+
+function insert(localization: Localization, values: [string, string, ts.Node]): void {
+    const key = values[0];
+    const value = values[1];
+    const node = values[2];
+    const parts = key.split('/');
+    parts.forEach((part, i) => {
+        let entry = localization[part];
+        if (i === parts.length - 1) {
+            if (typeof entry === 'object') {
+                throw new TypeScriptError(`Multiple translation keys already exist at '${key}'`, node);
+            }
+            localization[part] = value;
+        } else {
+            if (typeof entry === 'string') {
+                throw new TypeScriptError(`String entry already exists at '${parts.splice(0, i + 1).join('/')}'`, node);
+            }
+            if (!entry) {
+                entry = {};
+            }
+            localization[part] = entry;
+            localization = entry;
+        }
+    });
+}
+
+function collect(n: ts.Node, fn: (node: ts.Node) => boolean): ts.Node[] {
+    const result: ts.Node[] = [];
+
+    function loop(node: ts.Node): void {
+
+        const stepResult = fn(node);
+
+        if (stepResult) {
+            result.push(node);
+        } else {
+            ts.forEachChild(node, loop);
+        }
+    }
+
+    loop(n);
+    return result;
+}
+
+function isLocalizeCall(node: ts.Node): boolean {
+    if (!ts.isCallExpression(node)) {
+        return false;
+    }
+
+    return node.expression.getText() === 'nls.localize';
+}
+
+function extractFromLocalizeCall(node: ts.Node, options?: ExtractionOptions): [string, string, ts.Node] | undefined {
+    if (!ts.isCallExpression(node)) {
+        throw new TypeScriptError('Invalid node type', node);
+    }
+    const args = node.arguments;
+
+    if (args.length < 2) {
+        throw new TypeScriptError('Localize call needs at least 2 arguments', node);
+    }
+
+    const key = extractString(args[0]);
+    const value = extractString(args[1]);
+
+    if (isExcluded(options, key)) {
+        return undefined;
+    }
+
+    return [key, value, args[1]];
+}
+
+function extractFromLocalizedCommandCall(node: ts.Node, errors?: string[], options?: ExtractionOptions): {
+    label: [string, string, ts.Node],
+    category?: [string, string, ts.Node]
+} {
+    if (!ts.isCallExpression(node)) {
+        throw new TypeScriptError('Invalid node type', node);
+    }
+    const args = node.arguments;
+
+    if (args.length < 1) {
+        throw new TypeScriptError('Command localization call needs at least one argument', node);
+    }
+
+    const commandObj = args[0];
+
+    if (!ts.isObjectLiteralExpression(commandObj)) {
+        throw new TypeScriptError('First argument of "toLocalizedCommand" needs to be an object literal', node);
+    }
+
+    const properties = commandObj.properties;
+    const propertyMap = new Map<string, string>();
+    const relevantProps = ['id', 'label', 'category'];
+    let labelNode: ts.Node = node;
+
+    for (const property of properties) {
+        if (!property.name) {
+            continue;
+        }
+        if (!ts.isPropertyAssignment(property)) {
+            throw new TypeScriptError('Only property assignments in "toLocalizedCommand" are allowed', property);
+        }
+        if (!ts.isIdentifier(property.name)) {
+            throw new TypeScriptError('Only identifiers are allowed as property names in "toLocalizedCommand"', property);
+        }
+        const name = property.name.text;
+        if (!relevantProps.includes(property.name.text)) {
+            continue;
+        }
+        if (property.name.text === 'label') {
+            labelNode = property.initializer;
+        }
+
+        try {
+            const value = extractString(property.initializer);
+            propertyMap.set(name, value);
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
+        }
+    }
+
+    let labelKey = propertyMap.get('id');
+    let categoryKey: string | undefined = undefined;
+    let categoryNode: ts.Node | undefined;
+
+    // We have an explicit label translation key
+    if (args.length > 1) {
+        try {
+            const labelOverrideKey = extractStringOrUndefined(args[1]);
+            if (labelOverrideKey) {
+                labelKey = labelOverrideKey;
+                labelNode = args[1];
+            }
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
+        }
+    }
+
+    // We have an explicit category translation key
+    if (args.length > 2) {
+        try {
+            categoryKey = extractStringOrUndefined(args[2]);
+            categoryNode = args[2];
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
+        }
+    }
+
+    if (!labelKey) {
+        throw new TypeScriptError('No label key found', node);
+    }
+
+    if (!propertyMap.get('label')) {
+        throw new TypeScriptError('No default label found', node);
+    }
+
+    let categoryLocalization: [string, string, ts.Node] | undefined = undefined;
+    const categoryLabel = propertyMap.get('category');
+    if (categoryKey && categoryLabel && categoryNode) {
+        categoryLocalization = [categoryKey, categoryLabel, categoryNode];
+    }
+
+    return {
+        label: [labelKey, propertyMap.get('label')!, labelNode],
+        category: categoryLocalization
+    };
+}
+
+function extractStringOrUndefined(node: ts.Expression): string | undefined {
+    if (node.getText() === 'undefined') {
+        return undefined;
+    }
+    return extractString(node);
+}
+
+function extractString(node: ts.Expression): string {
+    if (ts.isIdentifier(node)) {
+        const reference = followReference(node);
+        if (!reference) {
+            throw new TypeScriptError(`Could not resolve reference to '${node.text}'`, node);
+        }
+        node = reference;
+    }
+    if (ts.isTemplateLiteral(node)) {
+        throw new TypeScriptError(
+            "Template literals are not supported for localization. Please use the additional arguments of the 'nls.localize' function to format strings",
+            node
+        );
+    }
+    if (!ts.isStringLiteralLike(node)) {
+        throw new TypeScriptError(`'${node.getText()}' is not a string constant`, node);
+    }
+
+    return unescapeString(node.text);
+}
+
+function followReference(node: ts.Identifier): ts.Expression | undefined {
+    const scope = collectScope(node);
+    const next = scope.get(node.text);
+    if (next && ts.isIdentifier(next)) {
+        return followReference(next);
+    }
+    return next;
+}
+
+function collectScope(node: ts.Node, map: Map<string, ts.Expression> = new Map()): Map<string, ts.Expression> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const locals = (node as any)['locals'] as Map<string, ts.Symbol>;
+    if (locals) {
+        for (const [key, value] of locals.entries()) {
+            if (!map.has(key)) {
+                const declaration = value.valueDeclaration;
+                if (declaration && ts.isVariableDeclaration(declaration) && declaration.initializer) {
+                    map.set(key, declaration.initializer);
+                }
+            }
+        }
+    }
+    if (node.parent) {
+        collectScope(node.parent, map);
+    }
+    return map;
+}
+
+function isCommandLocalizeUtility(node: ts.Node): boolean {
+    if (!ts.isCallExpression(node)) {
+        return false;
+    }
+
+    return node.expression.getText() === 'Command.toLocalizedCommand';
+}
+
+const unescapeMap: Record<string, string> = {
+    '\'': '\'',
+    '"': '"',
+    '\\': '\\',
+    'n': '\n',
+    'r': '\r',
+    't': '\t',
+    'b': '\b',
+    'f': '\f'
+};
+
+function unescapeString(str: string): string {
+    const result: string[] = [];
+    for (let i = 0; i < str.length; i++) {
+        const ch = str.charAt(i);
+        if (ch === '\\') {
+            if (i + 1 < str.length) {
+                const replace = unescapeMap[str.charAt(i + 1)];
+                if (replace !== undefined) {
+                    result.push(replace);
+                    i++;
+                    continue;
+                }
+            }
+        }
+        result.push(ch);
+    }
+    return result.join('');
+}

dev-packages/localization-manager/src/localization-manager.spec.ts

@@ -0,0 +1,91 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as assert from 'assert';
+import { DeeplParameters, DeeplResponse } from './deepl-api';
+import { LocalizationManager, LocalizationOptions } from './localization-manager';
+
+describe('localization-manager#translateLanguage', () => {
+
+    async function mockLocalization(parameters: DeeplParameters): Promise<DeeplResponse> {
+        return {
+            translations: parameters.text.map(value => ({
+                detected_source_language: '',
+                text: `[${value}]`
+            }))
+        };
+    }
+
+    const manager = new LocalizationManager(mockLocalization);
+    const defaultOptions: LocalizationOptions = {
+        authKey: '',
+        freeApi: false,
+        sourceFile: '',
+        targetLanguages: ['EN']
+    };
+
+    it('should translate a single value', async () => {
+        const input = {
+            key: 'value'
+        };
+        const target = {};
+        await manager.translateLanguage(input, target, 'EN', defaultOptions);
+        assert.deepStrictEqual(target, {
+            key: '[value]'
+        });
+    });
+
+    it('should translate nested values', async () => {
+        const input = {
+            a: {
+                b: 'b'
+            },
+            c: 'c'
+        };
+        const target = {};
+        await manager.translateLanguage(input, target, 'EN', defaultOptions);
+        assert.deepStrictEqual(target, {
+            a: {
+                b: '[b]'
+            },
+            c: '[c]'
+        });
+    });
+
+    it('should not override existing targets', async () => {
+        const input = {
+            a: 'a'
+        };
+        const target = {
+            a: 'b'
+        };
+        await manager.translateLanguage(input, target, 'EN', defaultOptions);
+        assert.deepStrictEqual(target, {
+            a: 'b'
+        });
+    });
+
+    it('should keep placeholders intact', async () => {
+        const input = {
+            key: '{1} {0}'
+        };
+        const target = {};
+        await manager.translateLanguage(input, target, 'EN', defaultOptions);
+        assert.deepStrictEqual(target, {
+            key: '[{1} {0}]'
+        });
+    });
+});

dev-packages/localization-manager/src/localization-manager.ts

@@ -0,0 +1,168 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as chalk from 'chalk';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { Localization, sortLocalization } from './common';
+import { deepl, DeeplLanguage, DeeplParameters, defaultLanguages, isSupportedLanguage } from './deepl-api';
+
+export interface LocalizationOptions {
+    freeApi: Boolean
+    authKey: string
+    sourceFile: string
+    sourceLanguage?: string
+    targetLanguages: string[]
+}
+
+export type LocalizationFunction = (parameters: DeeplParameters) => Promise<string[]>;
+
+export class LocalizationManager {
+
+    constructor(private localizationFn = deepl) { }
+
+    async localize(options: LocalizationOptions): Promise<boolean> {
+        let source: Localization = {};
+        const cwd = process.env.INIT_CWD || process.cwd();
+        const sourceFile = path.resolve(cwd, options.sourceFile);
+        try {
+            source = await fs.readJson(sourceFile);
+        } catch {
+            console.log(chalk.red(`Could not read file "${options.sourceFile}"`));
+            process.exit(1);
+        }
+        const languages: string[] = [];
+        for (const targetLanguage of options.targetLanguages) {
+            if (!isSupportedLanguage(targetLanguage)) {
+                console.log(chalk.yellow(`Language "${targetLanguage}" is not supported for automatic localization`));
+            } else {
+                languages.push(targetLanguage);
+            }
+        }
+        if (languages.length === 0) {
+            // No supported languages were found, default to all supported languages
+            console.log('No languages were specified, defaulting to all supported languages for VS Code');
+            languages.push(...defaultLanguages);
+        }
+        const existingTranslations: Map<string, Localization> = new Map();
+        for (const targetLanguage of languages) {
+            try {
+                const targetPath = this.translationFileName(sourceFile, targetLanguage);
+                existingTranslations.set(targetLanguage, await fs.readJson(targetPath));
+            } catch {
+                existingTranslations.set(targetLanguage, {});
+            }
+        }
+        const results = await Promise.all(languages.map(language => this.translateLanguage(source, existingTranslations.get(language)!, language, options)));
+        let result = results.reduce((acc, val) => acc && val, true);
+
+        for (const targetLanguage of languages) {
+            const targetPath = this.translationFileName(sourceFile, targetLanguage);
+            try {
+                const translation = existingTranslations.get(targetLanguage)!;
+                await fs.writeJson(targetPath, sortLocalization(translation), { spaces: 2 });
+            } catch {
+                console.error(chalk.red(`Error writing translated file to '${targetPath}'`));
+                result = false;
+            }
+        }
+        return result;
+    }
+
+    protected translationFileName(original: string, language: string): string {
+        const directory = path.dirname(original);
+        const fileName = path.basename(original, '.json');
+        return path.join(directory, `${fileName}.${language.toLowerCase()}.json`);
+    }
+
+    async translateLanguage(source: Localization, target: Localization, targetLanguage: string, options: LocalizationOptions): Promise<boolean> {
+        const map = this.buildLocalizationMap(source, target);
+        if (map.text.length > 0) {
+            try {
+                const translationResponse = await this.localizationFn({
+                    auth_key: options.authKey,
+                    free_api: options.freeApi,
+                    target_lang: targetLanguage.toUpperCase() as DeeplLanguage,
+                    source_lang: options.sourceLanguage?.toUpperCase() as DeeplLanguage,
+                    text: map.text.map(e => this.addIgnoreTags(e)),
+                    tag_handling: ['xml'],
+                    ignore_tags: ['x']
+                });
+                translationResponse.translations.forEach(({ text }, i) => {
+                    map.localize(i, this.removeIgnoreTags(text));
+                });
+                console.log(chalk.green(`Successfully translated ${map.text.length} value${map.text.length > 1 ? 's' : ''} for language "${targetLanguage}"`));
+                return true;
+            } catch (e) {
+                console.log(chalk.red(`Could not translate into language "${targetLanguage}"`), e);
+                return false;
+            }
+        } else {
+            console.log(`No translation necessary for language "${targetLanguage}"`);
+            return true;
+        }
+    }
+
+    protected addIgnoreTags(text: string): string {
+        return text.replace(/(\{\d*\})/g, '<x>$1</x>');
+    }
+
+    protected removeIgnoreTags(text: string): string {
+        return text.replace(/<x>(\{\d+\})<\/x>/g, '$1');
+    }
+
+    protected buildLocalizationMap(source: Localization, target: Localization): LocalizationMap {
+        const functionMap = new Map<number, (value: string) => void>();
+        const text: string[] = [];
+        const process = (s: Localization, t: Localization) => {
+            // Delete all extra keys in the target translation first
+            for (const key of Object.keys(t)) {
+                if (!(key in s)) {
+                    delete t[key];
+                }
+            }
+            for (const [key, value] of Object.entries(s)) {
+                if (!(key in t)) {
+                    if (typeof value === 'string') {
+                        functionMap.set(text.length, translation => t[key] = translation);
+                        text.push(value);
+                    } else {
+                        const newLocalization: Localization = {};
+                        t[key] = newLocalization;
+                        process(value, newLocalization);
+                    }
+                } else if (typeof value === 'object') {
+                    if (typeof t[key] === 'string') {
+                        t[key] = {};
+                    }
+                    process(value, t[key] as Localization);
+                }
+            }
+        };
+
+        process(source, target);
+
+        return {
+            text,
+            localize: (index, value) => functionMap.get(index)!(value)
+        };
+    }
+}
+
+export interface LocalizationMap {
+    text: string[]
+    localize: (index: number, value: string) => void
+}

dev-packages/localization-manager/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../configs/base.tsconfig",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "lib"
+  },
+  "include": [
+    "src"
+  ],
+  "references": []
+}