"Fossies" - the Fresh Open Source Software Archive

Member "vscode-1.49.1/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts" (16 Sep 2020, 21555 Bytes) of package /linux/misc/vscode-1.49.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) TypeScript source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "abstractTextMateService.ts": 1.48.2_vs_1.49.0.

    1 /*---------------------------------------------------------------------------------------------
    2  *  Copyright (c) Microsoft Corporation. All rights reserved.
    3  *  Licensed under the MIT License. See License.txt in the project root for license information.
    4  *--------------------------------------------------------------------------------------------*/
    5 
    6 import * as nls from 'vs/nls';
    7 import * as dom from 'vs/base/browser/dom';
    8 import { Color } from 'vs/base/common/color';
    9 import { onUnexpectedError } from 'vs/base/common/errors';
   10 import { Emitter, Event } from 'vs/base/common/event';
   11 import * as resources from 'vs/base/common/resources';
   12 import * as types from 'vs/base/common/types';
   13 import { equals as equalArray } from 'vs/base/common/arrays';
   14 import { URI } from 'vs/base/common/uri';
   15 import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
   16 import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationRegistry, StandardTokenType, LanguageIdentifier } from 'vs/editor/common/modes';
   17 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
   18 import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
   19 import { IModeService } from 'vs/editor/common/services/modeService';
   20 import { ILogService } from 'vs/platform/log/common/log';
   21 import { INotificationService, Severity } from 'vs/platform/notification/common/notification';
   22 import { IStorageService, StorageScope } from 'vs/platform/storage/common/storage';
   23 import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry';
   24 import { ITMSyntaxExtensionPoint, grammarsExtPoint } from 'vs/workbench/services/textMate/common/TMGrammars';
   25 import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService';
   26 import { ITextMateThemingRule, IWorkbenchThemeService, IWorkbenchColorTheme } from 'vs/workbench/services/themes/common/workbenchThemeService';
   27 import type { IGrammar, StackElement, IOnigLib, IRawTheme } from 'vscode-textmate';
   28 import { Disposable, IDisposable, dispose } from 'vs/base/common/lifecycle';
   29 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
   30 import { IValidGrammarDefinition, IValidEmbeddedLanguagesMap, IValidTokenTypeMap } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
   31 import { TMGrammarFactory } from 'vs/workbench/services/textMate/common/TMGrammarFactory';
   32 import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader';
   33 import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress';
   34 
   35 export abstract class AbstractTextMateService extends Disposable implements ITextMateService {
   36     public _serviceBrand: undefined;
   37 
   38     private readonly _onDidEncounterLanguage: Emitter<LanguageId> = this._register(new Emitter<LanguageId>());
   39     public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
   40 
   41     private readonly _styleElement: HTMLStyleElement;
   42     private readonly _createdModes: string[];
   43     private readonly _encounteredLanguages: boolean[];
   44 
   45     private _debugMode: boolean;
   46     private _debugModePrintFunc: (str: string) => void;
   47 
   48     private _grammarDefinitions: IValidGrammarDefinition[] | null;
   49     private _grammarFactory: TMGrammarFactory | null;
   50     private _tokenizersRegistrations: IDisposable[];
   51     protected _currentTheme: IRawTheme | null;
   52     protected _currentTokenColorMap: string[] | null;
   53 
   54     constructor(
   55         @IModeService private readonly _modeService: IModeService,
   56         @IWorkbenchThemeService private readonly _themeService: IWorkbenchThemeService,
   57         @IExtensionResourceLoaderService protected readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
   58         @INotificationService private readonly _notificationService: INotificationService,
   59         @ILogService private readonly _logService: ILogService,
   60         @IConfigurationService private readonly _configurationService: IConfigurationService,
   61         @IStorageService private readonly _storageService: IStorageService,
   62         @IProgressService private readonly _progressService: IProgressService
   63     ) {
   64         super();
   65         this._styleElement = dom.createStyleSheet();
   66         this._styleElement.className = 'vscode-tokens-styles';
   67         this._createdModes = [];
   68         this._encounteredLanguages = [];
   69 
   70         this._debugMode = false;
   71         this._debugModePrintFunc = () => { };
   72 
   73         this._grammarDefinitions = null;
   74         this._grammarFactory = null;
   75         this._tokenizersRegistrations = [];
   76 
   77         this._currentTheme = null;
   78         this._currentTokenColorMap = null;
   79 
   80         grammarsExtPoint.setHandler((extensions) => {
   81             this._grammarDefinitions = null;
   82             if (this._grammarFactory) {
   83                 this._grammarFactory.dispose();
   84                 this._grammarFactory = null;
   85                 this._onDidDisposeGrammarFactory();
   86             }
   87             this._tokenizersRegistrations = dispose(this._tokenizersRegistrations);
   88 
   89             this._grammarDefinitions = [];
   90             for (const extension of extensions) {
   91                 const grammars = extension.value;
   92                 for (const grammar of grammars) {
   93                     if (!this._validateGrammarExtensionPoint(extension.description.extensionLocation, grammar, extension.collector)) {
   94                         continue;
   95                     }
   96                     const grammarLocation = resources.joinPath(extension.description.extensionLocation, grammar.path);
   97 
   98                     const embeddedLanguages: IValidEmbeddedLanguagesMap = Object.create(null);
   99                     if (grammar.embeddedLanguages) {
  100                         let scopes = Object.keys(grammar.embeddedLanguages);
  101                         for (let i = 0, len = scopes.length; i < len; i++) {
  102                             let scope = scopes[i];
  103                             let language = grammar.embeddedLanguages[scope];
  104                             if (typeof language !== 'string') {
  105                                 // never hurts to be too careful
  106                                 continue;
  107                             }
  108                             let languageIdentifier = this._modeService.getLanguageIdentifier(language);
  109                             if (languageIdentifier) {
  110                                 embeddedLanguages[scope] = languageIdentifier.id;
  111                             }
  112                         }
  113                     }
  114 
  115                     const tokenTypes: IValidTokenTypeMap = Object.create(null);
  116                     if (grammar.tokenTypes) {
  117                         const scopes = Object.keys(grammar.tokenTypes);
  118                         for (const scope of scopes) {
  119                             const tokenType = grammar.tokenTypes[scope];
  120                             switch (tokenType) {
  121                                 case 'string':
  122                                     tokenTypes[scope] = StandardTokenType.String;
  123                                     break;
  124                                 case 'other':
  125                                     tokenTypes[scope] = StandardTokenType.Other;
  126                                     break;
  127                                 case 'comment':
  128                                     tokenTypes[scope] = StandardTokenType.Comment;
  129                                     break;
  130                             }
  131                         }
  132                     }
  133 
  134                     let languageIdentifier: LanguageIdentifier | null = null;
  135                     if (grammar.language) {
  136                         languageIdentifier = this._modeService.getLanguageIdentifier(grammar.language);
  137                     }
  138 
  139                     this._grammarDefinitions.push({
  140                         location: grammarLocation,
  141                         language: languageIdentifier ? languageIdentifier.id : undefined,
  142                         scopeName: grammar.scopeName,
  143                         embeddedLanguages: embeddedLanguages,
  144                         tokenTypes: tokenTypes,
  145                         injectTo: grammar.injectTo,
  146                     });
  147                 }
  148             }
  149 
  150             for (const createMode of this._createdModes) {
  151                 this._registerDefinitionIfAvailable(createMode);
  152             }
  153         });
  154 
  155         this._register(this._themeService.onDidColorThemeChange(() => {
  156             if (this._grammarFactory) {
  157                 this._updateTheme(this._grammarFactory, this._themeService.getColorTheme(), false);
  158             }
  159         }));
  160 
  161         // Generate some color map until the grammar registry is loaded
  162         let colorTheme = this._themeService.getColorTheme();
  163         let defaultForeground: Color = Color.transparent;
  164         let defaultBackground: Color = Color.transparent;
  165         for (let i = 0, len = colorTheme.tokenColors.length; i < len; i++) {
  166             let rule = colorTheme.tokenColors[i];
  167             if (!rule.scope && rule.settings) {
  168                 if (rule.settings.foreground) {
  169                     defaultForeground = Color.fromHex(rule.settings.foreground);
  170                 }
  171                 if (rule.settings.background) {
  172                     defaultBackground = Color.fromHex(rule.settings.background);
  173                 }
  174             }
  175         }
  176         TokenizationRegistry.setColorMap([null!, defaultForeground, defaultBackground]);
  177 
  178         this._modeService.onDidCreateMode((mode) => {
  179             let modeId = mode.getId();
  180             this._createdModes.push(modeId);
  181             this._registerDefinitionIfAvailable(modeId);
  182         });
  183     }
  184 
  185     public startDebugMode(printFn: (str: string) => void, onStop: () => void): void {
  186         if (this._debugMode) {
  187             this._notificationService.error(nls.localize('alreadyDebugging', "Already Logging."));
  188             return;
  189         }
  190 
  191         this._debugModePrintFunc = printFn;
  192         this._debugMode = true;
  193 
  194         if (this._debugMode) {
  195             this._progressService.withProgress(
  196                 {
  197                     location: ProgressLocation.Notification,
  198                     buttons: [nls.localize('stop', "Stop")]
  199                 },
  200                 (progress) => {
  201                     progress.report({
  202                         message: nls.localize('progress1', "Preparing to log TM Grammar parsing. Press Stop when finished.")
  203                     });
  204 
  205                     return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
  206                         vscodeOniguruma.setDefaultDebugCall(true);
  207                         progress.report({
  208                             message: nls.localize('progress2', "Now logging TM Grammar parsing. Press Stop when finished.")
  209                         });
  210                         return new Promise<void>((resolve, reject) => { });
  211                     });
  212                 },
  213                 (choice) => {
  214                     this._getVSCodeOniguruma().then((vscodeOniguruma) => {
  215                         this._debugModePrintFunc = () => { };
  216                         this._debugMode = false;
  217                         vscodeOniguruma.setDefaultDebugCall(false);
  218                         onStop();
  219                     });
  220                 }
  221             );
  222         }
  223     }
  224 
  225     private _canCreateGrammarFactory(): boolean {
  226         // Check if extension point is ready
  227         return (this._grammarDefinitions ? true : false);
  228     }
  229 
  230     private async _getOrCreateGrammarFactory(): Promise<TMGrammarFactory> {
  231         if (this._grammarFactory) {
  232             return this._grammarFactory;
  233         }
  234 
  235         const [vscodeTextmate, vscodeOniguruma] = await Promise.all([import('vscode-textmate'), this._getVSCodeOniguruma()]);
  236         const onigLib: Promise<IOnigLib> = Promise.resolve({
  237             createOnigScanner: (sources: string[]) => vscodeOniguruma.createOnigScanner(sources),
  238             createOnigString: (str: string) => vscodeOniguruma.createOnigString(str)
  239         });
  240 
  241         // Avoid duplicate instantiations
  242         if (this._grammarFactory) {
  243             return this._grammarFactory;
  244         }
  245 
  246         this._grammarFactory = new TMGrammarFactory({
  247             logTrace: (msg: string) => this._logService.trace(msg),
  248             logError: (msg: string, err: any) => this._logService.error(msg, err),
  249             readFile: (resource: URI) => this._extensionResourceLoaderService.readExtensionResource(resource)
  250         }, this._grammarDefinitions || [], vscodeTextmate, onigLib);
  251         this._onDidCreateGrammarFactory(this._grammarDefinitions || []);
  252 
  253         this._updateTheme(this._grammarFactory, this._themeService.getColorTheme(), true);
  254 
  255         return this._grammarFactory;
  256     }
  257 
  258     private _registerDefinitionIfAvailable(modeId: string): void {
  259         const languageIdentifier = this._modeService.getLanguageIdentifier(modeId);
  260         if (!languageIdentifier) {
  261             return;
  262         }
  263         if (!this._canCreateGrammarFactory()) {
  264             return;
  265         }
  266         const languageId = languageIdentifier.id;
  267 
  268         // Here we must register the promise ASAP (without yielding!)
  269         this._tokenizersRegistrations.push(TokenizationRegistry.registerPromise(modeId, (async () => {
  270             try {
  271                 const grammarFactory = await this._getOrCreateGrammarFactory();
  272                 if (!grammarFactory.has(languageId)) {
  273                     return null;
  274                 }
  275                 const r = await grammarFactory.createGrammar(languageId);
  276                 if (!r.grammar) {
  277                     return null;
  278                 }
  279                 const tokenization = new TMTokenization(r.grammar, r.initialState, r.containsEmbeddedLanguages);
  280                 tokenization.onDidEncounterLanguage((languageId) => {
  281                     if (!this._encounteredLanguages[languageId]) {
  282                         this._encounteredLanguages[languageId] = true;
  283                         this._onDidEncounterLanguage.fire(languageId);
  284                     }
  285                 });
  286                 return new TMTokenizationSupport(r.languageId, tokenization, this._notificationService, this._configurationService, this._storageService);
  287             } catch (err) {
  288                 onUnexpectedError(err);
  289                 return null;
  290             }
  291         })()));
  292     }
  293 
  294     private static _toColorMap(colorMap: string[]): Color[] {
  295         let result: Color[] = [null!];
  296         for (let i = 1, len = colorMap.length; i < len; i++) {
  297             result[i] = Color.fromHex(colorMap[i]);
  298         }
  299         return result;
  300     }
  301 
  302     private _updateTheme(grammarFactory: TMGrammarFactory, colorTheme: IWorkbenchColorTheme, forceUpdate: boolean): void {
  303         if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && AbstractTextMateService.equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors) && equalArray(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
  304             return;
  305         }
  306         this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors };
  307         this._currentTokenColorMap = colorTheme.tokenColorMap;
  308         this._doUpdateTheme(grammarFactory, this._currentTheme, this._currentTokenColorMap);
  309     }
  310 
  311     protected _doUpdateTheme(grammarFactory: TMGrammarFactory, theme: IRawTheme, tokenColorMap: string[]): void {
  312         grammarFactory.setTheme(theme, tokenColorMap);
  313         let colorMap = AbstractTextMateService._toColorMap(tokenColorMap);
  314         let cssRules = generateTokensCSSForColorMap(colorMap);
  315         this._styleElement.innerHTML = cssRules;
  316         TokenizationRegistry.setColorMap(colorMap);
  317     }
  318 
  319     private static equalsTokenRules(a: ITextMateThemingRule[] | null, b: ITextMateThemingRule[] | null): boolean {
  320         if (!b || !a || b.length !== a.length) {
  321             return false;
  322         }
  323         for (let i = b.length - 1; i >= 0; i--) {
  324             let r1 = b[i];
  325             let r2 = a[i];
  326             if (r1.scope !== r2.scope) {
  327                 return false;
  328             }
  329             let s1 = r1.settings;
  330             let s2 = r2.settings;
  331             if (s1 && s2) {
  332                 if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
  333                     return false;
  334                 }
  335             } else if (!s1 || !s2) {
  336                 return false;
  337             }
  338         }
  339         return true;
  340     }
  341 
  342     private _validateGrammarExtensionPoint(extensionLocation: URI, syntax: ITMSyntaxExtensionPoint, collector: ExtensionMessageCollector): boolean {
  343         if (syntax.language && ((typeof syntax.language !== 'string') || !this._modeService.isRegisteredMode(syntax.language))) {
  344             collector.error(nls.localize('invalid.language', "Unknown language in `contributes.{0}.language`. Provided value: {1}", grammarsExtPoint.name, String(syntax.language)));
  345             return false;
  346         }
  347         if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
  348             collector.error(nls.localize('invalid.scopeName', "Expected string in `contributes.{0}.scopeName`. Provided value: {1}", grammarsExtPoint.name, String(syntax.scopeName)));
  349             return false;
  350         }
  351         if (!syntax.path || (typeof syntax.path !== 'string')) {
  352             collector.error(nls.localize('invalid.path.0', "Expected string in `contributes.{0}.path`. Provided value: {1}", grammarsExtPoint.name, String(syntax.path)));
  353             return false;
  354         }
  355         if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) {
  356             collector.error(nls.localize('invalid.injectTo', "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.injectTo)));
  357             return false;
  358         }
  359         if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) {
  360             collector.error(nls.localize('invalid.embeddedLanguages', "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.embeddedLanguages)));
  361             return false;
  362         }
  363 
  364         if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) {
  365             collector.error(nls.localize('invalid.tokenTypes', "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.tokenTypes)));
  366             return false;
  367         }
  368 
  369         const grammarLocation = resources.joinPath(extensionLocation, syntax.path);
  370         if (!resources.isEqualOrParent(grammarLocation, extensionLocation)) {
  371             collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, grammarLocation.path, extensionLocation.path));
  372         }
  373         return true;
  374     }
  375 
  376     public async createGrammar(modeId: string): Promise<IGrammar | null> {
  377         const languageId = this._modeService.getLanguageIdentifier(modeId);
  378         if (!languageId) {
  379             return null;
  380         }
  381         const grammarFactory = await this._getOrCreateGrammarFactory();
  382         if (!grammarFactory.has(languageId.id)) {
  383             return null;
  384         }
  385         const { grammar } = await grammarFactory.createGrammar(languageId.id);
  386         return grammar;
  387     }
  388 
  389     protected _onDidCreateGrammarFactory(grammarDefinitions: IValidGrammarDefinition[]): void {
  390     }
  391 
  392     protected _onDidDisposeGrammarFactory(): void {
  393     }
  394 
  395     private _vscodeOniguruma: Promise<typeof import('vscode-oniguruma')> | null = null;
  396     private _getVSCodeOniguruma(): Promise<typeof import('vscode-oniguruma')> {
  397         if (!this._vscodeOniguruma) {
  398             this._vscodeOniguruma = this._doGetVSCodeOniguruma();
  399         }
  400         return this._vscodeOniguruma;
  401     }
  402 
  403     private async _doGetVSCodeOniguruma(): Promise<typeof import('vscode-oniguruma')> {
  404         const [vscodeOniguruma, wasm] = await Promise.all([import('vscode-oniguruma'), this._loadVSCodeOnigurumWASM()]);
  405         const options = {
  406             data: wasm,
  407             print: (str: string) => {
  408                 this._debugModePrintFunc(str);
  409             }
  410         };
  411         await vscodeOniguruma.loadWASM(options);
  412         return vscodeOniguruma;
  413     }
  414 
  415     protected abstract _loadVSCodeOnigurumWASM(): Promise<Response | ArrayBuffer>;
  416 }
  417 
  418 const donotAskUpdateKey = 'editor.maxTokenizationLineLength.donotask';
  419 
  420 class TMTokenizationSupport implements ITokenizationSupport {
  421     private readonly _languageId: LanguageId;
  422     private readonly _actual: TMTokenization;
  423     private _tokenizationWarningAlreadyShown: boolean;
  424     private _maxTokenizationLineLength: number;
  425 
  426     constructor(
  427         languageId: LanguageId,
  428         actual: TMTokenization,
  429         @INotificationService private readonly _notificationService: INotificationService,
  430         @IConfigurationService private readonly _configurationService: IConfigurationService,
  431         @IStorageService private readonly _storageService: IStorageService
  432     ) {
  433         this._languageId = languageId;
  434         this._actual = actual;
  435         this._tokenizationWarningAlreadyShown = !!(this._storageService.getBoolean(donotAskUpdateKey, StorageScope.GLOBAL));
  436         this._maxTokenizationLineLength = this._configurationService.getValue<number>('editor.maxTokenizationLineLength');
  437         this._configurationService.onDidChangeConfiguration(e => {
  438             if (e.affectsConfiguration('editor.maxTokenizationLineLength')) {
  439                 this._maxTokenizationLineLength = this._configurationService.getValue<number>('editor.maxTokenizationLineLength');
  440             }
  441         });
  442     }
  443 
  444     getInitialState(): IState {
  445         return this._actual.getInitialState();
  446     }
  447 
  448     tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult {
  449         throw new Error('Not supported!');
  450     }
  451 
  452     tokenize2(line: string, state: StackElement, offsetDelta: number): TokenizationResult2 {
  453         if (offsetDelta !== 0) {
  454             throw new Error('Unexpected: offsetDelta should be 0.');
  455         }
  456 
  457         // Do not attempt to tokenize if a line is too long
  458         if (line.length >= this._maxTokenizationLineLength) {
  459             if (!this._tokenizationWarningAlreadyShown) {
  460                 this._tokenizationWarningAlreadyShown = true;
  461                 this._notificationService.prompt(
  462                     Severity.Warning,
  463                     nls.localize('too many characters', "Tokenization is skipped for long lines for performance reasons. The length of a long line can be configured via `editor.maxTokenizationLineLength`."),
  464                     [{
  465                         label: nls.localize('neverAgain', "Don't Show Again"),
  466                         isSecondary: true,
  467                         run: () => this._storageService.store(donotAskUpdateKey, true, StorageScope.GLOBAL)
  468                     }]
  469                 );
  470             }
  471             console.log(`Line (${line.substr(0, 15)}...): longer than ${this._maxTokenizationLineLength} characters, tokenization skipped.`);
  472             return nullTokenize2(this._languageId, line, state, offsetDelta);
  473         }
  474 
  475         return this._actual.tokenize2(line, state);
  476     }
  477 }
  478 
  479 class TMTokenization extends Disposable {
  480 
  481     private readonly _grammar: IGrammar;
  482     private readonly _containsEmbeddedLanguages: boolean;
  483     private readonly _seenLanguages: boolean[];
  484     private readonly _initialState: StackElement;
  485 
  486     private readonly _onDidEncounterLanguage: Emitter<LanguageId> = this._register(new Emitter<LanguageId>());
  487     public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
  488 
  489     constructor(grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean) {
  490         super();
  491         this._grammar = grammar;
  492         this._initialState = initialState;
  493         this._containsEmbeddedLanguages = containsEmbeddedLanguages;
  494         this._seenLanguages = [];
  495     }
  496 
  497     public getInitialState(): IState {
  498         return this._initialState;
  499     }
  500 
  501     public tokenize2(line: string, state: StackElement): TokenizationResult2 {
  502         let textMateResult = this._grammar.tokenizeLine2(line, state);
  503 
  504         if (this._containsEmbeddedLanguages) {
  505             let seenLanguages = this._seenLanguages;
  506             let tokens = textMateResult.tokens;
  507 
  508             // Must check if any of the embedded languages was hit
  509             for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
  510                 let metadata = tokens[(i << 1) + 1];
  511                 let languageId = TokenMetadata.getLanguageId(metadata);
  512 
  513                 if (!seenLanguages[languageId]) {
  514                     seenLanguages[languageId] = true;
  515                     this._onDidEncounterLanguage.fire(languageId);
  516                 }
  517             }
  518         }
  519 
  520         let endState: StackElement;
  521         // try to save an object if possible
  522         if (state.equals(textMateResult.ruleStack)) {
  523             endState = state;
  524         } else {
  525             endState = textMateResult.ruleStack;
  526 
  527         }
  528 
  529         return new TokenizationResult2(textMateResult.tokens, endState);
  530     }
  531 }