feat(compiler): support skipping leading trivia in template source-ma…

…ps (#30095) Leading trivia, such as whitespace or comments, is confusing for developers looking at source-mapped templates, since they expect the source-map segment to start after the trivia. This commit adds skipping trivial characters to the lexer; and then implements that in the template parser. PR Close #30095
angular · Apr 25, 2019 · 304a12f · 304a12f
1 parent acaf1aa
commit 304a12f
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 7 deletions.
diff --git a/packages/compiler/src/ml_parser/lexer.ts b/packages/compiler/src/ml_parser/lexer.ts
@@ -95,6 +95,12 @@ export interface TokenizeOptions {
    * but the new line should increment the current line for source mapping.
    */
   escapedString?: boolean;
+  /**
+   * An array of characters that should be considered as leading trivia.
+   * Leading trivia are characters that are not important to the developer, and so should not be
+   * included in source-map segments.  A common example is whitespace.
+   */
+  leadingTriviaChars?: string[];
 }
 
 export function tokenize(
@@ -123,11 +129,11 @@ class _Tokenizer {
   private _cursor: CharacterCursor;
   private _tokenizeIcu: boolean;
   private _interpolationConfig: InterpolationConfig;
+  private _leadingTriviaCodePoints: number[]|undefined;
   private _currentTokenStart: CharacterCursor|null = null;
   private _currentTokenType: TokenType|null = null;
   private _expansionCaseStack: TokenType[] = [];
   private _inInterpolation: boolean = false;
-
   tokens: Token[] = [];
   errors: TokenError[] = [];
 
@@ -141,6 +147,8 @@ class _Tokenizer {
       options: TokenizeOptions) {
     this._tokenizeIcu = options.tokenizeExpansionForms || false;
     this._interpolationConfig = options.interpolationConfig || DEFAULT_INTERPOLATION_CONFIG;
+    this._leadingTriviaCodePoints =
+        options.leadingTriviaChars && options.leadingTriviaChars.map(c => c.codePointAt(0) || 0);
     const range =
         options.range || {endPos: _file.content.length, startPos: 0, startLine: 0, startCol: 0};
     this._cursor = options.escapedString ? new EscapedCharacterCursor(_file, range) :
@@ -236,8 +244,9 @@ class _Tokenizer {
           'Programming error - attempted to end a token which has no token type', null,
           this._cursor.getSpan(this._currentTokenStart));
     }
-    const token =
-        new Token(this._currentTokenType, parts, this._cursor.getSpan(this._currentTokenStart));
+    const token = new Token(
+        this._currentTokenType, parts,
+        this._cursor.getSpan(this._currentTokenStart, this._leadingTriviaCodePoints));
     this.tokens.push(token);
     this._currentTokenStart = null;
     this._currentTokenType = null;
@@ -772,7 +781,7 @@ interface CharacterCursor {
   /** Advance the cursor by one parsed character. */
   advance(): void;
   /** Get a span from the marked start point to the current point. */
-  getSpan(start?: this): ParseSourceSpan;
+  getSpan(start?: this, leadingTriviaCodePoints?: number[]): ParseSourceSpan;
   /** Get the parsed characters from the marked start point to the current point. */
   getChars(start: this): string;
   /** The number of characters left before the end of the cursor. */
@@ -831,8 +840,14 @@ class PlainCharacterCursor implements CharacterCursor {
 
   init(): void { this.updatePeek(this.state); }
 
-  getSpan(start?: this): ParseSourceSpan {
+  getSpan(start?: this, leadingTriviaCodePoints?: number[]): ParseSourceSpan {
     start = start || this;
+    if (leadingTriviaCodePoints) {
+      start = start.clone() as this;
+      while (this.diff(start) > 0 && leadingTriviaCodePoints.indexOf(start.peek()) !== -1) {
+        start.advance();
+      }
+    }
     return new ParseSourceSpan(
         new ParseLocation(start.file, start.state.offset, start.state.line, start.state.column),
         new ParseLocation(this.file, this.state.offset, this.state.line, this.state.column));

diff --git a/packages/compiler/src/render3/view/template.ts b/packages/compiler/src/render3/view/template.ts
@@ -53,6 +53,8 @@ const NG_PROJECT_AS_ATTR_NAME = 'ngProjectAs';
 const GLOBAL_TARGET_RESOLVERS = new Map<string, o.ExternalReference>(
     [['window', R3.resolveWindow], ['document', R3.resolveDocument], ['body', R3.resolveBody]]);
 
+const LEADING_TRIVIA_CHARS = [' ', '\n', '\r', '\t'];
+
 //  if (rf & flags) { .. }
 export function renderFlagCheckIfStmt(
     flags: core.RenderFlags, statements: o.Statement[]): o.IfStmt {
@@ -1733,8 +1735,9 @@ export function parseTemplate(
   const {interpolationConfig, preserveWhitespaces} = options;
   const bindingParser = makeBindingParser(interpolationConfig);
   const htmlParser = new HtmlParser();
-  const parseResult =
-      htmlParser.parse(template, templateUrl, {...options, tokenizeExpansionForms: true});
+  const parseResult = htmlParser.parse(
+      template, templateUrl,
+      {...options, tokenizeExpansionForms: true, leadingTriviaChars: LEADING_TRIVIA_CHARS});
 
   if (parseResult.errors && parseResult.errors.length > 0) {
     return {errors: parseResult.errors, nodes: [], styleUrls: [], styles: []};

diff --git a/packages/compiler/test/ml_parser/lexer_spec.ts b/packages/compiler/test/ml_parser/lexer_spec.ts
@@ -52,6 +52,18 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u
           [lex.TokenType.EOF, '2:5'],
         ]);
       });
+
+      it('should skip over leading trivia for source-span start', () => {
+        expect(tokenizeAndHumanizeLineColumn(
+                   '<t>\n \t a</t>', {leadingTriviaChars: ['\n', ' ', '\t']}))
+            .toEqual([
+              [lex.TokenType.TAG_OPEN_START, '0:0'],
+              [lex.TokenType.TAG_OPEN_END, '0:2'],
+              [lex.TokenType.TEXT, '1:3'],
+              [lex.TokenType.TAG_CLOSE, '1:4'],
+              [lex.TokenType.EOF, '1:8'],
+            ]);
+      });
     });
 
     describe('content ranges', () => {