Add Handlebars.parseWithoutProcessing (#1584)

When authoring tooling that parses Handlebars files and emits Handlebars
files, you often want to preserve the **exact** formatting of the input.
The changes in this commit add a new method to the `Handlebars`
namespace: `parseWithoutProcessing`. Unlike, `Handlebars.parse` (which
will mutate the parsed AST to apply whitespace control) this method will
parse the template and return it directly (**without** processing
:wink:).

For example, parsing the following template:

```hbs
 {{#foo}}
   {{~bar~}} {{baz~}}
 {{/foo}}
```

Using `Handlebars.parse`, the AST returned would have truncated the
following whitespace:

* The whitespace prior to the `{{#foo}}`
* The newline following `{{#foo}}`
* The leading whitespace before `{{~bar~}}`
* The whitespace between `{{~bar~}}` and `{{baz~}}`
* The newline after `{{baz~}}`
* The whitespace prior to the `{{/foo}}`

When `Handlebars.parse` is used from  `Handlebars.precompile` or
`Handlebars.compile`, this whitespace stripping is **very** important
(these behaviors are intentional, and generally lead to better rendered
output).

When the same template is parsed with
`Handlebars.parseWithoutProcessing` none of those modifications to the
AST are made. This enables "codemod tooling" (e.g. `prettier` and
`ember-template-recast`) to preserve the **exact** initial formatting.
Prior to these changes, those tools would have to _manually_ reconstruct
the whitespace that is lost prior to emitting source.

Author: rwjblue

docs/compiler-api.md

@@ -16,6 +16,34 @@ var ast = Handlebars.parse(myTemplate);
 Handlebars.precompile(ast);
 ```
 
+### Parsing
+
+There are two primary APIs that are used to parse an existing template into the AST:
+
+#### parseWithoutProcessing
+
+`Handlebars.parseWithoutProcessing` is the primary mechanism to turn a raw template string into the Handlebars AST described in this document. No processing is done on the resulting AST which makes this ideal for codemod (for source to source transformation) tooling.
+
+Example:
+
+```js
+let ast = Handlebars.parseWithoutProcessing(myTemplate);
+```
+
+#### parse
+
+`Handlebars.parse` will parse the template with `parseWithoutProcessing` (see above) then it will update the AST to strip extraneous whitespace. The whitespace stripping functionality handles two distinct situations:
+
+* Removes whitespace around dynamic statements that are on a line by themselves (aka "stand alone")
+* Applies "whitespace control" characters (i.e. `~`) by truncating the `ContentStatement` `value` property appropriately (e.g. `\n\n{{~foo}}` would have a `ContentStatement` with a `value` of `''`)
+
+`Handlebars.parse` is used internally by `Handlebars.precompile` and `Handlebars.compile`.
+
+Example:
+
+```js
+let ast = Handlebars.parse(myTemplate);
+```
 
 ### Basic
 

lib/handlebars.js

@@ -2,7 +2,7 @@ import runtime from './handlebars.runtime';
 
 // Compiler imports
 import AST from './handlebars/compiler/ast';
-import { parser as Parser, parse } from './handlebars/compiler/base';
+import { parser as Parser, parse, parseWithoutProcessing } from './handlebars/compiler/base';
 import { Compiler, compile, precompile } from './handlebars/compiler/compiler';
 import JavaScriptCompiler from './handlebars/compiler/javascript-compiler';
 import Visitor from './handlebars/compiler/visitor';
@@ -25,6 +25,7 @@ function create() {
   hb.JavaScriptCompiler = JavaScriptCompiler;
   hb.Parser = Parser;
   hb.parse = parse;
+  hb.parseWithoutProcessing = parseWithoutProcessing;
 
   return hb;
 }

lib/handlebars/compiler/base.js

@@ -8,7 +8,7 @@ export { parser };
 let yy = {};
 extend(yy, Helpers);
 
-export function parse(input, options) {
+export function parseWithoutProcessing(input, options) {
   // Just return if an already-compiled AST was passed in.
   if (input.type === 'Program') { return input; }
 
@@ -19,6 +19,14 @@ export function parse(input, options) {
     return new yy.SourceLocation(options && options.srcName, locInfo);
   };
 
+  let ast = parser.parse(input);
+
+  return ast;
+}
+
+export function parse(input, options) {
+  let ast = parseWithoutProcessing(input, options);
   let strip = new WhitespaceControl(options);
-  return strip.accept(parser.parse(input));
+
+  return strip.accept(ast);
 }

spec/ast.js

@@ -123,6 +123,40 @@ describe('ast', function() {
      });
   });
 
+  describe('whitespace control', function() {
+    describe('parse', function() {
+      it('mustache', function() {
+        let ast = Handlebars.parse('  {{~comment~}} ');
+
+        equals(ast.body[0].value, '');
+        equals(ast.body[2].value, '');
+      });
+
+      it('block statements', function() {
+        var ast = Handlebars.parse(' {{# comment~}} \nfoo\n {{~/comment}}');
+
+        equals(ast.body[0].value, '');
+        equals(ast.body[1].program.body[0].value, 'foo');
+      });
+    });
+
+    describe('parseWithoutProcessing', function() {
+      it('mustache', function() {
+        let ast = Handlebars.parseWithoutProcessing('  {{~comment~}} ');
+
+        equals(ast.body[0].value, '  ');
+        equals(ast.body[2].value, ' ');
+      });
+
+      it('block statements', function() {
+        var ast = Handlebars.parseWithoutProcessing(' {{# comment~}} \nfoo\n {{~/comment}}');
+
+        equals(ast.body[0].value, ' ');
+        equals(ast.body[1].program.body[0].value, ' \nfoo\n ');
+      });
+    });
+  });
+
   describe('standalone flags', function() {
     describe('mustache', function() {
       it('does not mark mustaches as standalone', function() {
@@ -131,6 +165,54 @@ describe('ast', function() {
         equals(!!ast.body[2].value, true);
       });
     });
+    describe('blocks - parseWithoutProcessing', function() {
+      it('block mustaches', function() {
+        var ast = Handlebars.parseWithoutProcessing(' {{# comment}} \nfoo\n {{else}} \n  bar \n  {{/comment}} '),
+          block = ast.body[1];
+
+        equals(ast.body[0].value, ' ');
+
+        equals(block.program.body[0].value, ' \nfoo\n ');
+        equals(block.inverse.body[0].value, ' \n  bar \n  ');
+
+        equals(ast.body[2].value, ' ');
+      });
+      it('initial block mustaches', function() {
+        var ast = Handlebars.parseWithoutProcessing('{{# comment}} \nfoo\n {{/comment}}'),
+          block = ast.body[0];
+
+        equals(block.program.body[0].value, ' \nfoo\n ');
+      });
+      it('mustaches with children', function() {
+        var ast = Handlebars.parseWithoutProcessing('{{# comment}} \n{{foo}}\n {{/comment}}'),
+          block = ast.body[0];
+
+        equals(block.program.body[0].value, ' \n');
+        equals(block.program.body[1].path.original, 'foo');
+        equals(block.program.body[2].value, '\n ');
+      });
+      it('nested block mustaches', function() {
+        var ast = Handlebars.parseWithoutProcessing('{{#foo}} \n{{# comment}} \nfoo\n {{else}} \n  bar \n  {{/comment}} \n{{/foo}}'),
+          body = ast.body[0].program.body,
+          block = body[1];
+
+        equals(body[0].value, ' \n');
+
+        equals(block.program.body[0].value, ' \nfoo\n ');
+        equals(block.inverse.body[0].value, ' \n  bar \n  ');
+      });
+      it('column 0 block mustaches', function() {
+        var ast = Handlebars.parseWithoutProcessing('test\n{{# comment}} \nfoo\n {{else}} \n  bar \n  {{/comment}} '),
+            block = ast.body[1];
+
+        equals(ast.body[0].omit, undefined);
+
+        equals(block.program.body[0].value, ' \nfoo\n ');
+        equals(block.inverse.body[0].value, ' \n  bar \n  ');
+
+        equals(ast.body[2].value, ' ');
+      });
+    });
     describe('blocks', function() {
       it('marks block mustaches as standalone', function() {
         var ast = Handlebars.parse(' {{# comment}} \nfoo\n {{else}} \n  bar \n  {{/comment}} '),
@@ -204,6 +286,18 @@ describe('ast', function() {
         equals(ast.body[2].value, '');
       });
     });
+    describe('partials - parseWithoutProcessing', function() {
+      it('simple partial', function() {
+        var ast = Handlebars.parseWithoutProcessing('{{> partial }} ');
+        equals(ast.body[1].value, ' ');
+      });
+      it('indented partial', function() {
+        var ast = Handlebars.parseWithoutProcessing('  {{> partial }} ');
+        equals(ast.body[0].value, '  ');
+        equals(ast.body[1].indent, '');
+        equals(ast.body[2].value, ' ');
+      });
+    });
     describe('partials', function() {
       it('marks partial as standalone', function() {
         var ast = Handlebars.parse('{{> partial }} ');
@@ -223,6 +317,17 @@ describe('ast', function() {
         equals(ast.body[1].omit, undefined);
       });
     });
+    describe('comments - parseWithoutProcessing', function() {
+      it('simple comment', function() {
+        var ast = Handlebars.parseWithoutProcessing('{{! comment }} ');
+        equals(ast.body[1].value, ' ');
+      });
+      it('indented comment', function() {
+        var ast = Handlebars.parseWithoutProcessing('  {{! comment }} ');
+        equals(ast.body[0].value, '  ');
+        equals(ast.body[2].value, ' ');
+      });
+    });
     describe('comments', function() {
       it('marks comment as standalone', function() {
         var ast = Handlebars.parse('{{! comment }} ');

types/index.d.ts

@@ -47,8 +47,8 @@ declare namespace Handlebars {
   }
 
   export interface ParseOptions {
-      srcName?: string,
-      ignoreStandalone?: boolean
+      srcName?: string;
+      ignoreStandalone?: boolean;
   }
 
   export function registerHelper(name: string, fn: HelperDelegate): void;
@@ -69,6 +69,7 @@ declare namespace Handlebars {
   export function Exception(message: string): void;
   export function log(level: number, obj: any): void;
   export function parse(input: string, options?: ParseOptions): hbs.AST.Program;
+  export function parseWithoutProcessing(input: string, options?: ParseOptions): hbs.AST.Program;
   export function compile<T = any>(input: any, options?: CompileOptions): HandlebarsTemplateDelegate<T>;
   export function precompile(input: any, options?: PrecompileOptions): TemplateSpecification;
   export function template<T = any>(precompilation: TemplateSpecification): HandlebarsTemplateDelegate<T>;

types/test.ts

@@ -192,4 +192,12 @@ switch(allthings.type) {
     break;
   default:
     break;
-}
+}
+
+function testParseWithoutProcessing() {
+  const parsedTemplate: hbs.AST.Program = Handlebars.parseWithoutProcessing('<p>Hello, my name is {{name}}.</p>', {
+    srcName: "/foo/bar/baz.hbs",
+  });
+
+  const parsedTemplateWithoutOptions: hbs.AST.Program = Handlebars.parseWithoutProcessing('<p>Hello, my name is {{name}}.</p>');
+}