Fix tabbed Literate CoffeeScript (#4345)

* failing test case

* add markdown parser for literate coffeescript

this should help to handle ligitimate markdown with indentation correctly

* Update generated code

* Update package.json

* Add `marked` dependency to browser version of CoffeeScript

* Update invertLiterate to use a randomly-generated token that we check for uniqueness, rather than a magic number that we hope might not occur in the code

* Fix typos

Author: GeoffreyBooth

Cakefile

@@ -140,6 +140,14 @@ task 'build:parser', 'rebuild the Jison parser (run build first)', ->
 
 task 'build:browser', 'rebuild the merged script for inclusion in the browser', ->
   code = ''
+  for {name, src} in [{name: 'marked', src: 'lib/marked.js'}]
+    code += """
+      require['#{name}'] = (function() {
+        var exports = {}, module = {exports: exports};
+        #{fs.readFileSync "node_modules/#{name}/#{src}"}
+        return module.exports;
+      })();
+    """
   for name in ['helpers', 'rewriter', 'lexer', 'parser', 'scope', 'nodes', 'sourcemap', 'coffee-script', 'browser']
     code += """
       require['./#{name}'] = (function() {

extras/coffee-script.js

@@ -44,5 +44,8 @@
     "highlight.js": "~9.7.0",
     "underscore": "~1.8.3",
     "docco": "~0.7.0"
+  },
+  "dependencies": {
+    "marked": "~0.3.6"
   }
 }

lib/coffee-script/helpers.js

@@ -2,6 +2,17 @@
 # the **Lexer**, **Rewriter**, and the **Nodes**. Merge objects, flatten
 # arrays, count characters, that sort of thing.
 
+marked = require 'marked'
+# marked.setOptions
+#   renderer: new marked.Renderer()
+#   gfm: true
+#   tables: true
+#   breaks: false
+#   pedantic: false
+#   sanitize: true
+#   smartLists: true
+#   smartypants: false
+
 # Peek at the beginning of a given string to see if it matches a sequence.
 exports.starts = (string, literal, start) ->
   literal is string.substr start, literal.length
@@ -67,19 +78,25 @@ exports.some = Array::some ? (fn) ->
   return true for e in this when fn e
   false
 
-# Simple function for inverting Literate CoffeeScript code by putting the
-# documentation in comments, producing a string of CoffeeScript code that
-# can be compiled "normally".
+# Simple function for extracting code from Literate CoffeeScript by stripping
+# out all non-code blocks, producing a string of CoffeeScript code that can
+# be compiled “normally.”
 exports.invertLiterate = (code) ->
-  maybe_code = true
-  lines = for line in code.split('\n')
-    if maybe_code and /^([ ]{4}|[ ]{0,3}\t)/.test line
-      line
-    else if maybe_code = /^\s*$/.test line
-      line
-    else
-      '# ' + line
-  lines.join '\n'
+  # Create a placeholder for tabs, that isn’t used anywhere in `code`, and then
+  # re-insert the tabs after code extraction.
+  generateRandomToken = ->
+    "#{Math.random() * Date.now()}"
+  while token is undefined or code.indexOf(token) isnt -1
+    token = generateRandomToken()
+
+  code = code.replace "\t", token
+  # Parse as markdown, discard everything except code blocks.
+  out = ""
+  for item in marked.lexer code, {}
+    out += "#{item.text}\n" if item.type is 'code'
+  # Put the tabs back in.
+  out.replace token, "\t"
+  out
 
 # Merge two jison-style location data objects together.
 # If `last` is not provided, this will simply return `first`.

package.json

@@ -55,3 +55,111 @@ Tabs work too:
 
 				test "tabbed code", ->
 					ok yes
+
+---
+
+    # keep track of whether code blocks are executed or not
+    executed = false
+
+<p>
+
+    executed = true # should not execute, this is just HTML para, not code!
+
+</p>
+
+    test "should ignore indented sections inside HTML", ->
+      eq executed, false
+
+---
+
+*   A list item with a code block:
+
+        test "basic literate CoffeeScript parsing", ->
+          ok yes
+
+---
+
+*   Lorem ipsum dolor sit amet, consectetuer adipiscing elit.
+    Aliquam hendrerit mi posuere lectus. Vestibulum enim wisi,
+    viverra nec, fringilla in, laoreet vitae, risus.
+
+*   Donec sit amet nisl. Aliquam semper ipsum sit amet velit.
+    Suspendisse id sem consectetuer libero luctus adipiscing.
+
+---
+
+1.  This is a list item with two paragraphs. Lorem ipsum dolor
+    sit amet, consectetuer adipiscing elit. Aliquam hendrerit
+    mi posuere lectus.
+
+    Vestibulum enim wisi, viverra nec, fringilla in, laoreet
+    vitae, risus. Donec sit amet nisl. Aliquam semper ipsum
+    sit amet velit.
+
+2.  Suspendisse id sem consectetuer libero luctus adipiscing.
+
+---
+
+1.  This is a list item with two paragraphs. Lorem ipsum dolor
+    sit amet, consectetuer adipiscing elit. Aliquam hendrerit
+    mi posuere lectus.
+
+    Vestibulum enim wisi, viverra nec, fringilla in, laoreet
+    vitae, risus. Donec sit amet nisl. Aliquam semper ipsum
+    sit amet velit.
+
+2.  Suspendisse id sem consectetuer libero luctus adipiscing.
+
+---
+
+*   A list item with a blockquote:
+
+    > This is a blockquote
+    > inside a list item.
+
+---
+
+This next one probably passes because a string is inoffensive in compiled js, also, can't get `marked` to parse it correctly, and not sure if empty line is permitted between title and reference
+
+This is [an example][id] reference-style link.
+[id]: http://example.com/
+
+    "Optional Title Here"
+
+---
+
+    executed = no
+
+1986. What a great season.
+          executed = yes
+
+and test...
+
+    test "should recognise indented code blocks in lists", ->
+      ok executed
+
+---
+
+    executed = no
+
+1986. What a great season.
+
+          executed = yes
+
+and test...
+
+    test "should recognise indented code blocks in lists with empty line as separator", ->
+      ok executed
+
+---
+
+    executed = no
+
+1986\. What a great season.
+           executed = yes
+
+and test...
+
+    test "should ignore indented code in escaped list like number", ->
+      eq executed, no
+