feat: Support separators in floating point literals (#1385)

Author: pulpdrew

Diff for: NOTICE

@@ -24,6 +24,7 @@ under the licensing terms detailed in LICENSE:
 * Julien Letellier <[email protected]>
 * Guido Zuidhof <[email protected]>
 * ncave <[email protected]>
+* Andrew Davis <[email protected]>
 
 Portions of this software are derived from third-party works licensed under
 the following terms:

Diff for: src/tokenizer.ts

@@ -1494,37 +1494,78 @@ export class Tokenizer extends DiagnosticEmitter {
   }
 
   readDecimalFloat(): f64 {
-    // TODO: numeric separators (parseFloat can't handle these)
     var text = this.source.text;
-    var pos = this.pos;
     var end = this.end;
-    var start = pos;
-    while (pos < end && isDecimalDigit(text.charCodeAt(pos))) {
-      ++pos;
-    }
-    if (pos < end && text.charCodeAt(pos) == CharCode.DOT) {
-      ++pos;
-      while (pos < end && isDecimalDigit(text.charCodeAt(pos))) {
-        ++pos;
-      }
+    var start = this.pos;
+    var sepCount = 0;
+
+    sepCount += this.readDecimalFloatPartial(false);
+    if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
+      ++this.pos;
+      sepCount += this.readDecimalFloatPartial();
     }
-    if (pos < end) {
-      let c = text.charCodeAt(pos);
+    if (this.pos < end) {
+      let c = text.charCodeAt(this.pos);
       if ((c | 32) == CharCode.e) {
         if (
-          ++pos < end &&
-          (c = text.charCodeAt(pos)) == CharCode.MINUS || c == CharCode.PLUS &&
-          isDecimalDigit(text.charCodeAt(pos + 1))
+          ++this.pos < end &&
+          (c = text.charCodeAt(this.pos)) == CharCode.MINUS || c == CharCode.PLUS &&
+          isDecimalDigit(text.charCodeAt(this.pos + 1))
         ) {
-          ++pos;
+          ++this.pos;
         }
-        while (pos < end && isDecimalDigit(text.charCodeAt(pos))) {
-          ++pos;
+        sepCount += this.readDecimalFloatPartial();
+      }
+    }
+    let result = text.substring(start, this.pos);
+    if (sepCount > 0) result = result.replaceAll("_", "");
+    return parseFloat(result);
+  }
+
+  /** Reads past one section of a decimal float literal. Returns the number of separators encountered. */
+  private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): u32 {
+    var text = this.source.text;
+    var pos = this.pos;
+    var start = pos;
+    var end = this.end;
+    var sepEnd = start;
+    var sepCount = 0;
+
+    while (pos < end) {
+      let c = text.charCodeAt(pos);
+
+      if (c == CharCode._) {
+        if (sepEnd == pos) {
+          this.error(
+            sepEnd == start
+              ? DiagnosticCode.Numeric_separators_are_not_allowed_here
+              : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
+            this.range(pos)
+          );
+        } else if (!allowLeadingZeroSep && pos - 1 == start && text.charCodeAt(pos - 1) == CharCode._0) {
+          this.error(
+            DiagnosticCode.Numeric_separators_are_not_allowed_here,
+            this.range(pos)
+          );
         }
+        sepEnd = pos + 1;
+        ++sepCount;
+      } else if (!isDecimalDigit(c)) {
+        break;
       }
+
+      ++pos;
+    }
+
+    if (pos != start && sepEnd == pos) {
+      this.error(
+        DiagnosticCode.Numeric_separators_are_not_allowed_here,
+        this.range(sepEnd - 1)
+      );
     }
+
     this.pos = pos;
-    return parseFloat(text.substring(start, pos));
+    return sepCount;
   }
 
   readHexFloat(): f64 {

Diff for: src/tsconfig.json

@@ -4,7 +4,7 @@
     "outDir": "../out",
     "allowJs": false,
     "sourceMap": true,
-    "target": "ES2016",
+    "target": "esnext",
     "strict": true
   },
   "include": [

Diff for: std/portable.json

@@ -7,6 +7,7 @@
     "downlevelIteration": true,
     "preserveConstEnums": true,
     "typeRoots": [ "types" ],
-    "types": [ "portable" ]
+    "types": [ "portable" ],
+    "lib": ["esnext", "esnext.string"]
   }
 }

Diff for: tests/parser/numeric-separators.ts

@@ -2,6 +2,12 @@
 0b01_01_01;
 0o12_12_12;
 0x23_23_23;
+1_000_000.1234_1234;
+1_0e1_0;
+1_000_000e-1_0;
+0.0_0;
+1_0e0_0;
+1_0e0_1;
 
 // error cases that should still continue parsing:
 
@@ -16,3 +22,19 @@
 
 0x23_23_23_; // 6188
 0x23__23_23; // 6189
+
+1000_.1234; // 6188
+1000._1234; // 6188
+1000.1234_; // 6188
+
+10__00.1234; // 6189
+1000.12__34; // 6189
+
+1_e2; // 6188
+1e_2; // 6188
+1e2_; // 6188
+1e-1__0; // 6189
+
+0_0.0; // 6188
+0_0.0_0; // 6188
+0_0e0_0; // 6188

Diff for: tests/parser/numeric-separators.ts.fixture.ts

@@ -2,6 +2,12 @@
 21;
 41610;
 2302755;
+1000000.12341234;
+100000000000;
+0.0001;
+0;
+10;
+100;
 111111;
 111111;
 21;
@@ -10,11 +16,35 @@
 41610;
 2302755;
 2302755;
-// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(8,9+0)
-// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(9,4+0)
-// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,11+0)
-// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,6+0)
-// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,11+0)
-// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,6+0)
+1000.1234;
+1000.1234;
+1000.1234;
+1000.1234;
+1000.1234;
+100;
+100;
+100;
+1e-10;
+0;
+0;
+0;
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,9+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,4+0)
 // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(17,11+0)
 // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(18,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(20,11+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(21,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(23,11+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(24,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(26,5+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(27,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(28,10+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(30,4+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(31,9+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(33,2+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(34,3+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(35,4+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(36,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(38,2+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(39,2+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(40,2+0)