From 7f380564b5ada7097176c04c82a227944c91d346 Mon Sep 17 00:00:00 2001 From: dcode Date: Fri, 17 Apr 2020 19:39:01 +0200 Subject: [PATCH 1/2] Support alternative file extensions --- cli/asc.d.ts | 2 + cli/asc.js | 110 ++++++++++++++++++------------ cli/asc.json | 4 ++ cli/util/find.d.ts | 2 - cli/util/find.js | 3 - package.json | 3 +- src/ast.ts | 23 +++++-- src/parser.ts | 2 +- src/program.ts | 2 +- src/util/text.ts | 7 ++ tests/extension/assembly/index.as | 2 + tests/extension/assembly/other.as | 3 + tests/extension/package.json | 6 ++ 13 files changed, 112 insertions(+), 57 deletions(-) create mode 100644 tests/extension/assembly/index.as create mode 100644 tests/extension/assembly/other.as create mode 100644 tests/extension/package.json diff --git a/cli/asc.d.ts b/cli/asc.d.ts index 49efa374d4..e866e649df 100644 --- a/cli/asc.d.ts +++ b/cli/asc.d.ts @@ -137,6 +137,8 @@ interface CompilerOptions { printrtti?: boolean; /** Disables terminal colors. */ noColors?: boolean; + /** Specifies an alternative file extension. */ + extension?: string; } /** Compiler API options. */ diff --git a/cli/asc.js b/cli/asc.js index 8a38d2d3ba..513c643457 100644 --- a/cli/asc.js +++ b/cli/asc.js @@ -42,6 +42,21 @@ const EOL = process.platform === "win32" ? "\r\n" : "\n"; const SEP = process.platform === "win32" ? "\\" : "/"; const binaryen = global.Binaryen || (global.Binaryen = require("binaryen")); +// Sets up an extension with its definition counterpart and relevant regexes. +function setupExtension(extension) { + if (!/\.[0-9a-zA-Z]{1,14}$/.test(extension)) throw Error("invalid extension: " + extension); + return { + ext: extension, + ext_d: ".d" + extension, + re: new RegExp("\\" + extension + "$"), + re_d: new RegExp("\\.d\\" + extension + "$"), + re_except_d: new RegExp("^(?!.*\\.d\\" + extension + "$).*\\" + extension + "$"), + re_index: new RegExp("(?:^|[\\\\\\/])index\\" + extension + "$") + }; +} + +const defaultExtension = setupExtension(".ts"); + // Proxy Binaryen's ready event Object.defineProperty(exports, "ready", { get: function() { return binaryen.ready; } @@ -99,8 +114,8 @@ exports.defaultShrinkLevel = 1; exports.libraryFiles = exports.isBundle ? BUNDLE_LIBRARY : (() => { // set up if not a bundle const libDir = path.join(__dirname, "..", "std", "assembly"); const bundled = {}; - find.files(libDir, find.TS_EXCEPT_DTS) - .forEach(file => bundled[file.replace(/\.ts$/, "")] = fs.readFileSync(path.join(libDir, file), "utf8" )); + find.files(libDir, defaultExtension.re_except_d) + .forEach(file => bundled[file.replace(defaultExtension.re, "")] = fs.readFileSync(path.join(libDir, file), "utf8" )); return bundled; })(); @@ -108,14 +123,14 @@ exports.libraryFiles = exports.isBundle ? BUNDLE_LIBRARY : (() => { // set up if exports.definitionFiles = exports.isBundle ? BUNDLE_DEFINITIONS : (() => { // set up if not a bundle const stdDir = path.join(__dirname, "..", "std"); return { - "assembly": fs.readFileSync(path.join(stdDir, "assembly", "index.d.ts"), "utf8"), - "portable": fs.readFileSync(path.join(stdDir, "portable", "index.d.ts"), "utf8") + "assembly": fs.readFileSync(path.join(stdDir, "assembly", "index" + defaultExtension.ext_d), "utf8"), + "portable": fs.readFileSync(path.join(stdDir, "portable", "index" + defaultExtension.ext_d), "utf8") }; })(); /** Convenience function that parses and compiles source strings directly. */ exports.compileString = (sources, options) => { - if (typeof sources === "string") sources = { "input.ts": sources }; + if (typeof sources === "string") sources = { ["input" + defaultExtension.ext]: sources }; const output = Object.create({ stdout: createMemoryStream(), stderr: createMemoryStream() @@ -159,6 +174,7 @@ exports.main = function main(argv, options, callback) { const writeFile = options.writeFile || writeFileNode; const listFiles = options.listFiles || listFilesNode; const stats = options.stats || createStats(); + let extension = defaultExtension; // Output must be specified if not present in the environment if (!stdout) throw Error("'options.stdout' must be specified"); @@ -203,6 +219,11 @@ exports.main = function main(argv, options, callback) { return callback(null); } + // Use another extension if requested + if (typeof args.extension === "string" && /\.[0-9a-zA-Z]{1,14}$/.test(args.extension)) { + extension = setupExtension(args.extension); + } + // Print the help message if requested or no source files are provided if (args.help || !argv.length) { var out = args.help ? stdout : stderr; @@ -212,9 +233,9 @@ exports.main = function main(argv, options, callback) { " " + color.cyan("asc") + " [entryFile ...] [options]", "", color.white("EXAMPLES"), - " " + color.cyan("asc") + " hello.ts", - " " + color.cyan("asc") + " hello.ts -b hello.wasm -t hello.wat", - " " + color.cyan("asc") + " hello1.ts hello2.ts -b -O > hello.wasm", + " " + color.cyan("asc") + " hello" + extension.ext, + " " + color.cyan("asc") + " hello" + extension.ext + " -b hello.wasm -t hello.wat", + " " + color.cyan("asc") + " hello1" + extension.ext + " hello2" + extension.ext + " -b -O > hello.wasm", "", color.white("OPTIONS"), ].concat( @@ -309,7 +330,7 @@ exports.main = function main(argv, options, callback) { let transformArgs = args.transform; for (let i = 0, k = transformArgs.length; i < k; ++i) { let filename = transformArgs[i].trim(); - if (!tsNodeRegistered && filename.endsWith('.ts')) { + if (!tsNodeRegistered && filename.endsWith(".ts")) { // ts-node requires .ts specifically require("ts-node").register({ transpileOnly: true, skipProject: true, compilerOptions: { target: "ES2016" } }); tsNodeRegistered = true; } @@ -353,7 +374,7 @@ exports.main = function main(argv, options, callback) { if (libPath.indexOf("/") >= 0) return; // in sub-directory: imported on demand stats.parseCount++; stats.parseTime += measure(() => { - assemblyscript.parse(program, exports.libraryFiles[libPath], exports.libraryPrefix + libPath + ".ts", false); + assemblyscript.parse(program, exports.libraryFiles[libPath], exports.libraryPrefix + libPath + extension.ext, false); }); }); const customLibDirs = []; @@ -364,7 +385,7 @@ exports.main = function main(argv, options, callback) { for (let i = 0, k = customLibDirs.length; i < k; ++i) { // custom let libDir = customLibDirs[i]; let libFiles; - if (libDir.endsWith(".ts")) { + if (libDir.endsWith(extension.ext)) { libFiles = [ path.basename(libDir) ]; libDir = path.dirname(libDir); } else { @@ -375,7 +396,7 @@ exports.main = function main(argv, options, callback) { let libText = readFile(libPath, libDir); if (libText === null) return callback(Error("Library file '" + libPath + "' not found.")); stats.parseCount++; - exports.libraryFiles[libPath.replace(/\.ts$/, "")] = libText; + exports.libraryFiles[libPath.replace(extension.re, "")] = libText; stats.parseTime += measure(() => { assemblyscript.parse(program, libText, exports.libraryPrefix + libPath, false); }); @@ -396,13 +417,13 @@ exports.main = function main(argv, options, callback) { const libraryPrefix = exports.libraryPrefix; const libraryFiles = exports.libraryFiles; - // Try file.ts, file/index.ts, file.d.ts + // Try file.ext, file/index.ext, file.d.ext if (!internalPath.startsWith(libraryPrefix)) { - if ((sourceText = readFile(sourcePath = internalPath + ".ts", baseDir)) == null) { - if ((sourceText = readFile(sourcePath = internalPath + "/index.ts", baseDir)) == null) { - // portable d.ts: uses the .js file next to it in JS or becomes an import in Wasm - sourcePath = internalPath + ".ts"; - sourceText = readFile(internalPath + ".d.ts", baseDir); + if ((sourceText = readFile(sourcePath = internalPath + extension.ext, baseDir)) == null) { + if ((sourceText = readFile(sourcePath = internalPath + "/index" + extension.ext, baseDir)) == null) { + // portable d.ext: uses the .js file next to it in JS or becomes an import in Wasm + sourcePath = internalPath + extension.ext; + sourceText = readFile(internalPath + extension.ext_d, baseDir); } } @@ -412,18 +433,18 @@ exports.main = function main(argv, options, callback) { const indexName = plainName + "/index"; if (libraryFiles.hasOwnProperty(plainName)) { sourceText = libraryFiles[plainName]; - sourcePath = libraryPrefix + plainName + ".ts"; + sourcePath = libraryPrefix + plainName + extension.ext; } else if (libraryFiles.hasOwnProperty(indexName)) { sourceText = libraryFiles[indexName]; - sourcePath = libraryPrefix + indexName + ".ts"; + sourcePath = libraryPrefix + indexName + extension.ext; } else { // custom lib dirs for (const libDir of customLibDirs) { - if ((sourceText = readFile(plainName + ".ts", libDir)) != null) { - sourcePath = libraryPrefix + plainName + ".ts"; + if ((sourceText = readFile(plainName + extension.ext, libDir)) != null) { + sourcePath = libraryPrefix + plainName + extension.ext; break; } else { - if ((sourceText = readFile(indexName + ".ts", libDir)) != null) { - sourcePath = libraryPrefix + indexName + ".ts"; + if ((sourceText = readFile(indexName + extension.ext, libDir)) != null) { + sourcePath = libraryPrefix + indexName + extension.ext; break; } } @@ -453,7 +474,7 @@ exports.main = function main(argv, options, callback) { try { let json = JSON.parse(jsonText); if (typeof json.ascMain === "string") { - mainPath = json.ascMain.replace(/[\/\\]index\.ts$/, ""); + mainPath = json.ascMain.replace(extension.re_index, ""); packageMains.set(packageName, mainPath); } } catch (e) { } @@ -461,17 +482,17 @@ exports.main = function main(argv, options, callback) { } const mainDir = path.join(currentPath, packageName, mainPath); const plainName = filePath; - if ((sourceText = readFile(path.join(mainDir, plainName + ".ts"), baseDir)) != null) { - sourcePath = libraryPrefix + packageName + "/" + plainName + ".ts"; - packageBases.set(sourcePath.replace(/\.ts$/, ""), path.join(currentPath, packageName)); - if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, plainName + ".ts") + EOL); + if ((sourceText = readFile(path.join(mainDir, plainName + extension.ext), baseDir)) != null) { + sourcePath = libraryPrefix + packageName + "/" + plainName + extension.ext; + packageBases.set(sourcePath.replace(extension.re, ""), path.join(currentPath, packageName)); + if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, plainName + extension.ext) + EOL); break; } else if (!isPackageRoot) { const indexName = filePath + "/index"; - if ((sourceText = readFile(path.join(mainDir, indexName + ".ts"), baseDir)) !== null) { - sourcePath = libraryPrefix + packageName + "/" + indexName + ".ts"; - packageBases.set(sourcePath.replace(/\.ts$/, ""), path.join(currentPath, packageName)); - if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, indexName + ".ts") + EOL); + if ((sourceText = readFile(path.join(mainDir, indexName + extension.ext), baseDir)) !== null) { + sourcePath = libraryPrefix + packageName + "/" + indexName + extension.ext; + packageBases.set(sourcePath.replace(extension.re, ""), path.join(currentPath, packageName)); + if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, indexName + extension.ext) + EOL); break; } } @@ -509,14 +530,14 @@ exports.main = function main(argv, options, callback) { let runtimeText = exports.libraryFiles[runtimePath]; if (runtimeText == null) { runtimePath = runtimeName; - runtimeText = readFile(runtimePath + ".ts", baseDir); + runtimeText = readFile(runtimePath + extension.ext, baseDir); if (runtimeText == null) return callback(Error("Runtime '" + runtimeName + "' not found.")); } else { runtimePath = "~lib/" + runtimePath; } stats.parseCount++; stats.parseTime += measure(() => { - assemblyscript.parse(program, runtimeText, runtimePath, true); + assemblyscript.parse(program, runtimeText, runtimePath + extension.ext, true); }); } @@ -524,18 +545,19 @@ exports.main = function main(argv, options, callback) { for (let i = 0, k = argv.length; i < k; ++i) { const filename = argv[i]; - let sourcePath = String(filename).replace(/\\/g, "/").replace(/(\.ts|\/)$/, ""); + let sourcePath = String(filename).replace(/\\/g, "/").replace(extension.re, "").replace(/[\\\/]$/, ""); + // Setting the path to relative path sourcePath = path.isAbsolute(sourcePath) ? path.relative(baseDir, sourcePath) : sourcePath; - // Try entryPath.ts, then entryPath/index.ts - let sourceText = readFile(sourcePath + ".ts", baseDir); + // Try entryPath.ext, then entryPath/index.ext + let sourceText = readFile(sourcePath + extension.ext, baseDir); if (sourceText == null) { - sourceText = readFile(sourcePath + "/index.ts", baseDir); - if (sourceText == null) return callback(Error("Entry file '" + sourcePath + ".ts' not found.")); - sourcePath += "/index.ts"; + sourceText = readFile(sourcePath + "/index" + extension.ext, baseDir); + if (sourceText == null) return callback(Error("Entry file '" + sourcePath + extension.ext + "' not found.")); + sourcePath += "/index" + extension.ext; } else { - sourcePath += ".ts"; + sourcePath += extension.ext; } stats.parseCount++; @@ -920,7 +942,7 @@ exports.main = function main(argv, options, callback) { map.sourceRoot = "./" + basename; let contents = []; map.sources.forEach((name, index) => { - let text = assemblyscript.getSource(program, name.replace(/\.ts$/, "")); + let text = assemblyscript.getSource(program, name.replace(extension.re, "")); if (text == null) return callback(Error("Source of file '" + name + "' not found.")); contents[index] = text; }); @@ -1061,7 +1083,7 @@ exports.main = function main(argv, options, callback) { var files; try { stats.readTime += measure(() => { - files = fs.readdirSync(path.join(baseDir, dirname)).filter(file => /^(?!.*\.d\.ts$).*\.ts$/.test(file)); + files = fs.readdirSync(path.join(baseDir, dirname)).filter(file => extension.re_except_d.test(file)) }); return files; } catch (e) { diff --git a/cli/asc.json b/cli/asc.json index e737ca117c..2b6b3f80cf 100644 --- a/cli/asc.json +++ b/cli/asc.json @@ -252,6 +252,10 @@ "type": "s", "default": "." }, + "extension": { + "description": "Specifies an alternative file extension to use.", + "type": "s" + }, "noUnsafe": { "description": [ "Disallows the use of unsafe features in user code.", diff --git a/cli/util/find.d.ts b/cli/util/find.d.ts index 2778f9aa29..cc2aeaed37 100644 --- a/cli/util/find.d.ts +++ b/cli/util/find.d.ts @@ -4,5 +4,3 @@ */ export function files(dirname: string, filter?: ((name: string) => bool) | RegExp): string[]; -export const TS: RegExp; -export const TS_EXCEPT_DTS: RegExp; diff --git a/cli/util/find.js b/cli/util/find.js index 0541679de5..57230109a7 100644 --- a/cli/util/find.js +++ b/cli/util/find.js @@ -19,6 +19,3 @@ function findFiles(dirname, filter) { } exports.files = findFiles; - -exports.TS = /\.ts$/; -exports.TS_EXCEPT_DTS = /(?:(?!\.d).{2}|^.{0,1})\.ts$/; diff --git a/package.json b/package.json index c4d41e8239..ba5cc41459 100644 --- a/package.json +++ b/package.json @@ -55,10 +55,11 @@ "check": "npm run check:config && npm run check:compiler && tsc --noEmit --target ESNEXT --module commonjs --experimentalDecorators tests/require/index", "check:config": "tsc --noEmit -p src --diagnostics --listFiles", "check:compiler": "tslint -c tslint.json --project src --formatters-dir lib/lint/formatters --format as", - "test": "npm run test:parser && npm run test:compiler && npm run test:packages", + "test": "npm run test:parser && npm run test:compiler && npm run test:packages && npm run test:extension", "test:parser": "node tests/parser", "test:compiler": "node tests/compiler", "test:packages": "cd tests/packages && npm run test", + "test:extension": "cd tests/extension && npm run test", "make": "npm run clean && npm test && npm run build && npm test", "all": "npm run check && npm run make", "docs": "typedoc --tsconfig tsconfig-docs.json --mode modules --name \"AssemblyScript Compiler API\" --out ./docs/api --ignoreCompilerErrors --excludeNotExported --excludePrivate --excludeExternals --exclude **/std/** --includeDeclarations --readme src/README.md", diff --git a/src/ast.ts b/src/ast.ts index 4a28106439..79a7ecf918 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -29,7 +29,8 @@ import { import { normalizePath, resolvePath, - CharCode + CharCode, + isTrivialAlphanum } from "./util"; /** Indicates the kind of a node. */ @@ -716,7 +717,7 @@ export abstract class Node { } else { // absolute if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath; } - node.internalPath = mangleInternalPath(normalizedPath); + node.internalPath = normalizedPath; } else { node.internalPath = null; } @@ -804,7 +805,7 @@ export abstract class Node { } else { // absolute in library if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath; } - node.internalPath = mangleInternalPath(normalizedPath); + node.internalPath = normalizedPath; return node; } @@ -825,7 +826,7 @@ export abstract class Node { } else { if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath; } - node.internalPath = mangleInternalPath(normalizedPath); + node.internalPath = normalizedPath; return node; } @@ -2088,7 +2089,19 @@ export function findDecorator(kind: DecoratorKind, decorators: DecoratorNode[] | /** Mangles an external to an internal path. */ export function mangleInternalPath(path: string): string { - if (path.endsWith(".ts")) path = path.substring(0, path.length - 3); + var pos = path.lastIndexOf("."); + var len = path.length; + if (pos >= 0 && len - pos >= 2) { // at least one char plus dot + let cur = pos; + while (++cur < len) { + if (!isTrivialAlphanum(path.charCodeAt(cur))) { + assert(false); // not a valid external path + return path; + } + } + return path.substring(0, pos); + } + assert(false); // not an external path return path; } diff --git a/src/parser.ts b/src/parser.ts index bfbc857053..182435df0d 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -125,7 +125,7 @@ export class Parser extends DiagnosticEmitter { /** Whether this is an entry file. */ isEntry: bool ): void { - // the frontend gives us paths with .ts endings + // the frontend gives us paths with file extensions var normalizedPath = normalizePath(path); var internalPath = mangleInternalPath(normalizedPath); // check if already processed diff --git a/src/program.ts b/src/program.ts index 9c07ab7703..bcd777140b 100644 --- a/src/program.ts +++ b/src/program.ts @@ -532,7 +532,7 @@ export class Program extends DiagnosticEmitter { ) { super(diagnostics); this.options = options; - var nativeSource = new Source(LIBRARY_SUBST, "[native code]", SourceKind.LIBRARY_ENTRY); + var nativeSource = new Source(LIBRARY_SUBST + ".wasm", "[native code]", SourceKind.LIBRARY_ENTRY); this.nativeSource = nativeSource; var nativeFile = new File(this, nativeSource); this.nativeFile = nativeFile; diff --git a/src/util/text.ts b/src/util/text.ts index caa983b625..574bf9c12f 100644 --- a/src/util/text.ts +++ b/src/util/text.ts @@ -185,6 +185,13 @@ export function isOctalDigit(c: i32): bool { return c >= CharCode._0 && c <= CharCode._7; } +/** Tests if the specified character code is trivially alphanumeric. */ +export function isTrivialAlphanum(code: i32): bool { + return code >= CharCode.a && code <= CharCode.z + || code >= CharCode.A && code <= CharCode.Z + || code >= CharCode._0 && code <= CharCode._9; +} + /** Tests if the specified character code is a valid start of an identifier. */ export function isIdentifierStart(c: i32): bool { const c0 = c | 32; // unify uppercases and lowercases a|A - z|Z diff --git a/tests/extension/assembly/index.as b/tests/extension/assembly/index.as new file mode 100644 index 0000000000..923334a12e --- /dev/null +++ b/tests/extension/assembly/index.as @@ -0,0 +1,2 @@ +import { add } from "./other"; +add(1, 2); diff --git a/tests/extension/assembly/other.as b/tests/extension/assembly/other.as new file mode 100644 index 0000000000..caed355404 --- /dev/null +++ b/tests/extension/assembly/other.as @@ -0,0 +1,3 @@ +export function add(a: i32, b: i32): i32 { + return a + b; +} diff --git a/tests/extension/package.json b/tests/extension/package.json new file mode 100644 index 0000000000..070b239379 --- /dev/null +++ b/tests/extension/package.json @@ -0,0 +1,6 @@ +{ + "scripts": { + "test": "npm run asbuild", + "asbuild": "node ../../bin/asc assembly/index.as --extension .as --runtime none --noEmit" + } +} \ No newline at end of file From 0351e51e549dce6cd7a748147e83b6a9ae81b6d2 Mon Sep 17 00:00:00 2001 From: dcode Date: Wed, 22 Apr 2020 04:56:17 +0200 Subject: [PATCH 2/2] accept extensions without leading dot --- cli/asc.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cli/asc.js b/cli/asc.js index 513c643457..ef95c0fdf1 100644 --- a/cli/asc.js +++ b/cli/asc.js @@ -44,7 +44,7 @@ const binaryen = global.Binaryen || (global.Binaryen = require("binaryen")); // Sets up an extension with its definition counterpart and relevant regexes. function setupExtension(extension) { - if (!/\.[0-9a-zA-Z]{1,14}$/.test(extension)) throw Error("invalid extension: " + extension); + if (!extension.startsWith(".")) extension = "." + extension; return { ext: extension, ext_d: ".d" + extension, @@ -220,8 +220,12 @@ exports.main = function main(argv, options, callback) { } // Use another extension if requested - if (typeof args.extension === "string" && /\.[0-9a-zA-Z]{1,14}$/.test(args.extension)) { - extension = setupExtension(args.extension); + if (typeof args.extension === "string") { + if (/^\.?[0-9a-zA-Z]{1,14}$/.test(args.extension)) { + extension = setupExtension(args.extension); + } else { + return callback(Error("Invalid extension: " + args.extension)); + } } // Print the help message if requested or no source files are provided