Replaceable jl_parse as single parser entry point

`jl_parse` calls a C function pointer which can be replaced with
jl_set_parser(). This should make it possible to replace the flisp
parser with julia code. Currently `jl_parse` takes Strings rather than C
buffers to make it more natural to implement in Julia code.

TODO: Could revisit this to decide whether it's desirable to require a
`String` or whether we should have non-copying `unsafe_wrap` and
`IOBuffer` for parsing.

Author: c42f

base/client.jl

@@ -153,8 +153,9 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool)
 end
 
 function _parse_input_line_core(s::String, filename::String)
-    ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
-               s, sizeof(s), filename, sizeof(filename))
+    JL_PARSE_TOPLEVEL = 3
+    ex,_ = ccall(:jl_parse, Any, (Any, Any, Cint, Cint),
+                 s, filename, 1, JL_PARSE_TOPLEVEL)
     if ex isa Expr && ex.head === :toplevel
         if isempty(ex.args)
             return nothing

base/meta.jl

@@ -172,18 +172,20 @@ julia> Meta.parse("x = 3, y = 5", 5)
 function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
                depwarn::Bool=true)
     # pos is one based byte offset.
-    # returns (expr, end_pos). expr is () in case of parse error.
-    bstr = String(str)
+    filename = "none"
+    JL_PARSE_ATOM = 1
+    JL_PARSE_STATEMENT = 2
+    rule = greedy ? JL_PARSE_STATEMENT : JL_PARSE_ATOM
     # For now, assume all parser warnings are depwarns
+    # TODO: remove parser-depwarn; parser no longer emits warnings.
     ex, pos = with_logger(depwarn ? current_logger() : NullLogger()) do
-        ccall(:jl_parse_string, Any,
-              (Ptr{UInt8}, Csize_t, Int32, Int32),
-              bstr, sizeof(bstr), pos-1, greedy ? 1 : 0)
+        ccall(:jl_parse, Any, (String, String, Cint, Cint),
+              str, filename, pos, rule)
     end
     if raise && isa(ex,Expr) && ex.head === :error
         throw(ParseError(ex.args[1]))
     end
-    return ex, pos+1 # C is zero-based, Julia is 1-based
+    return ex, pos
 end
 
 """

src/Makefile

@@ -42,7 +42,7 @@ endif
 RUNTIME_SRCS := \
 	jltypes gf typemap ast builtins module interpreter symbol \
 	dlload sys init task array dump staticdata toplevel jl_uv datatype \
-	simplevector runtime_intrinsics precompile \
+	simplevector runtime_intrinsics precompile frontend \
 	threading partr stackwalk gc gc-debug gc-pages gc-stacks method \
 	jlapi signal-handling safepoint timing subtype \
 	crc32c APInt-C processor

src/ast.c

@@ -27,6 +27,8 @@ extern "C" {
 #pragma warning(disable:4335)
 #endif
 
+// TODO: Move non-flisp stuff into frontend.c (symbol init + macro expansion etc)
+
 // head symbols for each expression type
 jl_sym_t *call_sym;    jl_sym_t *invoke_sym;
 jl_sym_t *empty_sym;   jl_sym_t *top_sym;
@@ -777,87 +779,61 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v)
     return julia_to_scm_noalloc2(fl_ctx, v);
 }
 
-typedef enum {
-    JL_PARSE_ATOM       = 1,
-    JL_PARSE_STATEMENTS = 2,
-    JL_PARSE_TOPLEVEL   = 3,
-} jl_parse_rule_t;
-
-// Parse string `content` starting at byte offset `start_pos` attributing it to
-// `filename`. Return an svec of (parse_result, final_pos)
-JL_DLLEXPORT jl_value_t *jl_fl_parse(const char *content, size_t content_len,
-                                     const char *filename, size_t filename_len,
-                                     int start_pos, jl_parse_rule_t rule)
+// Parse string `content` starting at 1-based index `start_pos` attributing the
+// content to `filename`. Return an svec of (parse_result, final_pos)
+JL_DLLEXPORT jl_value_t *jl_fl_parse(jl_value_t *text, jl_value_t *filename,
+                                     int start_pos, int rule)
 {
     JL_TIMING(PARSING);
-    if (start_pos < 0 || start_pos > content_len) {
-        jl_array_t *buf = jl_pchar_to_array(content, content_len);
-        JL_GC_PUSH1(&buf);
+    if (!jl_is_string(text) || !jl_is_string(filename)) {
+        jl_errorf("File content and name must be Strings");
+    }
+    if (start_pos < 1 || start_pos > (int)jl_string_len(text) + 1) {
         // jl_bounds_error roots the arguments.
-        jl_bounds_error((jl_value_t*)buf, jl_box_long(start_pos));
+        jl_bounds_error(jl_box_long(start_pos), text);
     }
-    else if (start_pos != 0 && rule == JL_PARSE_TOPLEVEL) {
+    else if (start_pos != 1 && rule == JL_PARSE_TOPLEVEL) {
         jl_error("Partial parsing not support by top level grammar rule");
     }
-    jl_value_t *expr=NULL, *pos1=NULL;
-    JL_GC_PUSH2(&expr, &pos1);
 
-    // Call into flisp
     jl_ast_context_t *ctx = jl_ast_ctx_enter();
     fl_context_t *fl_ctx = &ctx->fl;
-    value_t fl_content = cvalue_static_cstrn(fl_ctx, content, content_len);
-    value_t fl_filename = cvalue_static_cstrn(fl_ctx, filename, filename_len);
+    value_t fl_text = cvalue_static_cstrn(fl_ctx, jl_string_data(text),
+                                          jl_string_len(text));
+    value_t fl_filename = cvalue_static_cstrn(fl_ctx, jl_string_data(filename),
+                                              jl_string_len(filename));
+    value_t fl_expr;
+    size_t pos1 = 0;
     if (rule == JL_PARSE_TOPLEVEL) {
         value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")),
-                              fl_content, fl_filename);
-        expr = e == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, e, NULL);
-        pos1 = e == fl_ctx->FL_EOF ? jl_box_long(content_len) : jl_box_long(0);
+                              fl_text, fl_filename);
+        fl_expr = e;
+        pos1 = e == fl_ctx->FL_EOF ? jl_string_len(text) : 0;
     }
-    else if (rule == JL_PARSE_STATEMENTS || rule == JL_PARSE_ATOM) {
-        value_t greedy = rule == JL_PARSE_STATEMENTS ?
-                         fl_ctx->T : fl_ctx->F;
+    else if (rule == JL_PARSE_STATEMENT || rule == JL_PARSE_ATOM) {
+        value_t greedy = rule == JL_PARSE_STATEMENT ? fl_ctx->T : fl_ctx->F;
+        value_t offset = fixnum(start_pos-1);
         value_t p = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-parse-one")),
-                              fl_content, fl_filename, fixnum(start_pos), greedy);
-        value_t e = car_(p);
-        expr = e == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, e, NULL);
-        pos1 = jl_box_long(tosize(fl_ctx, cdr_(p), "parse"));
+                              fl_text, fl_filename, offset, greedy);
+        fl_expr = car_(p);
+        pos1 = tosize(fl_ctx, cdr_(p), "parse");
     }
-    jl_ast_ctx_leave(ctx);
-
-    if (expr == NULL) {
-        jl_errorf("Unknown grammar rule %d", (int)rule);
+    else {
+        jl_ast_ctx_leave(ctx);
+        jl_errorf("Unknown parse rule %d", (int)rule);
     }
 
-    jl_value_t *result = (jl_value_t*)jl_svec2(expr, pos1);
+    // Convert to julia values
+    jl_value_t *expr=NULL, *end_pos=NULL;
+    JL_GC_PUSH2(&expr, &end_pos);
+    expr = fl_expr == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, fl_expr, NULL);
+    end_pos = jl_box_long(pos1 + 1);
+    jl_ast_ctx_leave(ctx);
+    jl_value_t *result = (jl_value_t*)jl_svec2(expr, end_pos);
     JL_GC_POP();
     return result;
 }
 
-// parse an entire string like a file, reading multiple expressions
-JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len,
-                                      const char *filename, size_t filename_len)
-{
-    jl_value_t *p = jl_fl_parse(str, len, filename, filename_len,
-                                0, JL_PARSE_TOPLEVEL);
-    return jl_svecref(p, 0);
-}
-
-// this is for parsing one expression out of a string, keeping track of
-// the current position.
-JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
-                                         int pos0, int greedy)
-{
-    return jl_fl_parse(str, len, "none", 4, pos0,
-                       greedy ? JL_PARSE_STATEMENTS : JL_PARSE_ATOM);
-}
-
-// deprecated
-JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len,
-                                             const char *filename, size_t filename_len)
-{
-    return jl_parse_all(str, len, filename, filename_len);
-}
-
 // returns either an expression or a thunk
 jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule)
 {

src/frontend.c

@@ -0,0 +1,67 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#include "julia.h"
+#include "julia_internal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//------------------------------------------------------------------------------
+// Parsing
+
+// Pointer to the current parser
+jl_parse_func_t jl_current_parser = NULL;
+
+JL_DLLEXPORT void jl_set_parser(jl_parse_func_t parser)
+{
+    jl_current_parser = parser;
+}
+
+JL_DLLEXPORT jl_value_t *jl_parse(jl_value_t *text, jl_value_t *filename,
+                                  int start_pos, int rule)
+{
+    return (*jl_current_parser)(text, filename, start_pos, rule);
+}
+
+// C API
+// parse an entire string like a file, reading multiple expressions
+JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len,
+                                      const char *filename, size_t filename_len)
+{
+    jl_value_t *text = NULL;
+    jl_value_t *filename_ = NULL;
+    JL_GC_PUSH2(&text, &filename_);
+    text = jl_pchar_to_string(str, len);
+    filename_ = jl_pchar_to_string(filename, filename_len);
+    jl_value_t *p = jl_parse(text, filename_, 1, JL_PARSE_TOPLEVEL);
+    JL_GC_POP();
+    return jl_svecref(p, 0);
+}
+
+// this is for parsing one expression out of a string, keeping track of
+// the current position.
+JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
+                                         int pos0, int greedy)
+{
+    jl_value_t *text = NULL;
+    jl_value_t *filename_ = NULL;
+    JL_GC_PUSH2(&text, &filename_);
+    text = jl_pchar_to_string(str, len);
+    filename_ = jl_cstr_to_string("none");
+    jl_value_t *result = jl_parse(text, filename_, pos0+1,
+                                  greedy ? JL_PARSE_STATEMENT : JL_PARSE_ATOM);
+    JL_GC_POP();
+    return result;
+}
+
+// deprecated
+JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len,
+                                             const char *filename, size_t filename_len)
+{
+    return jl_parse_all(str, len, filename, filename_len);
+}
+
+#ifdef __cplusplus
+}
+#endif

src/init.c

@@ -734,6 +734,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)
 #endif
     jl_init_common_symbols();
     jl_init_flisp();
+    jl_set_parser(jl_fl_parse);
     jl_init_serializer();
 
     if (!jl_options.image_file) {

src/julia.h

@@ -1597,7 +1597,16 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *d
 JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz, jl_array_t *depmods);
 
 // front end interface
+typedef enum {
+    JL_PARSE_ATOM      = 1,
+    JL_PARSE_STATEMENT = 2,
+    JL_PARSE_TOPLEVEL  = 3,
+} jl_parse_rule_t;
+
 // parsing
+JL_DLLEXPORT jl_value_t *jl_parse(jl_value_t *text, jl_value_t *filename,
+                                  int start_pos, int rule);
+// TODO: Deprecate or convert to passing String to the next two?
 JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len,
                                       const char *filename, size_t filename_len);
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
@@ -1611,7 +1620,7 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *
 JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule);
 JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule,
                                                  const char *file, int line);
-// deprecated; use jl_parse_all
+// deprecated; use jl_parse
 JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len,
                                              const char *filename, size_t filename_len);
 

src/julia_internal.h

@@ -638,6 +638,13 @@ jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs
 
 int jl_has_meta(jl_array_t *body, jl_sym_t *sym);
 
+// Parser replacement
+typedef jl_value_t* (*jl_parse_func_t)(jl_value_t*, jl_value_t*, int, int);
+JL_DLLEXPORT void jl_set_parser(jl_parse_func_t parser);
+// Builtin flisp parser
+JL_DLLEXPORT jl_value_t *jl_fl_parse(jl_value_t *text, jl_value_t *filename,
+                                     int start_pos, int rule);
+
 //--------------------------------------------------
 // Backtraces