flambda-backend: Restructure LIBDIR: Move Dynlink, Str and Unix to sub-directories (port PR#11198) (#895)

* Install otherlibs to separate directories

* Update dune install for otherlibs

* Add Warnings.ghost_loc_in_file

Factors out some common usage between Warnings and Location for
zero-length ghost locations.

* Add a hook for misses in Load_path

Load_path.find{,_uncap} now invoke a hook before raising Not_found,
which provides a mechanism for automatically adding directories to the
Load_path if required.

* Display an alert if -I +unix, etc is missing

* flag -nocwd: do not implicitly add the current directory to the load path (port PR#11131)

* native_toplevel auto_include path

Co-authored-by: David Allsopp <[email protected]>
Co-authored-by: thomasRgl <[email protected]>

Author: 3 people

.depend

@@ -6257,6 +6257,7 @@ driver/compmisc.cmx : \
     utils/clflags.cmx \
     driver/compmisc.cmi
 driver/compmisc.cmi : \
+    utils/load_path.cmi \
     typing/env.cmi \
     utils/clflags.cmi
 driver/errors.cmo : \

Makefile.common-jst

@@ -202,11 +202,12 @@ install_for_test: _install
 	 done; \
 	 ln -s . lex; ln -s . yacc; \
 	 ln -s _install/lib/ocaml/compiler-libs compilerlibs; \
-	 mkdir -p otherlibs/{unix,dynlink/native,str}; \
+	 mkdir -p otherlibs/{unix,dynlink/native,str,bigarray}; \
 	 ln -s ../stdlib/threads otherlibs/systhreads; \
-	 $(cpl) stdlib/{lib,}unix* otherlibs/unix; \
-	 $(cpl) stdlib/dynlink* otherlibs/dynlink; \
-	 $(cpl) stdlib/{lib,}str* otherlibs/str; \
+         $(cpl) stdlib/unix/{lib,}unix* otherlibs/unix; \
+         $(cpl) stdlib/dynlink/dynlink* otherlibs/dynlink; \
+         $(cpl) stdlib/str/{lib,}str* otherlibs/str; \
+         ${cpl} stdlib/bigarray/bigarray* otherlibs/bigarray; \
 	 ln -s ../_build/main/$(ocamldir)/toplevel/byte/.ocamltoplevel.objs/byte toplevel; \
 	)
 
@@ -218,6 +219,7 @@ install_for_test: _install
 	cp $(main_build)/$(ocamldir)/.ocamlcommon.objs/byte/*.cmo _runtest/utils
 	rm -f _runtest/utils/{topdirs,opttopdirs}.cmi
 	cp _install/lib/ocaml/*.{cmi,cma,a,cmxa} _runtest/utils
+	cp _install/lib/ocaml/{unix,str,dynlink,bigarray}/*.{cmi,cma,a,cmxa} _runtest/utils
 	cp $(main_build)/$(ocamldir)/.ocamlcommon.objs/native/config.o _runtest/utils
 	# Needed for tests/warnings
 	cp $(ocamldir)/utils/warnings.ml _runtest/utils

compilerlibs/Makefile.compilerlibs

@@ -31,10 +31,10 @@ UTILS = \
   utils/identifiable.cmo \
   utils/numbers.cmo \
   utils/arg_helper.cmo \
-  utils/clflags.cmo \
-  utils/profile.cmo \
   utils/local_store.cmo \
   utils/load_path.cmo \
+  utils/clflags.cmo \
+  utils/profile.cmo \
   utils/terminfo.cmo \
   utils/ccomp.cmo \
   utils/warnings.cmo \

debugger/.depend

@@ -72,6 +72,7 @@ command_line.cmo : \
     debugger_lexer.cmi \
     debugger_config.cmi \
     debugcom.cmi \
+    ../driver/compmisc.cmi \
     checkpoints.cmi \
     breakpoints.cmi \
     command_line.cmi
@@ -109,6 +110,7 @@ command_line.cmx : \
     debugger_lexer.cmx \
     debugger_config.cmx \
     debugcom.cmx \
+    ../driver/compmisc.cmx \
     checkpoints.cmx \
     breakpoints.cmx \
     command_line.cmi
@@ -327,6 +329,7 @@ main.cmo : \
     exec.cmi \
     debugger_config.cmi \
     ../utils/config.cmi \
+    ../driver/compmisc.cmi \
     command_line.cmi \
     ../file_formats/cmi_format.cmi \
     ../utils/clflags.cmi \
@@ -349,6 +352,7 @@ main.cmx : \
     exec.cmx \
     debugger_config.cmx \
     ../utils/config.cmx \
+    ../driver/compmisc.cmx \
     command_line.cmx \
     ../file_formats/cmi_format.cmx \
     ../utils/clflags.cmx \
@@ -476,6 +480,7 @@ program_management.cmo : \
     history.cmi \
     ../typing/envaux.cmi \
     debugger_config.cmi \
+    ../driver/compmisc.cmi \
     breakpoints.cmi \
     program_management.cmi
 program_management.cmx : \
@@ -493,6 +498,7 @@ program_management.cmx : \
     history.cmx \
     ../typing/envaux.cmx \
     debugger_config.cmx \
+    ../driver/compmisc.cmx \
     breakpoints.cmx \
     program_management.cmi
 program_management.cmi :

debugger/command_line.ml

@@ -262,7 +262,7 @@ let instr_dir ppf lexbuf =
   let new_directory = argument_list_eol argument lexbuf in
     if new_directory = [] then begin
       if yes_or_no "Reinitialize directory list" then begin
-        Load_path.init !default_load_path;
+        Load_path.init ~auto_include:Compmisc.auto_include !default_load_path;
         Envaux.reset_cache ~preserve_persistent_env:false;
         Hashtbl.clear Debugger_config.load_path_for;
         flush_buffer_list ()

debugger/main.ml

@@ -230,7 +230,7 @@ let main () =
     if !Parameters.version
     then printf "\tOCaml Debugger version %s@.@." Config.version;
     Loadprinter.init();
-    Load_path.init !default_load_path;
+    Load_path.init ~auto_include:Compmisc.auto_include !default_load_path;
     Clflags.recursive_types := true;    (* Allow recursive types. *)
     toplevel_loop ();                   (* Toplevel. *)
     kill_program ();

debugger/program_management.ml

@@ -128,7 +128,8 @@ let initialize_loading () =
   end;
   Symbols.clear_symbols ();
   Symbols.read_symbols 0 !program_name;
-  Load_path.init (Load_path.get_paths () @ !Symbols.program_source_dirs);
+  let dirs = Load_path.get_paths () @ !Symbols.program_source_dirs in
+  Load_path.init ~auto_include:Compmisc.auto_include dirs;
   Envaux.reset_cache ~preserve_persistent_env:false;
   if !debug_loading then
     prerr_endline "Opening a socket...";

driver/compenv.ml

@@ -261,6 +261,7 @@ let read_one_param ppf position name v =
   | "noassert" -> set "noassert" [ noassert ] v
   | "noautolink" -> set "noautolink" [ no_auto_link ] v
   | "nostdlib" -> set "nostdlib" [ no_std_include ] v
+  | "nocwd" -> set "nocwd" [ no_cwd ] v
   | "linkall" -> set "linkall" [ link_everything ] v
   | "nolabels" -> set "nolabels" [ classic ] v
   | "principal" -> set "principal"  [ principal ] v

driver/compmisc.ml

@@ -13,8 +13,16 @@
 (*                                                                        *)
 (**************************************************************************)
 
+let auto_include find_in_dir fn =
+  if !Clflags.no_std_include then
+    raise Not_found
+  else
+    let alert = Location.auto_include_alert in
+    Load_path.auto_include_otherlibs alert find_in_dir fn
+
 (* Initialize the search path.
-   [dir] is always searched first (default: the current directory),
+   [dir] (default: the current directory)
+   is always searched first  unless -nocwd is specified,
    then the directories specified with the -I option (in command-line order),
    then the standard library directory (unless the -nostdlib option is given).
  *)
@@ -30,8 +38,13 @@ let init_path ?(dir="") () =
     !Compenv.first_include_dirs
   in
   let exp_dirs =
-    List.map (Misc.expand_directory Config.standard_library) dirs in
-  Load_path.init (dir :: List.rev_append exp_dirs (Clflags.std_include_dir ()));
+    List.map (Misc.expand_directory Config.standard_library) dirs
+  in
+  let dirs =
+    (if !Clflags.no_cwd then [] else [dir])
+    @ List.rev_append exp_dirs (Clflags.std_include_dir ())
+  in
+  Load_path.init ~auto_include dirs;
   Env.reset_cache ~preserve_persistent_env:false
 
 (* Return the initial environment in which compilation proceeds. *)

driver/compmisc.mli

@@ -22,3 +22,10 @@ val read_clflags_from_env : unit -> unit
 
 val with_ppf_dump : ?stdout:unit ->
   file_prefix:string -> (Format.formatter -> 'a) -> 'a
+
+val auto_include :
+  (Load_path.Dir.t -> string -> string option) -> string -> string
+(** [auto_include find_in_dir fn] is a callback function to be passed to
+    {!Load_path.init} and automatically adds [-I +lib] to the load path after
+    displaying an alert. *)
+

driver/main_args.ml

@@ -421,6 +421,10 @@ let mk_nostdlib f =
   " Do not add default directory to the list of include directories"
 ;;
 
+let mk_nocwd f =
+  "-nocwd", Arg.Unit f,
+  " Do not implicitly add the current directory to the load path"
+
 let mk_no_unbox_free_vars_of_closures f =
   "-no-unbox-free-vars-of-closures", Arg.Unit f,
   " Do not unbox variables that will appear inside function closures"
@@ -959,6 +963,7 @@ module type Common_options = sig
   val _noassert : unit -> unit
   val _nolabels : unit -> unit
   val _nostdlib : unit -> unit
+  val _nocwd : unit -> unit
   val _open : string -> unit
   val _ppx : string -> unit
   val _principal : unit -> unit
@@ -1247,6 +1252,7 @@ struct
     mk_noautolink_byt F._noautolink;
     mk_nolabels F._nolabels;
     mk_nostdlib F._nostdlib;
+    mk_nocwd F._nocwd;
     mk_nopervasives F._nopervasives;
     mk_o F._o;
     mk_opaque F._opaque;
@@ -1335,6 +1341,7 @@ struct
     mk_noprompt F._noprompt;
     mk_nopromptcont F._nopromptcont;
     mk_nostdlib F._nostdlib;
+    mk_nocwd F._nocwd;
     mk_nopervasives F._nopervasives;
     mk_open F._open;
     mk_ppx F._ppx;
@@ -1448,6 +1455,7 @@ struct
     mk_no_insn_sched F._no_insn_sched;
     mk_nolabels F._nolabels;
     mk_nostdlib F._nostdlib;
+    mk_nocwd F._nocwd;
     mk_nopervasives F._nopervasives;
     mk_no_unbox_free_vars_of_closures F._no_unbox_free_vars_of_closures;
     mk_no_unbox_specialised_args F._no_unbox_specialised_args;
@@ -1582,6 +1590,7 @@ module Make_opttop_options (F : Opttop_options) = struct
     mk_noprompt F._noprompt;
     mk_nopromptcont F._nopromptcont;
     mk_nostdlib F._nostdlib;
+    mk_nocwd F._nocwd;
     mk_nopervasives F._nopervasives;
     mk_no_unbox_free_vars_of_closures F._no_unbox_free_vars_of_closures;
     mk_no_unbox_specialised_args F._no_unbox_specialised_args;
@@ -1672,6 +1681,7 @@ struct
     mk_noassert F._noassert;
     mk_nolabels F._nolabels;
     mk_nostdlib F._nostdlib;
+    mk_nocwd F._nocwd;
     mk_open F._open;
     mk_pp F._pp;
     mk_ppx F._ppx;
@@ -1768,6 +1778,7 @@ module Default = struct
     let _noassert = set noassert
     let _nolabels = set classic
     let _nostdlib = set no_std_include
+    let _nocwd = set no_cwd
     let _open s = open_modules := (s :: (!open_modules))
     let _principal = set principal
     let _rectypes = set recursive_types

driver/main_args.mli

@@ -30,6 +30,7 @@ module type Common_options = sig
   val _noassert : unit -> unit
   val _nolabels : unit -> unit
   val _nostdlib : unit -> unit
+  val _nocwd : unit -> unit
   val _open : string -> unit
   val _ppx : string -> unit
   val _principal : unit -> unit

otherlibs/Makefile.otherlibs.common

@@ -88,6 +88,8 @@ $(LIBNAME).cmxs: $(LIBNAME).cmxa $(STUBSLIB)
 lib$(CLIBNAME).$(A): $(COBJS)
 	$(MKLIB_CMD) -oc $(CLIBNAME) $(COBJS) $(LDOPTS)
 
+INSTALL_LIBDIR_LIBNAME = $(INSTALL_LIBDIR)/$(LIBNAME)
+
 install::
 	if test -f dll$(CLIBNAME)$(EXT_DLL); then \
 	  $(INSTALL_PROG) \
@@ -96,15 +98,21 @@ install::
 ifneq "$(STUBSLIB)" ""
 	$(INSTALL_DATA) $(STUBSLIB) "$(INSTALL_LIBDIR)/"
 endif
-
+# If installing over a previous OCaml version, ensure the library is removed
+# from the previous installation.
+	rm -f $(addprefix "$(INSTALL_LIBDIR)"/, \
+          $(LIBNAME).cma $(CMIFILES) \
+          $(CMIFILES:.cmi=.mli) $(CMIFILES:.cmi=.cmti) \
+          $(CAMLOBJS_NAT) $(LIBNAME).cmxa $(LIBNAME).cmxs $(LIBNAME).$(A))
+	$(MKDIR) "$(INSTALL_LIBDIR_LIBNAME)"
 	$(INSTALL_DATA) \
 	  $(LIBNAME).cma $(CMIFILES) \
-	  "$(INSTALL_LIBDIR)/"
+	  "$(INSTALL_LIBDIR_LIBNAME)/"
 ifeq "$(INSTALL_SOURCE_ARTIFACTS)" "true"
 	$(INSTALL_DATA) \
 	  $(CMIFILES:.cmi=.mli) \
           $(CMIFILES:.cmi=.cmti) \
-	  "$(INSTALL_LIBDIR)/"
+	  "$(INSTALL_LIBDIR_LIBNAME)/"
 endif
 	if test -n "$(HEADERS)"; then \
 	  $(INSTALL_DATA) $(HEADERS) "$(INSTALL_LIBDIR)/caml/"; \
@@ -113,9 +121,9 @@ endif
 installopt:
 	$(INSTALL_DATA) \
 	   $(CAMLOBJS_NAT) $(LIBNAME).cmxa $(LIBNAME).$(A) \
-	   "$(INSTALL_LIBDIR)/"
+	   "$(INSTALL_LIBDIR_LIBNAME)/"
 	if test -f $(LIBNAME).cmxs; then \
-	  $(INSTALL_PROG) $(LIBNAME).cmxs "$(INSTALL_LIBDIR)"; \
+	  $(INSTALL_PROG) $(LIBNAME).cmxs "$(INSTALL_LIBDIR_LIBNAME)"; \
 	fi
 
 partialclean:

otherlibs/bigarray/dune

@@ -25,14 +25,14 @@
 
 (install
   (files
-    .bigarray.objs/native/bigarray.cmx
-    bigarray.cmxa
-    bigarray.a
-    bigarray.cmxs
-    bigarray.cma
-    .bigarray.objs/byte/bigarray.cmi
-    .bigarray.objs/byte/bigarray.cmti
-    bigarray.mli
+    (.bigarray.objs/native/bigarray.cmx as bigarray/bigarray.cmx)
+    (bigarray.cmxa as bigarray/bigarray.cmxa)
+    (bigarray.a as bigarray/bigarray.a)
+    (bigarray.cmxs as bigarray/bigarray.cmxs)
+    (bigarray.cma as bigarray/bigarray.cma)
+    (.bigarray.objs/byte/bigarray.cmi as bigarray/bigarray.cmi)
+    (.bigarray.objs/byte/bigarray.cmti as bigarray/bigarray.cmti)
+    (bigarray.mli as bigarray/bigarray.mli)
   )
   (section lib)
   (package ocaml))

otherlibs/dynlink/Makefile

@@ -77,13 +77,13 @@ COMPILERLIBS_SOURCES=\
   utils/identifiable.ml \
   utils/numbers.ml \
   utils/arg_helper.ml \
+  utils/local_store.ml \
+  utils/load_path.ml \
   utils/clflags.ml \
   utils/profile.ml \
   utils/consistbl.ml \
   utils/terminfo.ml \
   utils/warnings.ml \
-  utils/local_store.ml \
-  utils/load_path.ml \
   utils/int_replace_polymorphic_compare.ml \
   typing/ident.ml \
   utils/compilation_unit.ml \
@@ -236,21 +236,29 @@ $(eval $(call PROGRAM_SYNONYM,extract_crc))
 $(extract_crc): dynlink.cma dynlink_compilerlibs.cmo extract_crc.cmo
 	$(OCAMLC) -o $@ $^
 
+INSTALL_LIBDIR_DYNLINK = $(INSTALL_LIBDIR)/dynlink
+
 install:
+# If installing over a previous OCaml version, ensure dynlink is removed from
+# the previous installation.
+	rm -f "$(INSTALL_LIBDIR)"/dynlink.cm* "$(INSTALL_LIBDIR)/dynlink.mli" \
+        "$(INSTALL_LIBDIR)/dynlink.$(A)" \
+        $(addprefix "$(INSTALL_LIBDIR)/", $(notdir $(NATOBJS)))
+	$(MKDIR) "$(INSTALL_LIBDIR_DYNLINK)"
 	$(INSTALL_DATA) \
 	  dynlink.cmi dynlink.cma \
-	  "$(INSTALL_LIBDIR)"
+	  "$(INSTALL_LIBDIR_DYNLINK)"
 ifeq "$(INSTALL_SOURCE_ARTIFACTS)" "true"
 	$(INSTALL_DATA) \
 	  dynlink.cmti dynlink.mli \
-	  "$(INSTALL_LIBDIR)"
+	  "$(INSTALL_LIBDIR_DYNLINK)"
 endif
 
-installopt:
+installopt:g
 ifeq "$(strip $(NATDYNLINK))" "true"
 	$(INSTALL_DATA) \
 	  $(NATOBJS) dynlink.cmxa dynlink.$(A) \
-	  "$(INSTALL_LIBDIR)"
+	  "$(INSTALL_LIBDIR_DYNLINK)"
 endif
 
 partialclean:

otherlibs/dynlink/dune

@@ -374,18 +374,18 @@
 
 (install
   (files
-    .dynlink_internal.objs/native/dynlink_common.cmx
-    .dynlink_internal.objs/native/dynlink_platform_intf.cmx
-    .dynlink_internal.objs/native/dynlink_types.cmx
-    .dynlink_internal.objs/native/dynlink.cmx
-    dynlink.cma
-    dynlink.cmxa
-    dynlink.a
-;    dynlink.cmxs
-    dynlink.mli
-    dynlink.cmi
-    (.dynlink_internal.objs/byte/dynlink.cmti as dynlink.cmti)
-    (extract_crc.bc as extract_crc)
+    (.dynlink_internal.objs/native/dynlink_common.cmx as dynlink/dynlink_common.cmx)
+    (.dynlink_internal.objs/native/dynlink_platform_intf.cmx as dynlink/dynlink_platform_intf.cmx)
+    (.dynlink_internal.objs/native/dynlink_types.cmx as dynlink/dynlink_types.cmx)
+    (.dynlink_internal.objs/native/dynlink.cmx as dynlink/dynlink.cmx)
+    (dynlink.cma as dynlink/dynlink.cma)
+    (dynlink.cmxa as dynlink/dynlink.cmxa)
+    (dynlink.a as dynlink/dynlink.a)
+;    (dynlink.cmxs as dynlink/dynlink.cmxs)
+    (dynlink.mli as dynlink/dynlink.mli)
+    (dynlink.cmi as dynlink/dynlink.cmi)
+    (.dynlink_internal.objs/byte/dynlink.cmti as dynlink/dynlink.cmti)
+    (extract_crc.bc as dynlink/extract_crc)
 
     .dynlink_compilerlibs.objs/native/dynlink_compilerlibs__Misc.cmx
   )