flambda-backend: Backport 12508 (#2294)

* Backport 12508. shape-index tests + cms not working

* Fix tests format for shape-index

* store occurrences as an array instead as of a list

* Reimplement cms files

* extensions have no shape for now

* Update .depend

* code review

* fix build

* restored objinfo

* bootstrap

* fix build_upstream_closure

Author: poechsel

.depend

@@ -824,7 +824,6 @@ typing/env.cmo : \
     typing/predef.cmi \
     typing/persistent_env.cmi \
     typing/path.cmi \
-    parsing/parsetree.cmi \
     typing/mode.cmi \
     utils/misc.cmi \
     parsing/longident.cmi \
@@ -851,7 +850,6 @@ typing/env.cmx : \
     typing/predef.cmx \
     typing/persistent_env.cmx \
     typing/path.cmx \
-    parsing/parsetree.cmi \
     typing/mode.cmx \
     utils/misc.cmx \
     parsing/longident.cmx \
@@ -876,7 +874,6 @@ typing/env.cmi : \
     typing/subst.cmi \
     typing/shape.cmi \
     typing/path.cmi \
-    parsing/parsetree.cmi \
     typing/mode.cmi \
     utils/misc.cmi \
     parsing/longident.cmi \
@@ -1514,15 +1511,13 @@ typing/rec_check.cmi : \
     typing/typedtree.cmi \
     typing/ident.cmi
 typing/shape.cmo : \
-    utils/profile.cmi \
     typing/path.cmi \
     utils/misc.cmi \
     utils/identifiable.cmi \
     typing/ident.cmi \
     utils/compilation_unit.cmi \
     typing/shape.cmi
 typing/shape.cmx : \
-    utils/profile.cmx \
     typing/path.cmx \
     utils/misc.cmx \
     utils/identifiable.cmx \
@@ -1534,6 +1529,21 @@ typing/shape.cmi : \
     utils/identifiable.cmi \
     typing/ident.cmi \
     utils/compilation_unit.cmi
+typing/shape_reduce.cmo : \
+    typing/shape.cmi \
+    utils/local_store.cmi \
+    typing/ident.cmi \
+    typing/env.cmi \
+    typing/shape_reduce.cmi
+typing/shape_reduce.cmx : \
+    typing/shape.cmx \
+    utils/local_store.cmx \
+    typing/ident.cmx \
+    typing/env.cmx \
+    typing/shape_reduce.cmi
+typing/shape_reduce.cmi : \
+    typing/shape.cmi \
+    typing/env.cmi
 typing/signature_group.cmo : \
     typing/types.cmi \
     typing/ident.cmi \
@@ -1845,6 +1855,7 @@ typing/typedecl.cmo : \
     typing/typedecl_variance.cmi \
     typing/typedecl_separability.cmi \
     typing/subst.cmi \
+    typing/shape.cmi \
     typing/printtyp.cmi \
     typing/primitive.cmi \
     typing/predef.cmi \
@@ -1882,6 +1893,7 @@ typing/typedecl.cmx : \
     typing/typedecl_variance.cmx \
     typing/typedecl_separability.cmx \
     typing/subst.cmx \
+    typing/shape.cmx \
     typing/printtyp.cmx \
     typing/primitive.cmx \
     typing/predef.cmx \
@@ -1915,6 +1927,7 @@ typing/typedecl.cmi : \
     typing/typedtree.cmi \
     typing/typedecl_variance.cmi \
     typing/typedecl_separability.cmi \
+    typing/shape.cmi \
     typing/path.cmi \
     parsing/parsetree.cmi \
     parsing/longident.cmi \
@@ -2063,6 +2076,7 @@ typing/typemod.cmo : \
     typing/typeclass.cmi \
     typing/subst.cmi \
     typing/signature_group.cmi \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
     utils/profile.cmi \
     typing/printtyp.cmi \
@@ -2106,6 +2120,7 @@ typing/typemod.cmx : \
     typing/typeclass.cmx \
     typing/subst.cmx \
     typing/signature_group.cmx \
+    typing/shape_reduce.cmx \
     typing/shape.cmx \
     utils/profile.cmx \
     typing/printtyp.cmx \
@@ -4582,68 +4597,93 @@ file_formats/cmo_format.cmi : \
     typing/ident.cmi \
     utils/compilation_unit.cmi
 file_formats/cms_format.cmo : \
+    typing/types.cmi \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
     parsing/parsetree.cmi \
     utils/misc.cmi \
+    parsing/longident.cmi \
     parsing/location.cmi \
     parsing/lexer.cmi \
-    typing/env.cmi \
     utils/config.cmi \
     utils/compilation_unit.cmi \
+    file_formats/cmt_format.cmi \
     utils/clflags.cmi \
     file_formats/cms_format.cmi
 file_formats/cms_format.cmx : \
+    typing/types.cmx \
+    typing/shape_reduce.cmx \
     typing/shape.cmx \
     parsing/parsetree.cmi \
     utils/misc.cmx \
+    parsing/longident.cmx \
     parsing/location.cmx \
     parsing/lexer.cmx \
-    typing/env.cmx \
     utils/config.cmx \
     utils/compilation_unit.cmx \
+    file_formats/cmt_format.cmx \
     utils/clflags.cmx \
     file_formats/cms_format.cmi
 file_formats/cms_format.cmi : \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
     parsing/parsetree.cmi \
+    parsing/longident.cmi \
     parsing/location.cmi \
-    utils/compilation_unit.cmi
+    utils/compilation_unit.cmi \
+    file_formats/cmt_format.cmi
 file_formats/cmt_format.cmo : \
     typing/types.cmi \
     typing/typedtree.cmi \
     typing/tast_mapper.cmi \
+    typing/tast_iterator.cmi \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
+    typing/predef.cmi \
+    typing/path.cmi \
     utils/misc.cmi \
+    parsing/longident.cmi \
     parsing/location.cmi \
     utils/load_path.cmi \
     parsing/lexer.cmi \
     utils/import_info.cmi \
+    typing/ident.cmi \
     typing/env.cmi \
     utils/config.cmi \
     utils/compilation_unit.cmi \
     file_formats/cmi_format.cmi \
     utils/clflags.cmi \
+    typing/btype.cmi \
     file_formats/cmt_format.cmi
 file_formats/cmt_format.cmx : \
     typing/types.cmx \
     typing/typedtree.cmx \
     typing/tast_mapper.cmx \
+    typing/tast_iterator.cmx \
+    typing/shape_reduce.cmx \
     typing/shape.cmx \
+    typing/predef.cmx \
+    typing/path.cmx \
     utils/misc.cmx \
+    parsing/longident.cmx \
     parsing/location.cmx \
     utils/load_path.cmx \
     parsing/lexer.cmx \
     utils/import_info.cmx \
+    typing/ident.cmx \
     typing/env.cmx \
     utils/config.cmx \
     utils/compilation_unit.cmx \
     file_formats/cmi_format.cmx \
     utils/clflags.cmx \
+    typing/btype.cmx \
     file_formats/cmt_format.cmi
 file_formats/cmt_format.cmi : \
     typing/types.cmi \
     typing/typedtree.cmi \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
+    parsing/longident.cmi \
     parsing/location.cmi \
     utils/load_path.cmi \
     utils/import_info.cmi \
@@ -7259,6 +7299,7 @@ toplevel/byte/topeval.cmo : \
     toplevel/topcommon.cmi \
     bytecomp/symtable.cmi \
     lambda/simplif.cmi \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
     typing/printtyped.cmi \
     typing/printtyp.cmi \
@@ -7295,6 +7336,7 @@ toplevel/byte/topeval.cmx : \
     toplevel/topcommon.cmx \
     bytecomp/symtable.cmx \
     lambda/simplif.cmx \
+    typing/shape_reduce.cmx \
     typing/shape.cmx \
     typing/printtyped.cmx \
     typing/printtyp.cmx \
@@ -7401,6 +7443,7 @@ toplevel/native/topeval.cmo : \
     toplevel/topcommon.cmi \
     utils/symbol.cmi \
     lambda/simplif.cmi \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
     typing/printtyped.cmi \
     typing/printtyp.cmi \
@@ -7435,6 +7478,7 @@ toplevel/native/topeval.cmx : \
     toplevel/topcommon.cmx \
     utils/symbol.cmx \
     lambda/simplif.cmx \
+    typing/shape_reduce.cmx \
     typing/shape.cmx \
     typing/printtyped.cmx \
     typing/printtyp.cmx \
@@ -7737,11 +7781,15 @@ tools/make_opcodes.cmx : \
     tools/make_opcodes.cmi
 tools/make_opcodes.cmi :
 tools/objinfo.cmo : \
+    typing/typedtree.cmi \
     bytecomp/symtable.cmi \
     utils/symbol.cmi \
+    typing/shape_reduce.cmi \
     typing/shape.cmi \
     middle_end/printclambda.cmi \
+    parsing/pprintast.cmi \
     utils/misc.cmi \
+    parsing/location.cmi \
     lambda/lambda.cmi \
     utils/import_info.cmi \
     typing/ident.cmi \
@@ -7757,11 +7805,15 @@ tools/objinfo.cmo : \
     utils/binutils.cmi \
     tools/objinfo.cmi
 tools/objinfo.cmx : \
+    typing/typedtree.cmx \
     bytecomp/symtable.cmx \
     utils/symbol.cmx \
+    typing/shape_reduce.cmx \
     typing/shape.cmx \
     middle_end/printclambda.cmx \
+    parsing/pprintast.cmx \
     utils/misc.cmx \
+    parsing/location.cmx \
     lambda/lambda.cmx \
     utils/import_info.cmx \
     typing/ident.cmx \

Changes

@@ -741,6 +741,12 @@ Some of those changes will benefit all OCaml packages.
   bytecode executable (`ocamlc -custom`).
   (Antonin Décimo, review by Xavier Leroy)
 
+- #12508 : Add compiler-side support for project-wide occurrences in Merlin, by
+  generating index tables of all identifier occurrences. This extra data in .cmt
+  files is only added when the new flag -bin-annot-occurrences is passed.
+  (Ulysse Gérard, Nathanaëlle Courant, suggestions by Gabriel Scherer and Thomas
+  Refis, review by Florian Angeletti, Gabriel Scherer and Thomas Refis)
+
 ### Build system:
 
 - #11844: Reduce verbosity of `make` logs by printing program invocations in

boot/ocamlc

@@ -98,6 +98,7 @@ TYPING = \
   file_formats/cmi_format.cmo \
   typing/persistent_env.cmo \
   typing/env.cmo \
+  typing/shape_reduce.cmo \
   typing/errortrace.cmo \
   typing/typedtree.cmo \
   typing/signature_group.cmo \

compilerlibs/Makefile.compilerlibs

@@ -45,6 +45,13 @@ let mk_binannot_cms f =
   "-bin-annot-cms", Arg.Unit f, " Save shapes in <filename>.cms"
 ;;
 
+let mk_binannot_occurrences f =
+  "-bin-annot-occurrences", Arg.Unit f,
+  " Store every occurrence of a bound name in the .cmt file.\n\
+    This information can be used by external tools to provide\n\
+    features such as project-wide occurrences. This flag has\n\
+    no effect in the absence of '-bin-annot'."
+
 let mk_c f =
   "-c", Arg.Unit f, " Compile only (do not link)"
 
@@ -941,6 +948,7 @@ module type Compiler_options = sig
   val _as_parameter : unit -> unit
   val _binannot : unit -> unit
   val _binannot_cms : unit -> unit
+  val _binannot_occurrences : unit -> unit
   val _c : unit -> unit
   val _cc : string -> unit
   val _cclib : string -> unit
@@ -1140,6 +1148,7 @@ struct
     mk_as_parameter F._as_parameter;
     mk_binannot F._binannot;
     mk_binannot_cms F._binannot_cms;
+    mk_binannot_occurrences F._binannot_occurrences;
     mk_c F._c;
     mk_cc F._cc;
     mk_cclib F._cclib;
@@ -1353,6 +1362,7 @@ struct
     mk_as_parameter F._as_parameter;
     mk_binannot F._binannot;
     mk_binannot_cms F._binannot_cms;
+    mk_binannot_occurrences F._binannot_occurrences;
     mk_inline_branch_factor F._inline_branch_factor;
     mk_c F._c;
     mk_cc F._cc;
@@ -1921,6 +1931,7 @@ module Default = struct
     let _as_parameter = set as_parameter
     let _binannot = set binary_annotations
     let _binannot_cms = set binary_annotations_cms
+    let _binannot_occurrences = set store_occurrences
     let _c = set compile_only
     let _cc s = c_compiler := (Some s)
     let _cclib s = Compenv.defer (ProcessObjects (Misc.rev_split_words s))

driver/main_args.ml

@@ -89,6 +89,7 @@ module type Compiler_options = sig
   val _as_parameter : unit -> unit
   val _binannot : unit -> unit
   val _binannot_cms : unit -> unit
+  val _binannot_occurrences : unit -> unit
   val _c : unit -> unit
   val _cc : string -> unit
   val _cclib : string -> unit

driver/main_args.mli

@@ -86,7 +86,7 @@
    jane_syntax ; manual update: mli only files asttypes parsetree
 
    ;; TYPING
-   ident path jkind primitive shape types btype oprint subst predef datarepr
+   ident path jkind primitive shape shape_reduce types btype oprint subst predef datarepr
    cmi_format persistent_env env errortrace mode
    typedtree printtyped ctype printtyp includeclass mtype envaux includecore
    tast_iterator tast_mapper signature_group cmt_format cms_format untypeast