Merge from upstream

.cirrus.yml

@@ -89,7 +89,7 @@ yjit_task:
     matrix:
       - CC: clang-12
         configure: --enable-yjit=dev
-        rustup_init: --default-toolchain=1.58.1
+        rustup_init: --default-toolchain=1.58.0
       - CC: gcc-11
         configure: --enable-yjit
   id_script: id

.github/workflows/yjit-ubuntu.yml

@@ -48,8 +48,8 @@ jobs:
             configure: "--with-gcc=clang-12 --enable-yjit=dev"
 
           - test_task: "check"
-            configure: "--enable-yjit RUSTC='rustc +1.58.1'" # release build
-            rust_version: "1.58.1"
+            configure: "--enable-yjit RUSTC='rustc +1.58.0'" # release build
+            rust_version: "1.58.0"
 
           - test_task: "check"
             configure: "--enable-yjit=dev"

NEWS.md

@@ -192,6 +192,14 @@ Note: We're only listing outstanding class updates.
 
 * RubyVM::AbstractSyntaxTree
     * Add `error_tolerant` option for `parse`, `parse_file` and `of`. [[Feature #19013]]
+    * Add `keep_tokens` option for `parse`, `parse_file` and `of`. Add `#tokens` and `#all_tokens`
+      for `RubyVM::AbstractSyntaxTree::Node` [[Feature #19070]]
+
+    ```ruby
+    root = RubyVM::AbstractSyntaxTree.parse("x = 1 + 2", keep_tokens: true)
+    root.tokens # => [[0, :tIDENTIFIER, "x", [1, 0, 1, 1]], [1, :tSP, " ", [1, 1, 1, 2]], ...]
+    root.tokens.map{_1[2]}.join # => "x = 1 + 2"
+    ```
 
 * Set
     * Set is now available as a built-in class without the need for `require "set"`. [[Feature #16989]]
@@ -217,6 +225,10 @@ Note: We're only listing outstanding class updates.
     * A Struct class can also be initialized with keyword arguments
       without `keyword_init: true` on `Struct.new` [[Feature #16806]]
 
+* Time
+    * `Time#deconstruct_keys` is added, allowing to use `Time` instances
+      in pattern-matching expressions [[Feature #19071]]
+
 * TracePoint
     * TracePoint#binding now returns `nil` for `c_call`/`c_return` TracePoints.
       [[Bug #18487]]
@@ -235,7 +247,7 @@ Note: We're only listing outstanding class updates.
     * RubyGems 3.4.0.dev
     * bigdecimal 3.1.2
     * bundler 2.4.0.dev
-    * cgi 0.3.3
+    * cgi 0.3.5
     * date 3.2.3
     * erb 3.0.0
     * error_highlight 0.5.1
@@ -245,7 +257,7 @@ Note: We're only listing outstanding class updates.
     * io-nonblock 0.1.1
     * io-wait 0.3.0.pre
     * ipaddr 1.2.4
-    * irb 1.4.3
+    * irb 1.5.0
     * json 2.6.2
     * logger 1.5.1
     * net-http 0.3.0
@@ -359,7 +371,7 @@ The following deprecated APIs are removed.
 ### YJIT
 
 * Support arm64 / aarch64 on UNIX platforms.
-* Building YJIT requires Rust 1.58.1+. [[Feature #18481]]
+* Building YJIT requires Rust 1.58.0+. [[Feature #18481]]
 * Physical memory for JIT code is lazily allocated. Unlike Ruby 3.1,
   the RSS of a Ruby process is minimized because virtual memory pages
   allocated by `--yjit-exec-mem-size` will not be mapped to physical
@@ -437,5 +449,7 @@ The following deprecated APIs are removed.
 [Feature #19013]: https://bugs.ruby-lang.org/issues/19013
 [Feature #19026]: https://bugs.ruby-lang.org/issues/19026
 [Feature #19060]: https://bugs.ruby-lang.org/issues/19060
+[Feature #19070]: https://bugs.ruby-lang.org/issues/19070
 [Bug #19100]:     https://bugs.ruby-lang.org/issues/19100
 [Feature #19135]: https://bugs.ruby-lang.org/issues/19135
+[Feature #19071]: https://bugs.ruby-lang.org/issues/19071

README.md

@@ -54,50 +54,6 @@ see [Building Ruby](doc/contributing/building_ruby.md)
 
 https://www.ruby-lang.org/
 
-## Commands
-
-The following commands are available on IRB.
-
-* `cwws`
-  * Show the current workspace.
-* `cb`, `cws`, `chws`
-  * Change the current workspace to an object.
-* `bindings`, `workspaces`
-  * Show workspaces.
-* `pushb`, `pushws`
-  * Push an object to the workspace stack.
-* `popb`, `popws`
-  * Pop a workspace from the workspace stack.
-* `load`
-  * Load a Ruby file.
-* `require`
-  * Require a Ruby file.
-* `source`
-  * Loads a given file in the current session.
-* `irb`
-  * Start a child IRB.
-* `jobs`
-  * List of current sessions.
-* `fg`
-  * Switches to the session of the given number.
-* `kill`
-  * Kills the session with the given number.
-* `help`
-  * Enter the mode to look up RI documents.
-* `irb_info`
-  * Show information about IRB.
-* `ls`
-  * Show methods, constants, and variables.
-    `-g [query]` or `-G [query]` allows you to filter out the output.
-* `measure`
-  * `measure` enables the mode to measure processing time. `measure :off` disables it.
-* `$`, `show_source`
-  * Show the source code of a given method or constant.
-* `@`, `whereami`
-  * Show the source code around binding.irb again.
-* `debug`
-  * Start the debugger of debug.gem.
-
 ## Documentation
 
 - [English](https://docs.ruby-lang.org/en/master/index.html)

array.c

@@ -2086,7 +2086,7 @@ rb_ary_first(int argc, VALUE *argv, VALUE ary)
  *
  *  If +self+ is empty, returns +nil+.
  *
- *  When non-negative \Innteger argument +n+ is given,
+ *  When non-negative \Integer argument +n+ is given,
  *  returns the last +n+ elements in a new \Array:
  *
  *    a = [:foo, 'bar', 2]

ast.c

@@ -64,8 +64,8 @@ ast_new_internal(rb_ast_t *ast, const NODE *node)
     return obj;
 }
 
-static VALUE rb_ast_parse_str(VALUE str, VALUE keep_script_lines, VALUE error_tolerant);
-static VALUE rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant);
+static VALUE rb_ast_parse_str(VALUE str, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens);
+static VALUE rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens);
 
 static VALUE
 ast_parse_new(void)
@@ -85,32 +85,33 @@ ast_parse_done(rb_ast_t *ast)
 }
 
 static VALUE
-ast_s_parse(rb_execution_context_t *ec, VALUE module, VALUE str, VALUE keep_script_lines, VALUE error_tolerant)
+ast_s_parse(rb_execution_context_t *ec, VALUE module, VALUE str, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens)
 {
-    return rb_ast_parse_str(str, keep_script_lines, error_tolerant);
+    return rb_ast_parse_str(str, keep_script_lines, error_tolerant, keep_tokens);
 }
 
 static VALUE
-rb_ast_parse_str(VALUE str, VALUE keep_script_lines, VALUE error_tolerant)
+rb_ast_parse_str(VALUE str, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens)
 {
     rb_ast_t *ast = 0;
 
     StringValue(str);
     VALUE vparser = ast_parse_new();
     if (RTEST(keep_script_lines)) rb_parser_keep_script_lines(vparser);
     if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser);
+    if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser);
     ast = rb_parser_compile_string_path(vparser, Qnil, str, 1);
     return ast_parse_done(ast);
 }
 
 static VALUE
-ast_s_parse_file(rb_execution_context_t *ec, VALUE module, VALUE path, VALUE keep_script_lines, VALUE error_tolerant)
+ast_s_parse_file(rb_execution_context_t *ec, VALUE module, VALUE path, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens)
 {
-    return rb_ast_parse_file(path, keep_script_lines, error_tolerant);
+    return rb_ast_parse_file(path, keep_script_lines, error_tolerant, keep_tokens);
 }
 
 static VALUE
-rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant)
+rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens)
 {
     VALUE f;
     rb_ast_t *ast = 0;
@@ -122,6 +123,7 @@ rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant)
     VALUE vparser = ast_parse_new();
     if (RTEST(keep_script_lines)) rb_parser_keep_script_lines(vparser);
     if (RTEST(error_tolerant))  rb_parser_error_tolerant(vparser);
+    if (RTEST(keep_tokens))  rb_parser_keep_tokens(vparser);
     ast = rb_parser_compile_file_path(vparser, Qnil, f, 1);
     rb_io_close(f);
     return ast_parse_done(ast);
@@ -141,14 +143,15 @@ lex_array(VALUE array, int index)
 }
 
 static VALUE
-rb_ast_parse_array(VALUE array, VALUE keep_script_lines, VALUE error_tolerant)
+rb_ast_parse_array(VALUE array, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens)
 {
     rb_ast_t *ast = 0;
 
     array = rb_check_array_type(array);
     VALUE vparser = ast_parse_new();
     if (RTEST(keep_script_lines)) rb_parser_keep_script_lines(vparser);
     if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser);
+    if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser);
     ast = rb_parser_compile_generic(vparser, lex_array, Qnil, array, 1);
     return ast_parse_done(ast);
 }
@@ -208,7 +211,7 @@ node_id_for_backtrace_location(rb_execution_context_t *ec, VALUE module, VALUE l
 }
 
 static VALUE
-ast_s_of(rb_execution_context_t *ec, VALUE module, VALUE body, VALUE keep_script_lines, VALUE error_tolerant)
+ast_s_of(rb_execution_context_t *ec, VALUE module, VALUE body, VALUE keep_script_lines, VALUE error_tolerant, VALUE keep_tokens)
 {
     VALUE node, lines = Qnil;
     const rb_iseq_t *iseq;
@@ -247,13 +250,13 @@ ast_s_of(rb_execution_context_t *ec, VALUE module, VALUE body, VALUE keep_script
     }
 
     if (!NIL_P(lines) || !NIL_P(lines = script_lines(path))) {
-        node = rb_ast_parse_array(lines, keep_script_lines, error_tolerant);
+        node = rb_ast_parse_array(lines, keep_script_lines, error_tolerant, keep_tokens);
     }
     else if (e_option) {
-        node = rb_ast_parse_str(rb_e_script, keep_script_lines, error_tolerant);
+        node = rb_ast_parse_str(rb_e_script, keep_script_lines, error_tolerant, keep_tokens);
     }
     else {
-        node = rb_ast_parse_file(path, keep_script_lines, error_tolerant);
+        node = rb_ast_parse_file(path, keep_script_lines, error_tolerant, keep_tokens);
     }
 
     return node_find(node, node_id);
@@ -715,6 +718,15 @@ ast_node_last_column(rb_execution_context_t *ec, VALUE self)
     return INT2NUM(nd_last_column(data->node));
 }
 
+static VALUE
+ast_node_all_tokens(rb_execution_context_t *ec, VALUE self)
+{
+    struct ASTNodeData *data;
+    TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data);
+
+    return rb_ast_tokens(data->ast);
+}
+
 static VALUE
 ast_node_inspect(rb_execution_context_t *ec, VALUE self)
 {

ast.rb

@@ -29,8 +29,8 @@ module RubyVM::AbstractSyntaxTree
   #
   #    RubyVM::AbstractSyntaxTree.parse("x = 1 + 2")
   #    # => #<RubyVM::AbstractSyntaxTree::Node:SCOPE@1:0-1:9>
-  def self.parse string, keep_script_lines: false, error_tolerant: false
-    Primitive.ast_s_parse string, keep_script_lines, error_tolerant
+  def self.parse string, keep_script_lines: false, error_tolerant: false, keep_tokens: false
+    Primitive.ast_s_parse string, keep_script_lines, error_tolerant, keep_tokens
   end
 
   #  call-seq:
@@ -44,8 +44,8 @@ def self.parse string, keep_script_lines: false, error_tolerant: false
   #
   #     RubyVM::AbstractSyntaxTree.parse_file("my-app/app.rb")
   #     # => #<RubyVM::AbstractSyntaxTree::Node:SCOPE@1:0-31:3>
-  def self.parse_file pathname, keep_script_lines: false, error_tolerant: false
-    Primitive.ast_s_parse_file pathname, keep_script_lines, error_tolerant
+  def self.parse_file pathname, keep_script_lines: false, error_tolerant: false, keep_tokens: false
+    Primitive.ast_s_parse_file pathname, keep_script_lines, error_tolerant, keep_tokens
   end
 
   #  call-seq:
@@ -63,8 +63,8 @@ def self.parse_file pathname, keep_script_lines: false, error_tolerant: false
   #
   #     RubyVM::AbstractSyntaxTree.of(method(:hello))
   #     # => #<RubyVM::AbstractSyntaxTree::Node:SCOPE@1:0-3:3>
-  def self.of body, keep_script_lines: false, error_tolerant: false
-    Primitive.ast_s_of body, keep_script_lines, error_tolerant
+  def self.of body, keep_script_lines: false, error_tolerant: false, keep_tokens: false
+    Primitive.ast_s_of body, keep_script_lines, error_tolerant, keep_tokens
   end
 
   #  call-seq:
@@ -136,6 +136,46 @@ def last_column
       Primitive.ast_node_last_column
     end
 
+    #  call-seq:
+    #     node.tokens -> array
+    #
+    # Returns tokens corresponding to the location of the node.
+    # Returns nil if keep_tokens is not enabled when parse method is called.
+    # Token is an array of:
+    #
+    #  - id
+    #  - token type
+    #  - source code text
+    #  - location [first_lineno, first_column, last_lineno, last_column]
+    #
+    #    root = RubyVM::AbstractSyntaxTree.parse("x = 1 + 2", keep_tokens: true)
+    #    root.tokens # => [[0, :tIDENTIFIER, "x", [1, 0, 1, 1]], [1, :tSP, " ", [1, 1, 1, 2]], ...]
+    #    root.tokens.map{_1[2]}.join # => "x = 1 + 2"
+    def tokens
+      return nil unless all_tokens
+
+      all_tokens.each_with_object([]) do |token, a|
+        loc = token.last
+        if ([first_lineno, first_column] <=> [loc[0], loc[1]]) <= 0 &&
+           ([last_lineno, last_column]   <=> [loc[2], loc[3]]) >= 0
+           a << token
+        end
+      end
+    end
+
+    #  call-seq:
+    #     node.all_tokens -> array
+    #
+    # Returns all tokens for the input script regardless the receiver node.
+    # Returns nil if keep_tokens is not enabled when parse method is called.
+    #
+    #    root = RubyVM::AbstractSyntaxTree.parse("x = 1 + 2", keep_tokens: true)
+    #    root.all_tokens # => [[0, :tIDENTIFIER, "x", [1, 0, 1, 1]], [1, :tSP, " ", [1, 1, 1, 2]], ...]
+    #    root.children[-1].all_tokens # => [[0, :tIDENTIFIER, "x", [1, 0, 1, 1]], [1, :tSP, " ", [1, 1, 1, 2]], ...]
+    def all_tokens
+      Primitive.ast_node_all_tokens
+    end
+
     #  call-seq:
     #     node.children -> array
     #

common.mk

@@ -1400,6 +1400,17 @@ extract-gems$(gnumake:yes=-sequential): PHONY
 	    -e 'end' \
 	    gems/bundled_gems
 
+extract-gems: outdate-bundled-gems
+outdate-bundled-gems: PHONY
+	$(Q) $(BASERUBY) -C "$(srcdir)" \
+	-rfileutils \
+	-e "Dir.glob('.bundle/gems/*/') {|g|" \
+	-e   "FileUtils::Verbose.rm_rf(g) unless File.exist?(%[gems/#{File.basename(g)}.gem])" \
+	-e "}" \
+	-e "Dir.glob('.bundle/specifications/*.gemspec') {|g|" \
+	-e   "FileUtils::Verbose.rm_f(g) unless File.exist?(%[gems/#{File.basename(g, '.gemspec')}.gem])" \
+	-e "}"
+
 update-bundled_gems: PHONY
 	$(Q) $(RUNRUBY) -rrubygems \
 	     $(tooldir)/update-bundled_gems.rb \
@@ -15822,6 +15833,7 @@ time.$(OBJEXT): $(top_srcdir)/internal/compar.h
 time.$(OBJEXT): $(top_srcdir)/internal/compilers.h
 time.$(OBJEXT): $(top_srcdir)/internal/fixnum.h
 time.$(OBJEXT): $(top_srcdir)/internal/gc.h
+time.$(OBJEXT): $(top_srcdir)/internal/hash.h
 time.$(OBJEXT): $(top_srcdir)/internal/numeric.h
 time.$(OBJEXT): $(top_srcdir)/internal/rational.h
 time.$(OBJEXT): $(top_srcdir)/internal/serial.h

configure.ac

@@ -65,6 +65,7 @@ AC_ARG_WITH(baseruby,
 	[
 		AC_PATH_PROG([BASERUBY], [ruby], [false])
 	])
+# BASERUBY must be >= 2.2.0. Note that `"2.2.0" > "2.2"` is true.
 AS_IF([test "$HAVE_BASERUBY" != no -a "`RUBYOPT=- $BASERUBY --disable=gems -e 'print 42 if RUBY_VERSION > "2.2"' 2>/dev/null`" = 42], [
     AS_CASE(["$build_os"], [mingw*], [
         # Can MSys shell run a command with a drive letter?

doc/contributing/building_ruby.md

@@ -18,7 +18,7 @@
     * libffi
     * libyaml
     * libexecinfo (FreeBSD)
-    * rustc - 1.58.1 or later (if you wish to build [YJIT](/doc/yjit/yjit.md))
+    * rustc - 1.58.0 or later (if you wish to build [YJIT](/doc/yjit/yjit.md))
 
 3. Checkout the CRuby source code: