(PUP-7224) Replace Rgen AST model with one generated from Pcore

This commit adds the ast.rb file (generated from the ast.pp using the
previously added rake task "gen_pcore_ast") and removes the Rgen based
predecessor. Since the new model is immutable, this has consequences
on two major areas. One is how offset, length, and the locator is handled
and the other relates to how containers are accessed.

In the old model, all expressions had a reference to their parent. This
is very hard to achieve in a tree of immutable elements (parent would
need to exist before the children that it is supposed to contain are
created). In the new model, there is instead one common element that
is referenced from every other element, and that is the Locator. Keeping
a reference to the Locator is just a cheap as keeping a reference to a
parent and it has the additional advantage of completely removing the
need for SourcePosAdapters. An element that knows its locator, offset,
and length, can respond directly to #file, #line, #pos using derived
attributes.

The lack of parent also prompted a change in the validator (it often
consults the container of elements, and at times, needs to travererse
all the way to the root). The solution for this was to let the
corresponds to the current parent chain). That stack makes it possible
for the validator to navigate the parents at all times.

Author: thallgren

Diff for: lib/puppet/info_service/class_information_service.rb

@@ -105,7 +105,6 @@ def value_as_literal(value_expr)
 
   # Extracts the source for the expression
   def extract_value_source(value_expr)
-    position = Puppet::Pops::Adapters::SourcePosAdapter.adapt(value_expr)
-    position.extract_tree_text
+    value_expr.locator.extract_tree_text(value_expr)
   end
 end

Diff for: lib/puppet/parser/ast/pops_bridge.rb

@@ -207,7 +207,7 @@ def instantiate_HostClassDefinition(o, modname)
 
     def instantiate_ResourceTypeDefinition(o, modname)
       instance = Puppet::Resource::Type.new(:definition, o.name, @context.merge(args_from_definition(o, modname, ExpressionSupportingReturn)))
-      Puppet::Pops::Loaders.register_runtime3_type(instance.name, Puppet::Pops::Adapters::SourcePosAdapter.adapt(o).to_uri)
+      Puppet::Pops::Loaders.register_runtime3_type(instance.name, o.locator.to_uri(o))
       instance
     end
 
@@ -269,7 +269,7 @@ def instantiate_FunctionDefinition(function_definition, modname)
 
       # Instantiate Function, and store it in the loader
       typed_name, f = Puppet::Pops::Loader::PuppetFunctionInstantiator.create_from_model(function_definition, loader)
-      loader.set_entry(typed_name, f, Puppet::Pops::Adapters::SourcePosAdapter.adapt(function_definition).to_uri)
+      loader.set_entry(typed_name, f, function_definition.locator.to_uri(function_definition))
 
       nil # do not want the function to inadvertently leak into 3x
     end

Diff for: lib/puppet/pops.rb

@@ -39,7 +39,6 @@ module Pops
     require 'puppet/pops/label_provider'
     require 'puppet/pops/validation'
     require 'puppet/pops/issue_reporter'
-    require 'puppet/pops/model/model'
 
     require 'puppet/pops/time/timespan'
     require 'puppet/pops/time/timestamp'
@@ -52,6 +51,7 @@ module Pops
     require 'puppet/pops/merge_strategy'
 
     module Model
+      require 'puppet/pops/model/ast'
       require 'puppet/pops/model/tree_dumper'
       require 'puppet/pops/model/ast_transformer'
       require 'puppet/pops/model/factory'
@@ -149,4 +149,6 @@ module Serialization
   require 'puppet/bindings'
   require 'puppet/functions'
   require 'puppet/loaders'
+
+  Puppet::Pops::Model.register_pcore_types
 end

Diff for: lib/puppet/pops/adapters.rb

@@ -11,96 +11,32 @@ class DocumentationAdapter < Adaptable::Adapter
     attr_accessor :documentation
   end
 
-  # A SourcePosAdapter holds a reference to  a *Positioned* object (object that has offset and length).
-  # This somewhat complex structure makes it possible to correctly refer to a source position
-  # in source that is embedded in some resource; a parser only sees the embedded snippet of source text
-  # and does not know where it was embedded. It also enables lazy evaluation of source positions (they are
-  # rarely needed - typically just when there is an error to report.
-  #
-  # @note It is relatively expensive to compute line and position on line - it is not something that
-  #   should be done for every token or model object.
-  #
-  # @see Utils#find_adapter, Utils#find_closest_positioned
-  #
-  class SourcePosAdapter < Adaptable::Adapter
-    attr_accessor :locator
-    attr_reader :adapted
-
-    def self.create_adapter(o)
-      new(o)
-    end
-
-    def initialize(o)
-      @adapted = o
+  # This class is for backward compatibility only. It's not really an adapter but it is
+  # needed for the puppetlabs-strings gem
+  # @deprecated
+  class SourcePosAdapter
+    def self.adapt(object)
+      new(object)
     end
 
-    def locator
-      # The locator is always the parent locator, all positioned objects are positioned within their
-      # parent. If a positioned object also has a locator that locator is for its children!
-      #
-      @locator ||= self.class.find_locator(@adapted.eContainer)
+    def initialize(object)
+      @object = object
     end
 
-    # @api private
-    def self.find_locator(o)
-      raise ArgumentError, 'InternalError: SourcePosAdapter for something that has no locator among parents' if o.nil?
-      found_locator = o.respond_to?(:locator) ? o.locator : nil
-      return found_locator unless found_locator.nil?
-      adapter = get(o)
-      return adapter.locator unless adapter.nil?
-      container = o.eContainer
-      container.nil? ? nil : find_locator(container)
-    end
-
-    def offset
-      @adapted.offset
+    def file
+      @object.file
     end
 
-    def length
-      @adapted.length
-    end
-
-    # Produces the line number for the given offset.
-    # @note This is an expensive operation
-    #
     def line
-      locator.line_for_offset(offset)
+      @object.line
     end
 
-    # Produces the position on the line of the given offset.
-    # @note This is an expensive operation
-    #
     def pos
-      locator.pos_on_line(offset)
+      @object.pos
     end
 
-    # Extracts the text represented by this source position (the string is obtained from the locator)
     def extract_text
-      locator.extract_text(offset, length)
-    end
-
-    def extract_tree_text
-      first = @adapted.offset
-      last = first + @adapted.length
-      @adapted.eAllContents.each do |m|
-        m_offset = m.offset
-        next if m_offset.nil?
-        first = m_offset if m_offset < first
-        m_last = m_offset + m.length
-        last = m_last if m_last > last
-      end
-      locator.extract_text(first, last-first)
-    end
-
-    # Produces an URI with path?line=n&pos=n. If origin is unknown the URI is string:?line=n&pos=n
-    def to_uri
-      f = locator.file
-      if f.nil? || f.empty?
-        f = 'string:'
-      else
-        f = Puppet::Util.path_to_uri(f).to_s
-      end
-      URI("#{f}?line=#{line.to_s}&pos=#{pos.to_s}")
+      @object.locator.extract_text(@object.offset, @object.length)
     end
   end
 
@@ -149,7 +85,7 @@ class PathsAndNameCacheAdapter < Puppet::Pops::Adaptable::Adapter
     # @param instance
     # @api private
     def self.loader_name_by_source(environment, instance, file)
-      file = find_file(instance) if file.nil?
+      file = instance.file if file.nil?
       return nil if file.nil?
       pn_adapter = PathsAndNameCacheAdapter.adapt(environment) do |a|
         a.paths ||= environment.modulepath.map { |p| Pathname.new(p) }
@@ -181,12 +117,6 @@ def self.find_module_for_dir(environment, paths, dir)
       end
       nil
     end
-
-    # @api private
-    def self.find_file(instance)
-      source_pos = Utils.find_closest_positioned(instance)
-      source_pos.nil? ? nil : source_pos.locator.file
-    end
   end
 end
 end

Diff for: lib/puppet/pops/containment.rb

@@ -84,19 +84,19 @@ def all_containment_getters(element)
     end
 
     def collect_getters(eclass, containments)
-        eclass.eStructuralFeatures.select {|r| r.is_a?(RGen::ECore::EReference) && r.containment}.each do |r|
-          n = r.name
-          containments << :"get#{n[0..0].upcase + ( n[1..-1] || "" )}"
+      eclass.eStructuralFeatures.select {|r| r.is_a?(RGen::ECore::EReference) && r.containment}.each do |r|
+        n = r.name
+        containments << :"get#{n[0..0].upcase + ( n[1..-1] || "" )}"
+      end
+      eclass.eSuperTypes.each do |t|
+        if cached = @@cache[ t.instanceClass ]
+          containments.concat(cached)
+        else
+          super_containments = []
+          collect_getters(t, super_containments)
+          @@cache[ t.instanceClass ] = super_containments
+          containments.concat(super_containments)
         end
-        eclass.eSuperTypes.each do |t|
-          if cached = @@cache[ t.instanceClass ]
-            containments.concat(cached)
-          else
-            super_containments = []
-            collect_getters(t, super_containments)
-            @@cache[ t.instanceClass ] = super_containments
-            containments.concat(super_containments)
-          end
       end
     end
 

Diff for: lib/puppet/pops/evaluator/collector_transformer.rb

@@ -23,16 +23,11 @@ def transform(o, scope)
 
     fail "Resource type #{type} doesn't exist" unless resource_type
 
-    adapter = Adapters::SourcePosAdapter.adapt(o)
-    line_num = adapter.line
-    position = adapter.pos
-    file_path = adapter.locator.file
-
     if !o.operations.empty?
       overrides = {
         :parameters => o.operations.map{ |x| @@evaluator.evaluate(x, scope)}.flatten,
-        :file       => file_path,
-        :line       => [line_num, position],
+        :file       => o.file,
+        :line       => [o.line, o.pos],
         :source     => scope.source,
         :scope      => scope
       }

Diff for: lib/puppet/pops/evaluator/evaluator_impl.rb

@@ -1,4 +1,3 @@
-require 'rgen/ecore/ecore'
 require 'puppet/parser/scope'
 require 'puppet/pops/evaluator/compare_operator'
 require 'puppet/pops/evaluator/relationship_operator'
@@ -376,7 +375,7 @@ def eval_ArithmeticExpression(o, scope)
     right = evaluate(o.right_expr, scope)
 
     begin
-      result = calculate(left, right, o.operator, o.left_expr, o.right_expr, scope)
+      result = calculate(left, right, o, scope)
     rescue ArgumentError => e
       fail(Issues::RUNTIME_ERROR, o, {:detail => e.message}, e)
     end
@@ -386,11 +385,13 @@ def eval_ArithmeticExpression(o, scope)
 
   # Handles binary expression where lhs and rhs are array/hash or numeric and operator is +, - , *, % / << >>
   #
-  def calculate(left, right, operator, left_o, right_o, scope)
+  def calculate(left, right, bin_expr, scope)
+    operator = bin_expr.operator
     unless ARITHMETIC_OPERATORS.include?(operator)
-      fail(Issues::UNSUPPORTED_OPERATOR, left_o.eContainer, {:operator => o.operator})
+      fail(Issues::UNSUPPORTED_OPERATOR, bin_expr, {:operator => operator})
     end
 
+    left_o = bin_expr.left_expr
     if (left.is_a?(Array) || left.is_a?(Hash)) && COLLECTION_OPERATORS.include?(operator)
       # Handle operation on collections
       case operator
@@ -407,7 +408,7 @@ def calculate(left, right, operator, left_o, right_o, scope)
     else
       # Handle operation on numeric
       left = coerce_numeric(left, left_o, scope)
-      right = coerce_numeric(right, right_o, scope)
+      right = coerce_numeric(right, bin_expr.right_expr, scope)
       begin
         if operator == '%' && (left.is_a?(Float) || right.is_a?(Float))
           # Deny users the fun of seeing severe rounding errors and confusing results
@@ -430,7 +431,7 @@ def calculate(left, right, operator, left_o, right_o, scope)
       rescue NoMethodError => e
         fail(Issues::OPERATOR_NOT_APPLICABLE, left_o, {:operator => operator, :left_value => left})
       rescue ZeroDivisionError => e
-        fail(Issues::DIV_BY_ZERO, right_o)
+        fail(Issues::DIV_BY_ZERO, bin_expr.right_expr)
       end
       case result
       when Float
@@ -439,7 +440,7 @@ def calculate(left, right, operator, left_o, right_o, scope)
         end
       when Integer
         if result < MIN_INTEGER || result > MAX_INTEGER
-          fail(Issues::NUMERIC_OVERFLOW, left_o.eContainer, {:value => result})
+          fail(Issues::NUMERIC_OVERFLOW, bin_expr, {:value => result})
         end
       end
       result

Diff for: lib/puppet/pops/evaluator/external_syntax_support.rb

@@ -20,8 +20,7 @@ def assert_external_syntax(scope, result, syntax, reference_expr)
     # Call checker and give it the location information from the expression
     # (as opposed to where the heredoc tag is (somewhere on the line above)).
     acceptor = Puppet::Pops::Validation::Acceptor.new()
-    source_pos = Puppet::Pops::Utils.find_closest_positioned(reference_expr)
-    checker.check(result, syntax, acceptor, source_pos)
+    checker.check(result, syntax, acceptor, reference_expr)
 
     if acceptor.error_count > 0
       checker_message = "Invalid produced text having syntax: '#{syntax}'."

Diff for: lib/puppet/pops/evaluator/runtime3_support.rb

@@ -439,21 +439,7 @@ def is_boolean? x
   end
 
   def extract_file_line(o)
-    positioned = find_closest_with_offset(o)
-    unless positioned.nil?
-      locator = Adapters::SourcePosAdapter.find_locator(positioned)
-      return [locator.file, locator.line_for_offset(positioned.offset)] unless locator.nil?
-    end
-    [nil, -1]
-  end
-
-  def find_closest_with_offset(o)
-    if o.offset.nil?
-      c = o.eContainer
-      c.nil? ? nil : find_closest_with_offset(c)
-    else
-      o
-    end
+    o.is_a?(Model::Positioned) ? [o.file, o.line] : [nil, -1]
   end
 
   # Creates a diagnostic producer

Diff for: lib/puppet/pops/loader/type_definition_instantiator.rb

@@ -51,7 +51,7 @@ def self.create_from_model(type_definition, loader)
     loader.set_entry(
       typed_name,
       type,
-      Adapters::SourcePosAdapter.adapt(type_definition).to_uri)
+      type_definition.locator.to_uri(type_definition))
     type
   end