[backport] backport scala#234 and scala#327 to 1.1.x branch

fixes scala#233 on 1.1.x branch

Author: soya

Diff for: build.sbt

@@ -42,7 +42,15 @@ lazy val parserCombinators = crossProject(JVMPlatform, JSPlatform, NativePlatfor
           case _             => file(dir.getPath ++ "-2.11-2.12")
         }
       }
-    }
+    },
+    mimaBinaryIssueFilters ++= {
+      import com.typesafe.tools.mima.core._, ProblemFilters._
+      Seq(
+        // these are safe to exclude because they're `private[combinator]`
+        exclude[ReversedMissingMethodProblem]("scala.util.parsing.combinator.Parsers.Success"),
+        exclude[ReversedMissingMethodProblem]("scala.util.parsing.combinator.Parsers.selectLastFailure"),
+      )
+    },
   )
   .jvmSettings(
     OsgiKeys.exportPackage := Seq(s"scala.util.parsing.*;version=${version.value}"),

Diff for: shared/src/main/scala/scala/util/parsing/combinator/Parsers.scala

@@ -133,13 +133,21 @@ trait Parsers {
    *  @param next   The parser's remaining input
    */
   case class Success[+T](result: T, override val next: Input) extends ParseResult[T] {
-    def map[U](f: T => U) = Success(f(result), next)
-    def mapPartial[U](f: PartialFunction[T, U], error: T => String): ParseResult[U]
-       = if(f.isDefinedAt(result)) Success(f(result), next)
-         else Failure(error(result), next)
+    def lastFailure: Option[Failure] = None
 
-    def flatMapWithNext[U](f: T => Input => ParseResult[U]): ParseResult[U]
-      = f(result)(next)
+    def map[U](f: T => U) = Success(f(result), next, lastFailure)
+
+    def mapPartial[U](f: PartialFunction[T, U], error: T => String): ParseResult[U] =
+      if(f.isDefinedAt(result)) Success(f(result), next, lastFailure)
+      else Failure(error(result), next)
+
+    def flatMapWithNext[U](f: T => Input => ParseResult[U]): ParseResult[U] = f(result)(next) match {
+      case s @ Success(result, rest) =>
+        val failure = selectLastFailure(this.lastFailure, s.lastFailure)
+        Success(result, rest, failure)
+      case f: Failure => selectLastFailure(Some(f), lastFailure).get
+      case e: Error => e
+    }
 
     def filterWithError(p: T => Boolean, error: T => String, position: Input): ParseResult[T] =
       if (p(result)) this
@@ -188,10 +196,16 @@ trait Parsers {
     /** The toString method of a Failure yields an error message. */
     override def toString = "["+next.pos+"] failure: "+msg+"\n\n"+next.pos.longString
 
-    def append[U >: Nothing](a: => ParseResult[U]): ParseResult[U] = { val alt = a; alt match {
-      case Success(_, _) => alt
-      case ns: NoSuccess => if (alt.next.pos < next.pos) this else alt
-    }}
+    def append[U >: Nothing](a: => ParseResult[U]): ParseResult[U] = {
+      val alt = a
+
+      alt match {
+        case s @ Success(result, rest) =>
+          val failure = selectLastFailure(Some(this), s.lastFailure)
+          Success(result, rest, failure)
+        case ns: NoSuccess => if (alt.next.pos < next.pos) this else alt
+      }
+    }
   }
 
   /** The fatal failure case of ParseResult: contains an error-message and
@@ -210,6 +224,19 @@ trait Parsers {
   def Parser[T](f: Input => ParseResult[T]): Parser[T]
     = new Parser[T]{ def apply(in: Input) = f(in) }
 
+  private[combinator] def Success[U](res: U, next: Input, failure: Option[Failure]): Success[U] =
+    new Success(res, next) { override val lastFailure: Option[Failure] = failure }
+
+  private[combinator] def selectLastFailure(failure0: Option[Failure], failure1: Option[Failure]): Option[Failure] =
+    (failure0, failure1) match {
+      case (Some(f0), Some(f1)) =>
+        if(f0.next.pos < f1.next.pos) Some(f1)
+        else                          Some(f0)
+      case (Some(f0), _) => Some(f0)
+      case (_, Some(f1)) => Some(f1)
+      case _ => None
+    }
+
   def OnceParser[T](f: Input => ParseResult[T]): Parser[T] with OnceParser[T]
     = new Parser[T] with OnceParser[T] { def apply(in: Input) = f(in) }
 
@@ -630,7 +657,7 @@ trait Parsers {
    */
   def acceptIf(p: Elem => Boolean)(err: Elem => String): Parser[Elem] = Parser { in =>
     if (in.atEnd) Failure("end of input", in)
-    else if (p(in.first)) Success(in.first, in.rest)
+    else if (p(in.first)) Success(in.first, in.rest, None)
     else Failure(err(in.first), in)
   }
 
@@ -649,7 +676,7 @@ trait Parsers {
    */
   def acceptMatch[U](expected: String, f: PartialFunction[Elem, U]): Parser[U] = Parser{ in =>
     if (in.atEnd) Failure("end of input", in)
-    else if (f.isDefinedAt(in.first)) Success(f(in.first), in.rest)
+    else if (f.isDefinedAt(in.first)) Success(f(in.first), in.rest, None)
     else Failure(expected+" expected", in)
   }
 
@@ -682,7 +709,7 @@ trait Parsers {
    * @param v The result for the parser
    * @return A parser that always succeeds, with the given result `v`
    */
-  def success[T](v: T) = Parser{ in => Success(v, in) }
+  def success[T](v: T) = Parser{ in => Success(v, in, None) }
 
   /** A helper method that turns a `Parser` into one that will
    *  print debugging information to stdout before and after
@@ -747,19 +774,24 @@ trait Parsers {
     lazy val p = p0 // lazy argument
     val elems = new ListBuffer[T]
 
-    def continue(in: Input): ParseResult[List[T]] = {
+    def continue(in: Input, failure: Option[Failure]): ParseResult[List[T]] = {
       val p0 = p    // avoid repeatedly re-evaluating by-name parser
-      @tailrec def applyp(in0: Input): ParseResult[List[T]] = p0(in0) match {
-        case Success(x, rest) => elems += x ; applyp(rest)
+      @tailrec def applyp(in0: Input, failure: Option[Failure]): ParseResult[List[T]] = p0(in0) match {
+        case s @ Success(x, rest) =>
+          val selectedFailure = selectLastFailure(s.lastFailure, failure)
+          elems += x
+          applyp(rest, selectedFailure)
         case e @ Error(_, _)  => e  // still have to propagate error
-        case _                => Success(elems.toList, in0)
+        case f: Failure =>
+          val selectedFailure = selectLastFailure(failure, Some(f))
+          Success(elems.toList, in0, selectedFailure)
       }
 
-      applyp(in)
+      applyp(in, failure)
     }
 
     first(in) match {
-      case Success(x, rest) => elems += x ; continue(rest)
+      case s @ Success(x, rest) => elems += x ; continue(rest, s.lastFailure)
       case ns: NoSuccess    => ns
     }
   }
@@ -779,14 +811,14 @@ trait Parsers {
       val elems = new ListBuffer[T]
       val p0 = p    // avoid repeatedly re-evaluating by-name parser
 
-      @tailrec def applyp(in0: Input): ParseResult[List[T]] =
-        if (elems.length == num) Success(elems.toList, in0)
+      @tailrec def applyp(in0: Input, failure: Option[Failure]): ParseResult[List[T]] =
+        if (elems.length == num) Success(elems.toList, in0, failure)
         else p0(in0) match {
-          case Success(x, rest) => elems += x ; applyp(rest)
+          case s @ Success(x, rest) => elems += x ; applyp(rest, s.lastFailure)
           case ns: NoSuccess    => ns
         }
 
-      applyp(in)
+      applyp(in, None)
     }
 
   /** A parser generator for non-empty repetitions.
@@ -868,7 +900,7 @@ trait Parsers {
   def not[T](p: => Parser[T]): Parser[Unit] = Parser { in =>
     p(in) match {
       case Success(_, _)  => Failure("Expected failure", in)
-      case _              => Success((), in)
+      case _              => Success((), in, None)
     }
   }
 
@@ -882,7 +914,7 @@ trait Parsers {
    */
   def guard[T](p: => Parser[T]): Parser[T] = Parser { in =>
     p(in) match{
-      case s@ Success(s1,_) => Success(s1, in)
+      case s@ Success(s1,_) => Success(s1, in, s.lastFailure)
       case e => e
     }
   }
@@ -897,7 +929,7 @@ trait Parsers {
    */
   def positioned[T <: Positional](p: => Parser[T]): Parser[T] = Parser { in =>
     p(in) match {
-      case Success(t, in1) => Success(if (t.pos == NoPosition) t setPos in.pos else t, in1)
+      case s @ Success(t, in1) => Success(if (t.pos == NoPosition) t setPos in.pos else t, in1, s.lastFailure)
       case ns: NoSuccess => ns
     }
   }
@@ -915,7 +947,10 @@ trait Parsers {
     def apply(in: Input) = p(in) match {
       case s @ Success(out, in1) =>
         if (in1.atEnd) s
-        else Failure("end of input expected", in1)
+        else s.lastFailure match {
+          case Some(failure) => failure
+          case _ => Failure("end of input expected", in1)
+        }
       case ns => ns
     }
   }

Diff for: shared/src/main/scala/scala/util/parsing/combinator/RegexParsers.scala

@@ -94,7 +94,7 @@ trait RegexParsers extends Parsers {
         j += 1
       }
       if (i == s.length)
-        Success(source.subSequence(start, j).toString, in.drop(j - offset))
+        Success(source.subSequence(start, j).toString, in.drop(j - offset), None)
       else  {
         val found = if (start == source.length()) "end of source" else "'"+source.charAt(start)+"'"
         Failure("'"+s+"' expected but "+found+" found", in.drop(start - offset))
@@ -111,7 +111,8 @@ trait RegexParsers extends Parsers {
       (r findPrefixMatchOf (new SubSequence(source, start))) match {
         case Some(matched) =>
           Success(source.subSequence(start, start + matched.end).toString,
-                  in.drop(start + matched.end - offset))
+                  in.drop(start + matched.end - offset),
+                  None)
         case None =>
           val found = if (start == source.length()) "end of source" else "'"+source.charAt(start)+"'"
           Failure("string matching regex '"+r+"' expected but "+found+" found", in.drop(start - offset))

Diff for: shared/src/test/scala/scala/util/parsing/combinator/JavaTokenParsersTest.scala

@@ -45,7 +45,7 @@ class JavaTokenParsersTest {
     def parseFailure(s: String, errorColPos: Int): Unit = {
       val parseResult = parseAll(ident, s)
       parseResult match {
-        case Failure(_, next) =>
+        case Failure(msg, next) =>
           val pos = next.pos
           assertEquals(1, pos.line)
           assertEquals(errorColPos, pos.column)
@@ -65,7 +65,7 @@ class JavaTokenParsersTest {
     parseFailure("with-s", 5)
     // we♥scala
     parseFailure("we\u2665scala", 3)
-    parseFailure("with space", 6)
+    parseFailure("with space", 5)
   }
 
   @Test
@@ -87,7 +87,7 @@ class JavaTokenParsersTest {
       case e @ Failure(message, next) =>
         assertEquals(next.pos.line, 1)
         assertEquals(next.pos.column, 7)
-        assert(message.endsWith(s"end of input expected"))
+        assert(message.endsWith("string matching regex '(?i)AND' expected but 's' found"))
       case _ => sys.error(parseResult1.toString)
     }
 
@@ -111,7 +111,7 @@ class JavaTokenParsersTest {
       case Failure(message, next) =>
         assertEquals(next.pos.line, 1)
         assertEquals(next.pos.column, 1)
-        assert(message.endsWith(s"end of input expected"))
+        assert(message.endsWith(s"identifier expected but '-' found"))
       case _ => sys.error(parseResult.toString)
     }
 

Diff for: shared/src/test/scala/scala/util/parsing/combinator/RegexParsersTest.scala

@@ -13,7 +13,7 @@
 package scala.util.parsing.combinator
 
 import org.junit.Test
-import org.junit.Assert.assertEquals
+import org.junit.Assert.{ assertEquals, assertTrue }
 
 class RegexParsersTest {
   @Test
@@ -112,4 +112,71 @@ class RegexParsersTest {
     val success = parseAll(twoWords, "first second").asInstanceOf[Success[(String, String)]]
     assertEquals(("second", "first"), success.get)
   }
+
+  @Test
+  def hierarchicalRepSuccess: Unit = {
+    case class Node(a: String, b: String)
+
+    object parser extends RegexParsers {
+      def top: Parser[List[List[Node]]] = rep(nodes)
+      def nodes: Parser[List[Node]] = "{" ~> rep(node) <~ "}"
+      def node: Parser[Node] = "[a-z]+".r ~ ":" ~ "[a-z]+".r ^^ { case a ~ _ ~ b => Node(a, b) }
+    }
+
+    import parser._
+
+    val success0 = parseAll(top, "{ a : b c : d}").get
+    assertEquals(List(List(Node("a", "b"), Node("c", "d"))), success0)
+    val success1 = parseAll(top, "{ a : b } { c : d }").get
+    assertEquals(List(List(Node("a", "b")), List(Node("c", "d"))), success1)
+    val success2 = parseAll(top, "{} {}").get
+    assertEquals(List(List(), List()), success2)
+    val success3 = parseAll(top, "").get
+    assertEquals(List(), success3)
+  }
+
+  @Test
+  def hierarchicalRepFailure: Unit = {
+    case class Node(a: String, b: String)
+
+    object parser extends RegexParsers {
+      def top: Parser[List[List[Node]]] = rep(nodes)
+      def nodes: Parser[List[Node]] = "{" ~> rep(node) <~ "}"
+      def node: Parser[Node] = "[a-z]+".r ~ ":" ~ "[a-z]+".r ^^ { case a ~ _ ~ b => Node(a, b) }
+    }
+
+    def test(src: String, expect: String, column: Int): Unit = {
+      import parser._
+      val result = parseAll(top, src)
+      result match {
+        case Failure(msg, next) =>
+          assertEquals(column, next.pos.column)
+          assertEquals(expect, msg)
+        case _ =>
+          sys.error(result.toString)
+      }
+    }
+
+    test("{ a : b c : }", "string matching regex '[a-z]+' expected but '}' found", 13)
+    test("{", "'}' expected but end of source found", 2)
+  }
+
+  @Test
+  def ifElseTest: Unit = {
+    object parser extends RegexParsers {
+      def top: Parser[List[Unit]] = rep(ifelse)
+      def ifelse: Parser[Unit] = "IF" ~ condition ~ "THEN" ~ "1"~ "END" ^^ { _ => }
+      def condition: Parser[String] = "TRUE" | "FALSE"
+    }
+
+    import parser._
+    val res = parseAll(top, "IF FALSE THEN 1 IF TRUE THEN 1 END")
+    res match {
+      case Failure(msg, next) =>
+        assertEquals(17, next.pos.column)
+        assertEquals("'END' expected but 'I' found", msg)
+      case _ =>
+        sys.error(res.toString)
+    }
+  }
 }