Ability to only run one test section in file in yaml test runner (#4404)

You can now pass -s "Test section name" to the rest-api-spec test runner
to only run a single test inside a file. Before you could only filter to
a file as whole.

This also makes sure args with spaces are quoted when passing down from
our build tooling (super rudimentary)

We now emit Skip messages during the run so we can see the test itself
skipped. Because in our current flow we run the test setup before
evaluating the `skip` block its hard to see nothing happened because
things are happening (setup requests)

Lastly the proxy detection now includes `xpack` test suite.

`mitmproxy -k` will work if we reroute https traffic to it.

Author: Mpdreamz

build/scripts/Commandline.fs

@@ -127,6 +127,11 @@ Execution hints can be provided anywhere on the command line
                not (x.StartsWith("random:")) && 
                not (x.StartsWith("docs:")) &&
                not (x.StartsWith("ref:")))
+            |> List.map(fun (s:string) ->
+                let containsSpace = s.Contains(" ")
+                match s with | x when containsSpace -> sprintf "\"%s\"" x | s -> s
+            )
+            
         let target = 
             match (filteredArgs |> List.tryHead) with
             | Some t -> t.Replace("-one", "")

tests/Tests.YamlRunner/Commands.fs

@@ -35,7 +35,7 @@ let ReadTests (tests:LocateResults list) =
 
     tests |> List.map (fun t -> { Folder= t.Folder; Files = readPaths t.Paths})
 
-let RunTests (tests:YamlTestFolder list) client version namedSuite = async {
+let RunTests (tests:YamlTestFolder list) client version namedSuite sectionFilter = async {
     do! Async.SwitchToNewThread()
 
     let f = tests.Length
@@ -45,7 +45,7 @@ let RunTests (tests:YamlTestFolder list) client version namedSuite = async {
     runner.GlobalSetup() 
     let a (i, v) = async {
         let mainMessage = sprintf "[%i/%i] Folders : %s | " (i+1) f v.Folder
-        let! op = runner.RunTestsInFolder mainMessage v
+        let! op = runner.RunTestsInFolder mainMessage v sectionFilter
         return v, op |> Seq.toList
     }
     let x =

tests/Tests.YamlRunner/Program.fs

@@ -12,6 +12,7 @@ type Arguments =
     | [<First; MainCommand; CliPrefix(CliPrefix.None)>] NamedSuite of TestSuite
     | [<AltCommandLine("-f")>]Folder of string
     | [<AltCommandLine("-t")>]TestFile of string
+    | [<AltCommandLine("-s")>]TestSection of string
     | [<AltCommandLine("-e")>]Endpoint of string
     | [<AltCommandLine("-r")>]Revision of string
     | [<AltCommandLine("-o")>]JUnitOutputFile of string
@@ -24,6 +25,7 @@ type Arguments =
             | Revision _ -> "The git revision to reference (commit/branch/tag). defaults to `master`"
             | Folder _ -> "Only run tests in this folder"
             | TestFile _ -> "Only run tests starting with this filename"
+            | TestSection _ -> "Only run test with this name (best used in conjuction with -t)"
             | Endpoint _ -> "The elasticsearch endpoint to run tests against"
             | JUnitOutputFile _ -> "The path and file name to use for the junit xml output, defaults to a random tmp filename"
             | Profile _ -> "Print out process id and wait for confirmation to kick off the tests"
@@ -33,7 +35,7 @@ let private runningProxy = runningMitmProxy || Process.GetProcessesByName("fiddl
 let private defaultEndpoint namedSuite = 
     let host = 
         match (runningProxy, namedSuite) with
-        | (true, Oss) -> "ipv.fiddler"
+        | (true, _) -> "ipv.fiddler"
         | _ -> "localhost"
     let https = match namedSuite with | XPack -> "s" | _ -> ""
     sprintf "http%s://%s:9200" https host;
@@ -54,7 +56,7 @@ let private createClient endpoint namedSuite =
     // proxy 
     let proxySettings =
         match (runningMitmProxy, namedSuite) with
-        | (true, Oss) -> settings.Proxy(Uri("http://ipv4.fiddler:8080"), String(null), String(null))
+        | (true, _) -> settings.Proxy(Uri("http://ipv4.fiddler:8080"), String(null), String(null))
         | _ -> settings
     // auth
     let authSettings =
@@ -104,6 +106,7 @@ let runMain (parsed:ParseResults<Arguments>) = async {
     let namedSuite = parsed.TryGetResult NamedSuite |> Option.defaultValue Oss
     let directory = parsed.TryGetResult Folder //|> Option.defaultValue "indices.create" |> Some
     let file = parsed.TryGetResult TestFile //|> Option.defaultValue "10_basic.yml" |> Some
+    let section = parsed.TryGetResult TestSection //|> Option.defaultValue "10_basic.yml" |> Some
     let endpoint = parsed.TryGetResult Endpoint |> Option.defaultValue (defaultEndpoint namedSuite)
     let profile = parsed.TryGetResult Profile |> Option.defaultValue false
     let passedRevision = parsed.TryGetResult Revision
@@ -115,13 +118,13 @@ let runMain (parsed:ParseResults<Arguments>) = async {
 
     printfn "Found version %s downloading specs from: %s" version revision
 
-    let! locateResults = Commands.LocateTests namedSuite revision directory file
-    let readResults = Commands.ReadTests locateResults
+    let! locateResults = Commands.LocateTests namedSuite revision directory file 
+    let readResults = Commands.ReadTests locateResults 
     if profile then
         printf "Waiting for profiler to attach to pid: %O" <| Process.GetCurrentProcess().Id
         Console.ReadKey() |> ignore
 
-    let! runResults = Commands.RunTests readResults client version namedSuite
+    let! runResults = Commands.RunTests readResults client version namedSuite section
     let summary = Commands.ExportTests runResults outputFile
 
     Commands.PrettyPrintResults outputFile

tests/Tests.YamlRunner/TestsRunner.fs

@@ -54,7 +54,7 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite
             })
         (m, executedOperations)
 
-    member private this.RunTestFile subProgressbar (file:YamlTestDocument) = async {
+    member private this.RunTestFile subProgressbar (file:YamlTestDocument) sectionFilter = async {
         let m section ops = this.CreateOperations section file.FileInfo ops subProgressbar
         let bootstrap section operations =
             let ops = operations |> Option.map (m section) |> Option.toList |> List.collect (fun (s, ops) -> ops)
@@ -65,6 +65,10 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite
         let sections =
             file.Tests
             |> List.map (fun s -> s.Operations |> m s.Name)
+            |> List.filter(fun s ->
+                let (name, _) = s
+                match sectionFilter with | Some s when s <> name -> false | _ -> true
+            )
             |> List.collect (fun s ->
                 let (name, ops) = s
                 [(name, setup @ ops)]
@@ -88,7 +92,9 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite
                         match r with
                         | Succeeded context -> Some (r, tl)
                         | NotSkipped context -> Some (r, tl)
-                        | Skipped (context, reason) -> Some (r, [])
+                        | Skipped (context, reason) ->
+                            subProgressbar.WriteLine <| sprintf "%s: %s " r.Name (r.Context.Operation.Log())
+                            Some (r, [])
                         | Failed context -> Some (r, [])
                     | [] -> None
                 )
@@ -142,7 +148,7 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite
             client.Indices.Refresh<VoidResponse>("_all") |> ignore
 
 
-    member this.RunTestsInFolder mainMessage (folder:YamlTestFolder) = async {
+    member this.RunTestsInFolder mainMessage (folder:YamlTestFolder) sectionFilter = async {
         let l = folder.Files.Length
         let run (i, document) = async {
             let file = sprintf "%s/%s" document.FileInfo.Directory.Name document.FileInfo.Name
@@ -151,7 +157,7 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite
             let message = sprintf "Inspecting file for sections" 
             use p = progress.Spawn(0, message, barOptions)
 
-            let! result = this.RunTestFile p document
+            let! result = this.RunTestFile p document sectionFilter
 
             return document, result
         }