From 7a7e3d9b3136c19d3b881029814512bfd0de07ea Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 20 Feb 2020 13:43:48 +0100 Subject: [PATCH] Ability to only run one test section in file You can now pass -s "Test section name" to the rest-api-spec test runner to only run a single test inside a file. Before you could only filter to a file as whole. This also makes sure args with spaces are quoted when passing down from our build tooling (super rudimentary) We now emit Skip messages during the run so we can see the test itself skipped. Because in our current flow we run the test setup before evaluating the `skip` block its hard to see nothing happened because things are happening (setup requests) Lastly the proxy detection now includes `xpack` test suite. `mitmproxy -k` will work if we reroute https traffic to it. --- build/scripts/Commandline.fs | 5 +++++ tests/Tests.YamlRunner/Commands.fs | 4 ++-- tests/Tests.YamlRunner/Program.fs | 13 ++++++++----- tests/Tests.YamlRunner/TestsRunner.fs | 14 ++++++++++---- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/build/scripts/Commandline.fs b/build/scripts/Commandline.fs index fd102d3ab7d..822e16b7c05 100644 --- a/build/scripts/Commandline.fs +++ b/build/scripts/Commandline.fs @@ -127,6 +127,11 @@ Execution hints can be provided anywhere on the command line not (x.StartsWith("random:")) && not (x.StartsWith("docs:")) && not (x.StartsWith("ref:"))) + |> List.map(fun (s:string) -> + let containsSpace = s.Contains(" ") + match s with | x when containsSpace -> sprintf "\"%s\"" x | s -> s + ) + let target = match (filteredArgs |> List.tryHead) with | Some t -> t.Replace("-one", "") diff --git a/tests/Tests.YamlRunner/Commands.fs b/tests/Tests.YamlRunner/Commands.fs index 2118940d2b4..3586a921bfa 100644 --- a/tests/Tests.YamlRunner/Commands.fs +++ b/tests/Tests.YamlRunner/Commands.fs @@ -35,7 +35,7 @@ let ReadTests (tests:LocateResults list) = tests |> List.map (fun t -> { Folder= t.Folder; Files = readPaths t.Paths}) -let RunTests (tests:YamlTestFolder list) client version namedSuite = async { +let RunTests (tests:YamlTestFolder list) client version namedSuite sectionFilter = async { do! Async.SwitchToNewThread() let f = tests.Length @@ -45,7 +45,7 @@ let RunTests (tests:YamlTestFolder list) client version namedSuite = async { runner.GlobalSetup() let a (i, v) = async { let mainMessage = sprintf "[%i/%i] Folders : %s | " (i+1) f v.Folder - let! op = runner.RunTestsInFolder mainMessage v + let! op = runner.RunTestsInFolder mainMessage v sectionFilter return v, op |> Seq.toList } let x = diff --git a/tests/Tests.YamlRunner/Program.fs b/tests/Tests.YamlRunner/Program.fs index aeb62ee9596..ebe70037ed6 100644 --- a/tests/Tests.YamlRunner/Program.fs +++ b/tests/Tests.YamlRunner/Program.fs @@ -12,6 +12,7 @@ type Arguments = | [] NamedSuite of TestSuite | []Folder of string | []TestFile of string + | []TestSection of string | []Endpoint of string | []Revision of string | []JUnitOutputFile of string @@ -24,6 +25,7 @@ type Arguments = | Revision _ -> "The git revision to reference (commit/branch/tag). defaults to `master`" | Folder _ -> "Only run tests in this folder" | TestFile _ -> "Only run tests starting with this filename" + | TestSection _ -> "Only run test with this name (best used in conjuction with -t)" | Endpoint _ -> "The elasticsearch endpoint to run tests against" | JUnitOutputFile _ -> "The path and file name to use for the junit xml output, defaults to a random tmp filename" | Profile _ -> "Print out process id and wait for confirmation to kick off the tests" @@ -33,7 +35,7 @@ let private runningProxy = runningMitmProxy || Process.GetProcessesByName("fiddl let private defaultEndpoint namedSuite = let host = match (runningProxy, namedSuite) with - | (true, Oss) -> "ipv.fiddler" + | (true, _) -> "ipv.fiddler" | _ -> "localhost" let https = match namedSuite with | XPack -> "s" | _ -> "" sprintf "http%s://%s:9200" https host; @@ -54,7 +56,7 @@ let private createClient endpoint namedSuite = // proxy let proxySettings = match (runningMitmProxy, namedSuite) with - | (true, Oss) -> settings.Proxy(Uri("http://ipv4.fiddler:8080"), String(null), String(null)) + | (true, _) -> settings.Proxy(Uri("http://ipv4.fiddler:8080"), String(null), String(null)) | _ -> settings // auth let authSettings = @@ -104,6 +106,7 @@ let runMain (parsed:ParseResults) = async { let namedSuite = parsed.TryGetResult NamedSuite |> Option.defaultValue Oss let directory = parsed.TryGetResult Folder //|> Option.defaultValue "indices.create" |> Some let file = parsed.TryGetResult TestFile //|> Option.defaultValue "10_basic.yml" |> Some + let section = parsed.TryGetResult TestSection //|> Option.defaultValue "10_basic.yml" |> Some let endpoint = parsed.TryGetResult Endpoint |> Option.defaultValue (defaultEndpoint namedSuite) let profile = parsed.TryGetResult Profile |> Option.defaultValue false let passedRevision = parsed.TryGetResult Revision @@ -115,13 +118,13 @@ let runMain (parsed:ParseResults) = async { printfn "Found version %s downloading specs from: %s" version revision - let! locateResults = Commands.LocateTests namedSuite revision directory file - let readResults = Commands.ReadTests locateResults + let! locateResults = Commands.LocateTests namedSuite revision directory file + let readResults = Commands.ReadTests locateResults if profile then printf "Waiting for profiler to attach to pid: %O" <| Process.GetCurrentProcess().Id Console.ReadKey() |> ignore - let! runResults = Commands.RunTests readResults client version namedSuite + let! runResults = Commands.RunTests readResults client version namedSuite section let summary = Commands.ExportTests runResults outputFile Commands.PrettyPrintResults outputFile diff --git a/tests/Tests.YamlRunner/TestsRunner.fs b/tests/Tests.YamlRunner/TestsRunner.fs index 686ad473a32..57a350027c2 100644 --- a/tests/Tests.YamlRunner/TestsRunner.fs +++ b/tests/Tests.YamlRunner/TestsRunner.fs @@ -54,7 +54,7 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite }) (m, executedOperations) - member private this.RunTestFile subProgressbar (file:YamlTestDocument) = async { + member private this.RunTestFile subProgressbar (file:YamlTestDocument) sectionFilter = async { let m section ops = this.CreateOperations section file.FileInfo ops subProgressbar let bootstrap section operations = let ops = operations |> Option.map (m section) |> Option.toList |> List.collect (fun (s, ops) -> ops) @@ -65,6 +65,10 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite let sections = file.Tests |> List.map (fun s -> s.Operations |> m s.Name) + |> List.filter(fun s -> + let (name, _) = s + match sectionFilter with | Some s when s <> name -> false | _ -> true + ) |> List.collect (fun s -> let (name, ops) = s [(name, setup @ ops)] @@ -88,7 +92,9 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite match r with | Succeeded context -> Some (r, tl) | NotSkipped context -> Some (r, tl) - | Skipped (context, reason) -> Some (r, []) + | Skipped (context, reason) -> + subProgressbar.WriteLine <| sprintf "%s: %s " r.Name (r.Context.Operation.Log()) + Some (r, []) | Failed context -> Some (r, []) | [] -> None ) @@ -142,7 +148,7 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite client.Indices.Refresh("_all") |> ignore - member this.RunTestsInFolder mainMessage (folder:YamlTestFolder) = async { + member this.RunTestsInFolder mainMessage (folder:YamlTestFolder) sectionFilter = async { let l = folder.Files.Length let run (i, document) = async { let file = sprintf "%s/%s" document.FileInfo.Directory.Name document.FileInfo.Name @@ -151,7 +157,7 @@ type TestRunner(client:IElasticLowLevelClient, version: string, suite: TestSuite let message = sprintf "Inspecting file for sections" use p = progress.Spawn(0, message, barOptions) - let! result = this.RunTestFile p document + let! result = this.RunTestFile p document sectionFilter return document, result }