support reading stdout/stderr from streams

It can be useful to let the user run the test command(s) itself, for example
when complex shell scripts are involved which need to invoke `go test` multiple
times.

With -stdin, the `go test` stdout is expected on stdin of gotestsum, so it can
be used in a pipe. To detect abnormal termination of the test commands, bash
with "set -o pipefail" should be used. Beware that such failures are not
detected by gotestsum.

To also capture stderr with gotestsum, stderr must get redirected like this:

   mkfifo /tmp/pipe
   go test ... 2>/tmp/pipe | gotestsum -stdin -stderr 3 3</tmp/pipe

Author: pohly

README.md

@@ -17,6 +17,7 @@ source with `go install gotest.tools/gotestsum@latest`. To run without installin
 - Print a [summary](#summary) of the test run after running all the tests.
 - Use any [`go test` flag](#custom-go-test-command),
   run a script with [`--raw-command`](#custom-go-test-command),
+  run `go test` manually and [pipe the output](#pipe) into `gotestsum`
   or [run a compiled test binary](#executing-a-compiled-test-binary).
 
 **CI and Automation**
@@ -306,6 +307,41 @@ gotestsum --raw-command ./profile.sh ./...
 TEST_DIRECTORY=./io/http gotestsum
 ```
 
+### Pipe into gotestsum
+
+When using a shell script which decides how to invoke `go test`, it can be
+difficult to generate a script for use with `--raw-command`. A more natural
+approach in a shell script is using a pipe:
+
+**Example: simple pipe**
+```
+go test . | gotestsum --stdin
+```
+
+As with `--raw-command` above, only `test2json` output is allowed on
+stdin. Anything else causes `gotestsum` to fail with a parser error.
+
+In this simple example, stderr of the test goes to the console and is not
+captured by `gotestsum`. To get that behavior, stderr of the first command can
+be redirected to a named pipe and then be read from there by `gotestsum`:
+
+**Example: redirect stdout and stderr**
+```
+mkfifo /tmp/stderr-pipe
+
+go test 2>/tmp/stderr-pipe | gotestsum --stdin --stderr 3 3</tmp/stderr-pipe
+```
+
+Note that `gotestsum` is not aware of a non-zero exit code of the test
+command. Bash's `pipefile` can be used to detect such a failure:
+
+**Example: pipefail**
+```
+set -o pipefail # bashism
+go test . | gotestsum --stdin
+res=$? # captures result of `go test` or `gotestsum`
+```
+
 ### Executing a compiled test binary
 
 `gotestsum` supports executing a compiled test binary (created with `go test -c`) by running

cmd/main.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -70,6 +71,10 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 		"use different icons, see help for options")
 	flags.BoolVar(&opts.rawCommand, "raw-command", false,
 		"don't prepend 'go test -json' to the 'go test' command")
+	flags.BoolVar(&opts.readStdin, "stdin", false,
+		"don't run any command, instead read go test stdout from stdin")
+	flags.IntVar(&opts.readStderrFD, "stderr", 0,
+		"read go test stderr from a certain `file descriptor` (only valid in combination with -stdin)")
 	flags.BoolVar(&opts.ignoreNonJSONOutputLines, "ignore-non-json-output-lines", false,
 		"write non-JSON 'go test' output lines to stderr instead of failing")
 	flags.Lookup("ignore-non-json-output-lines").Hidden = true
@@ -176,6 +181,8 @@ type options struct {
 	formatOptions                testjson.FormatOptions
 	debug                        bool
 	rawCommand                   bool
+	readStdin                    bool
+	readStderrFD                 int
 	ignoreNonJSONOutputLines     bool
 	jsonFile                     string
 	jsonFileTimingEvents         string
@@ -198,6 +205,8 @@ type options struct {
 	version                      bool
 
 	// shims for testing
+	stdin  io.Reader
+	fd3    io.Reader
 	stdout io.Writer
 	stderr io.Writer
 }
@@ -212,6 +221,15 @@ func (o options) Validate() error {
 		return fmt.Errorf("-failfast can not be used with --rerun-fails " +
 			"because not all test cases will run")
 	}
+	if o.rawCommand && o.readStdin {
+		return errors.New("--stdin and --raw-command are mutually exclusive")
+	}
+	if o.readStdin && len(o.args) > 0 {
+		return fmt.Errorf("--stdin does not support additional arguments (%q)", o.args)
+	}
+	if o.readStderrFD > 0 && !o.readStdin {
+		return errors.New("--stderr depends on --stdin")
+	}
 	return nil
 }
 
@@ -264,29 +282,52 @@ func run(opts *options) error {
 		return err
 	}
 
-	goTestProc, err := startGoTestFn(ctx, "", goTestCmdArgs(opts, rerunOpts{}))
-	if err != nil {
-		return err
-	}
-
 	handler, err := newEventHandler(opts)
 	if err != nil {
 		return err
 	}
 	defer handler.Close() // nolint: errcheck
 	cfg := testjson.ScanConfig{
-		Stdout:                   goTestProc.stdout,
-		Stderr:                   goTestProc.stderr,
 		Handler:                  handler,
 		Stop:                     cancel,
 		IgnoreNonJSONOutputLines: opts.ignoreNonJSONOutputLines,
 	}
+
+	var goTestProc *proc
+	if opts.readStdin {
+		cfg.Stdout = os.Stdin
+		if opts.stdin != nil {
+			cfg.Stdout = opts.stdin
+		}
+		if opts.readStderrFD > 0 {
+			if opts.readStderrFD == 3 && opts.fd3 != nil {
+				cfg.Stderr = opts.fd3
+			} else {
+				cfg.Stderr = os.NewFile(uintptr(opts.readStderrFD), fmt.Sprintf("go test stderr on fd %d", opts.stderr))
+			}
+		} else {
+			cfg.Stderr = bytes.NewReader(nil)
+		}
+	} else {
+		p, err := startGoTestFn(ctx, "", goTestCmdArgs(opts, rerunOpts{}))
+		if err != nil {
+			return err
+		}
+		goTestProc = p
+		cfg.Stdout = p.stdout
+		cfg.Stderr = p.stderr
+	}
+
 	exec, err := testjson.ScanTestOutput(cfg)
 	handler.Flush()
 	if err != nil {
 		return finishRun(opts, exec, err)
 	}
 
+	if opts.readStdin {
+		return finishRun(opts, exec, nil)
+	}
+
 	exitErr := goTestProc.cmd.Wait()
 	if signum := atomic.LoadInt32(&goTestProc.signal); signum != 0 {
 		return finishRun(opts, exec, exitError{num: signalExitCode + int(signum)})

cmd/main_e2e_test.go

@@ -275,3 +275,102 @@ func TestE2E_IgnoresWarnings(t *testing.T) {
 	)
 	golden.Assert(t, out, "e2e/expected/"+t.Name())
 }
+
+func TestE2E_StdinNoError(t *testing.T) {
+	t.Setenv("GITHUB_ACTIONS", "no")
+
+	flags, opts := setupFlags("gotestsum")
+	args := []string{
+		"--stdin",
+		"--format=testname",
+	}
+	assert.NilError(t, flags.Parse(args))
+	opts.args = flags.Args()
+
+	bufStdout := new(bytes.Buffer)
+	opts.stdout = bufStdout
+	bufStderr := new(bytes.Buffer)
+	opts.stderr = bufStderr
+
+	opts.stdin = strings.NewReader(`{"Time":"2024-06-16T14:46:00.343974039+02:00","Action":"start","Package":"example.com/test"}
+{"Time":"2024-06-16T14:46:00.378597503+02:00","Action":"run","Package":"example.com/test","Test":"TestSomething"}
+{"Time":"2024-06-16T14:46:00.378798569+02:00","Action":"pass","Package":"example.com/test","Test":"TestSomething","Elapsed":0}
+{"Time":"2024-06-16T14:46:00.404809796+02:00","Action":"pass","Package":"example.com/test","Elapsed":0.061}
+`)
+
+	err := run(opts)
+	assert.NilError(t, err)
+	out := text.ProcessLines(t, bufStdout,
+		text.OpRemoveSummaryLineElapsedTime,
+		text.OpRemoveTestElapsedTime,
+		filepath.ToSlash, // for windows
+	)
+	golden.Assert(t, out, "e2e/expected/"+t.Name())
+}
+
+func TestE2E_StdinFailure(t *testing.T) {
+	t.Setenv("GITHUB_ACTIONS", "no")
+
+	flags, opts := setupFlags("gotestsum")
+	args := []string{
+		"--stdin",
+		"--format=testname",
+	}
+	assert.NilError(t, flags.Parse(args))
+	opts.args = flags.Args()
+
+	bufStdout := new(bytes.Buffer)
+	opts.stdout = bufStdout
+	bufStderr := new(bytes.Buffer)
+	opts.stderr = bufStderr
+
+	opts.stdin = strings.NewReader(`{"Time":"2024-06-16T14:46:00.343974039+02:00","Action":"start","Package":"example.com/test"}
+{"Time":"2024-06-16T14:46:00.378597503+02:00","Action":"run","Package":"example.com/test","Test":"TestSomething"}
+{"Time":"2024-06-16T14:46:00.378798569+02:00","Action":"fail","Package":"example.com/test","Test":"TestSomething","Elapsed":0}
+{"Time":"2024-06-16T14:46:00.404809796+02:00","Action":"pass","Package":"example.com/test","Elapsed":0.061}
+`)
+
+	err := run(opts)
+	assert.NilError(t, err)
+	out := text.ProcessLines(t, bufStdout,
+		text.OpRemoveSummaryLineElapsedTime,
+		text.OpRemoveTestElapsedTime,
+		filepath.ToSlash, // for windows
+	)
+	golden.Assert(t, out, "e2e/expected/"+t.Name())
+}
+
+func TestE2E_StdinStderr(t *testing.T) {
+	t.Setenv("GITHUB_ACTIONS", "no")
+
+	flags, opts := setupFlags("gotestsum")
+	args := []string{
+		"--stdin",
+		"--stderr=3",
+		"--format=testname",
+	}
+	assert.NilError(t, flags.Parse(args))
+	opts.args = flags.Args()
+
+	bufStdout := new(bytes.Buffer)
+	opts.stdout = bufStdout
+	bufStderr := new(bytes.Buffer)
+	opts.stderr = bufStderr
+
+	opts.stdin = strings.NewReader(`{"Time":"2024-06-16T14:46:00.343974039+02:00","Action":"start","Package":"example.com/test"}
+{"Time":"2024-06-16T14:46:00.378597503+02:00","Action":"run","Package":"example.com/test","Test":"TestSomething"}
+{"Time":"2024-06-16T14:46:00.378798569+02:00","Action":"pass","Package":"example.com/test","Test":"TestSomething","Elapsed":0}
+{"Time":"2024-06-16T14:46:00.404809796+02:00","Action":"pass","Package":"example.com/test","Elapsed":0.061}
+`)
+
+	opts.fd3 = strings.NewReader(`build failure`)
+
+	err := run(opts)
+	assert.NilError(t, err)
+	out := text.ProcessLines(t, bufStdout,
+		text.OpRemoveSummaryLineElapsedTime,
+		text.OpRemoveTestElapsedTime,
+		filepath.ToSlash, // for windows
+	)
+	golden.Assert(t, out, "e2e/expected/"+t.Name())
+}

cmd/main_test.go

@@ -88,6 +88,21 @@ func TestOptions_Validate_FromFlags(t *testing.T) {
 			args:     []string{"--rerun-fails", "--packages=./...", "--", "-failfast"},
 			expected: "-failfast can not be used with --rerun-fails",
 		},
+		{
+			name:     "raw-command and stdin mutually exclusive",
+			args:     []string{"--raw-command", "--stdin"},
+			expected: "--stdin and --raw-command are mutually exclusive",
+		},
+		{
+			name:     "stdin must not be used with args",
+			args:     []string{"--stdin", "--", "-coverprofile=/tmp/out"},
+			expected: `--stdin does not support additional arguments (["-coverprofile=/tmp/out"])`,
+		},
+		{
+			name:     "stderr depends on stdin",
+			args:     []string{"--stderr", "4"},
+			expected: "--stderr depends on --stdin",
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

cmd/testdata/e2e/expected/TestE2E_StdinFailure

@@ -0,0 +1,7 @@
+FAIL example.com/test.TestSomething
+PASS example.com/test
+
+=== Failed
+=== FAIL: example.com/test TestSomething
+
+DONE 1 tests, 1 failure

cmd/testdata/e2e/expected/TestE2E_StdinNoError

@@ -0,0 +1,4 @@
+PASS example.com/test.TestSomething
+PASS example.com/test
+
+DONE 1 tests

cmd/testdata/e2e/expected/TestE2E_StdinStderr

@@ -0,0 +1,7 @@
+PASS example.com/test.TestSomething
+PASS example.com/test
+
+=== Errors
+build failure
+
+DONE 1 tests, 1 error

cmd/testdata/gotestsum-help-text

@@ -26,6 +26,8 @@ Flags:
       --rerun-fails-max-failures int                do not rerun any tests if the initial run has more than this number of failures (default 10)
       --rerun-fails-report string                   write a report to the file, of the tests that were rerun
       --rerun-fails-run-root-test                   rerun the entire root testcase when any of its subtests fail, instead of only the failed subtest
+      --stderr file descriptor                      read go test stderr from a certain file descriptor (only valid in combination with -stdin)
+      --stdin                                       don't run any command, instead read go test stdout from stdin
       --version                                     show version and exit
       --watch                                       watch go files, and run tests when a file is modified
       --watch-chdir                                 in watch mode change the working directory to the directory with the modified file before running tests