Add package to generate random filesystem hierarchies for testing (#13)

Author: gammazero

random/files/doc.go

@@ -0,0 +1,5 @@
+// Package files provides functionality for creating filesystem hierarchies of
+// random files an directories. This is useful for testing that needs to
+// operate on directory trees. Random results are reproducible by reusing the
+// same seed. Random values are not cryptographically secure.
+package files

random/files/files.go

@@ -0,0 +1,193 @@
+package files
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"path/filepath"
+
+	"github.com/ipfs/go-test/random"
+)
+
+const (
+	fileNameSize  = 16
+	fileNameAlpha = "abcdefghijklmnopqrstuvwxyz01234567890-_"
+)
+
+type Config struct {
+	// Depth is the depth of the directory tree including the root directory.
+	Depth int
+	// Dirs is the number of subdirectories at each depth.
+	Dirs int
+	// Files is the number of files at each depth.
+	Files int
+	// FileSize sets the number of random bytes in each file.
+	FileSize int64
+	// Where to write display output, such as os.Stdout. Default is nil.
+	Out io.Writer
+	// RandomDirss specifies whether or not to randomize the number of
+	// subdirectoriess from 1 to the value configured by Dirs.
+	RandomDirs bool
+	// RandomFiles specifies whether or not to randomize the number of files
+	// from 1 to the value configured by Files.
+	RandomFiles bool
+	// RandomSize specifies whether or not to randomize the file size from 1 to
+	// the value configured by FileSize.
+	RandomSize bool
+	// Seed sets the seen for the random number generator when set to a
+	// non-zero value.
+	Seed int64
+}
+
+func DefaultConfig() Config {
+	return Config{
+		Depth:       2,
+		Dirs:        5,
+		Files:       10,
+		FileSize:    4096,
+		RandomDirs:  false,
+		RandomFiles: false,
+		RandomSize:  true,
+	}
+}
+
+func validateConfig(cfg *Config) error {
+	if cfg.Depth < 1 || cfg.Depth > 64 {
+		return errors.New("depth out of range, must be between 1 and 64")
+	}
+	if cfg.Dirs < 0 || cfg.Dirs > 64 {
+		return errors.New("dirs out of range, must be between 0 and 64")
+	}
+	if cfg.Files < 0 || cfg.Files > 64 {
+		return errors.New("files out of range, must be between 0 and 64")
+	}
+	if cfg.FileSize < 0 {
+		return errors.New("file size out of range, must be 0 or greater")
+	}
+	if cfg.Depth > 1 && cfg.Dirs < 1 {
+		return errors.New("dirs must be at least 1 for depth > 1")
+	}
+
+	return nil
+}
+
+func Create(cfg Config, paths ...string) error {
+	if len(paths) == 0 {
+		return errors.New("must provide at least 1 root directory path")
+	}
+	err := validateConfig(&cfg)
+	if err != nil {
+		return err
+	}
+
+	var rnd *rand.Rand
+	if cfg.Seed == 0 {
+		rnd = random.NewRand()
+	} else {
+		rnd = random.NewSeededRand(cfg.Seed)
+	}
+
+	for _, root := range paths {
+		err := os.MkdirAll(root, 0755)
+		if err != nil {
+			return err
+		}
+
+		err = writeTree(rnd, root, 1, &cfg)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}
+
+func writeTree(rnd *rand.Rand, root string, depth int, cfg *Config) error {
+	nFiles := cfg.Files
+	if nFiles != 0 {
+		if cfg.RandomFiles && nFiles > 1 {
+			nFiles = rnd.Intn(nFiles) + 1
+		}
+
+		for i := 0; i < nFiles; i++ {
+			if err := writeFile(rnd, root, cfg); err != nil {
+				return err
+			}
+		}
+	}
+
+	return writeSubdirs(rnd, root, depth, cfg)
+}
+
+func writeSubdirs(rnd *rand.Rand, root string, depth int, cfg *Config) error {
+	if depth == cfg.Depth {
+		return nil
+	}
+	depth++
+
+	nDirs := cfg.Dirs
+	if cfg.RandomDirs && nDirs > 1 {
+		nDirs = rnd.Intn(nDirs) + 1
+	}
+
+	for i := 0; i < nDirs; i++ {
+		if err := writeSubdir(rnd, root, depth, cfg); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func writeSubdir(rnd *rand.Rand, root string, depth int, cfg *Config) error {
+	name := randomFilename(rnd)
+	root = filepath.Join(root, name)
+	if err := os.MkdirAll(root, 0755); err != nil {
+		return err
+	}
+
+	if cfg.Out != nil {
+		fmt.Fprintln(cfg.Out, root+"/")
+	}
+
+	return writeTree(rnd, root, depth, cfg)
+}
+
+func randomFilename(rnd *rand.Rand) string {
+	n := rnd.Intn(fileNameSize-4) + 4
+	b := make([]byte, n)
+	for i := 0; i < n; i++ {
+		b[i] = fileNameAlpha[rnd.Intn(len(fileNameAlpha))]
+	}
+	return string(b)
+}
+
+func writeFile(rnd *rand.Rand, root string, cfg *Config) error {
+	name := randomFilename(rnd)
+	filePath := filepath.Join(root, name)
+	f, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+
+	if cfg.FileSize > 0 {
+		fileSize := cfg.FileSize
+		if cfg.RandomSize && fileSize > 1 {
+			fileSize = rnd.Int63n(fileSize) + 1
+		}
+
+		if _, err := io.CopyN(f, rnd, fileSize); err != nil {
+			f.Close()
+			return err
+		}
+	}
+
+	if cfg.Out != nil {
+		fmt.Fprintln(cfg.Out, filePath)
+	}
+
+	return f.Close()
+}

random/files/files_test.go

@@ -0,0 +1,87 @@
+package files_test
+
+import (
+	"bufio"
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/ipfs/go-test/random/files"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRandomFiles(t *testing.T) {
+	var b bytes.Buffer
+	cfg := files.DefaultConfig()
+	cfg.Depth = 2
+	cfg.Dirs = 5
+	cfg.Files = 3
+	cfg.Out = &b
+
+	roots := []string{"foo"}
+	err := files.Create(cfg, roots...)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		for _, root := range roots {
+			os.RemoveAll(root)
+		}
+	})
+
+	t.Logf("Created file hierarchy:\n%s", b.String())
+
+	var lines int
+	scanner := bufio.NewScanner(&b)
+	for scanner.Scan() {
+		lines++
+	}
+	require.NoError(t, scanner.Err())
+
+	subdirs := 0
+	if cfg.Depth > 1 {
+		dirsAtDepth := cfg.Dirs
+		subdirs += dirsAtDepth
+		for i := 0; i < cfg.Depth-2; i++ {
+			dirsAtDepth *= cfg.Dirs
+			subdirs += dirsAtDepth
+		}
+	}
+	linesPerSubdir := cfg.Files + 1
+	expect := ((subdirs * linesPerSubdir) + cfg.Files) * len(roots)
+	require.Equal(t, expect, lines)
+}
+
+func TestRandomFilesValidation(t *testing.T) {
+	cfg := files.DefaultConfig()
+	err := files.Create(cfg)
+	require.Error(t, err)
+
+	cfg.Depth = 0
+	require.Error(t, files.Create(cfg, "foo"))
+	cfg.Depth = 65
+	require.Error(t, files.Create(cfg, "foo"))
+
+	cfg = files.DefaultConfig()
+
+	cfg.Dirs = -1
+	require.Error(t, files.Create(cfg, "foo"))
+	cfg.Dirs = 65
+	require.Error(t, files.Create(cfg, "foo"))
+
+	cfg = files.DefaultConfig()
+
+	cfg.Files = -1
+	require.Error(t, files.Create(cfg, "foo"))
+	cfg.Files = 65
+	require.Error(t, files.Create(cfg, "foo"))
+
+	cfg = files.DefaultConfig()
+
+	cfg.FileSize = -1
+	require.Error(t, files.Create(cfg, "foo"))
+
+	cfg = files.DefaultConfig()
+
+	cfg.Depth = 2
+	cfg.Dirs = 0
+	require.Error(t, files.Create(cfg, "foo"))
+}

random/files/random-files/README.md

@@ -0,0 +1,104 @@
+# random-files - create random filesystem hierarchies
+
+random-files creates random filesystem hierarchies for testing
+
+## Install
+
+```
+go install github.com/ipfs/go-test/random/files/random-files
+```
+
+## Usage
+
+```sh
+> random-files -help
+usage: ./random-files [options] <path>...
+Write a random filesystem hierarchy to each <path>
+
+Options:
+  -depth int
+        depth of the directory tree including the root directory (default 2)
+  -dirs int
+        number of subdirectories at each depth (default 5)
+  -files int
+        number of files at each depth (default 10)
+  -filesize int
+        file fize, or the max file size id RandomSize is true (default 4096)
+  -q    do not print files and directories
+  -random-dirs
+        randomize number of subdirectories, from 1 to -Dirs
+  -random-files
+        randomize number of files, from 1 to -Files
+  -random-size
+        randomize file size, from 1 to -FileSize (default true)
+  -seed int
+        random seed, 0 for current time
+```
+
+## Examples
+
+```sh
+> random-files -depth=2 -files=3 -seed=1701 foo
+foo/rwd67uvnj9yz-
+foo/7vovyvr9
+foo/fjv0w0
+foo/gyubi50rec5/
+foo/gyubi50rec5/vr6x-ce4uupj
+foo/gyubi50rec5/ob9ud0e8lt_2e
+foo/gyubi50rec5/11gip6zea
+foo/nzu5j29-sh-ku4/
+foo/nzu5j29-sh-ku4/vcs1629n
+foo/nzu5j29-sh-ku4/rky_i_qsxrp
+foo/nzu5j29-sh-ku4/xr1usy5ic0
+foo/w30dzrx2w4_d/
+foo/w30dzrx2w4_d/7ued6
+foo/w30dzrx2w4_d/r1d3j
+foo/w30dzrx2w4_d/av7d09i-av
+foo/s6ha-58/
+foo/s6ha-58/nukjsxg7t
+foo/s6ha-58/7of_84
+foo/s6ha-58/h0jgq8mu1n7u
+foo/tq_8/
+foo/tq_8/sx-a2jgmz_mk6
+foo/tq_8/9hzrksz8
+foo/tq_8/8b5swu
+```
+
+It made:
+
+```sh
+> tree foo
+foo
+├── 7vovyvr9
+├── fjv0w0
+├── gyubi50rec5
+│   ├── 11gip6zea
+│   ├── ob9ud0e8lt_2e
+│   └── vr6x-ce4uupj
+├── nzu5j29-sh-ku4
+│   ├── rky_i_qsxrp
+│   ├── vcs1629n
+│   └── xr1usy5ic0
+├── rwd67uvnj9yz-
+├── s6ha-58
+│   ├── 7of_84
+│   ├── h0jgq8mu1n7u
+│   └── nukjsxg7t
+├── tq_8
+│   ├── 8b5swu
+│   ├── 9hzrksz8
+│   └── sx-a2jgmz_mk6
+└── w30dzrx2w4_d
+    ├── 7ued6
+    ├── av7d09i-av
+    └── r1d3j
+
+6 directories, 18 files
+```
+
+Note: Specifying the same seed will produce the same results.
+
+
+### Acknowledgments
+
+Credit to Juan Benet as the author of [`go-random-files`](https://github.com/jbenet/go-random-files) from which this code was derived.