[FUTURE] [EXP] filesystem package

.golangci.yml

@@ -31,6 +31,7 @@ linters:
     - revive
     # Gocritic
     - gocritic
+    - forbidigo
 
     # 3. We used to use these, but have now removed them
 
@@ -41,6 +42,10 @@ linters:
     # - nakedret
 
   settings:
+    forbidigo:
+      forbid:
+        - pattern: ^os\.WriteFile.*$
+          msg: os.WriteFile is neither atomic nor durable - use nerdctl filesystem.WriteFile instead
     staticcheck:
       checks:
         # Below is the default set

cmd/nerdctl/compose/compose_up_linux_test.go

@@ -19,7 +19,6 @@
 import (
 	"fmt"
 	"io"
-	"os"
 	"strings"
 	"testing"
 	"time"
@@ -32,6 +31,7 @@
 
 	"github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers"
 	"github.com/containerd/nerdctl/v2/pkg/composer/serviceparser"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nettestutil"
@@ -503,7 +503,7 @@
 	// write the env.common file to tmpdir
 	tmpDir := t.TempDir()
 	envFilePath := fmt.Sprintf("%s/env.common", tmpDir)
-	err := os.WriteFile(envFilePath, []byte("TEST_ENV_INJECTION=WORKS\n"), 0644)
+	err := filesystem.WriteFile(envFilePath, []byte("TEST_ENV_INJECTION=WORKS\n"), 0644)
 	assert.NilError(t, err)
 
 	dockerComposeYAML := fmt.Sprintf(`

cmd/nerdctl/container/container_cp_acid_linux_test.go

@@ -26,6 +26,7 @@
 	"gotest.tools/v3/icmd"
 
 	"github.com/containerd/nerdctl/v2/pkg/containerutil"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
@@ -70,7 +71,7 @@
 			base.Cmd("exec", roContainer, "sh", "-euxc", "cd /vol2/dir2/rw; ln -s ../../../ relativelinktoroot").AssertOK()
 			base.Cmd("exec", roContainer, "sh", "-euxc", "cd /vol2/dir2/rw; ln -s / absolutelinktoroot").AssertOK()
 			// Create file on the host
-			err := os.WriteFile(sourceFile, sourceFileContent, filePerm)
+			err := filesystem.WriteFile(sourceFile, sourceFileContent, filePerm)
 			assert.NilError(t, err)
 		}
 

cmd/nerdctl/container/container_cp_linux_test.go

@@ -28,6 +28,7 @@ import (
 	"gotest.tools/v3/icmd"
 
 	"github.com/containerd/nerdctl/v2/pkg/containerutil"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
@@ -453,7 +454,7 @@ func TestCopyFromContainer(t *testing.T) {
 						ExitCode: 0,
 					},
 					setup: func(base *testutil.Base, container string, destPath string) {
-						err := os.WriteFile(destPath, []byte(""), filePerm)
+						err := filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -464,7 +465,7 @@ func TestCopyFromContainer(t *testing.T) {
 						ExitCode: 0,
 					},
 					setup: func(base *testutil.Base, container string, destPath string) {
-						err := os.WriteFile(destPath, []byte(""), filePerm)
+						err := filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -476,7 +477,7 @@ func TestCopyFromContainer(t *testing.T) {
 						Err:      containerutil.ErrDestinationIsNotADir.Error(),
 					},
 					setup: func(base *testutil.Base, container string, destPath string) {
-						err := os.WriteFile(destPath, []byte(""), filePerm)
+						err := filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -488,7 +489,7 @@ func TestCopyFromContainer(t *testing.T) {
 						Err:      containerutil.ErrDestinationIsNotADir.Error(),
 					},
 					setup: func(base *testutil.Base, container string, destPath string) {
-						err := os.WriteFile(destPath, []byte(""), filePerm)
+						err := filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -588,7 +589,7 @@ func TestCopyFromContainer(t *testing.T) {
 					setup: func(base *testutil.Base, container string, destPath string) {
 						err := os.MkdirAll(filepath.Dir(destPath), dirPerm)
 						assert.NilError(t, err)
-						err = os.WriteFile(destPath, []byte(""), filePerm)
+						err = filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -602,7 +603,7 @@ func TestCopyFromContainer(t *testing.T) {
 					setup: func(base *testutil.Base, container string, destPath string) {
 						err := os.MkdirAll(filepath.Dir(destPath), dirPerm)
 						assert.NilError(t, err)
-						err = os.WriteFile(destPath, []byte(""), filePerm)
+						err = filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -616,7 +617,7 @@ func TestCopyFromContainer(t *testing.T) {
 					setup: func(base *testutil.Base, container string, destPath string) {
 						err := os.MkdirAll(filepath.Dir(destPath), dirPerm)
 						assert.NilError(t, err)
-						err = os.WriteFile(destPath, []byte(""), filePerm)
+						err = filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -630,7 +631,7 @@ func TestCopyFromContainer(t *testing.T) {
 					setup: func(base *testutil.Base, container string, destPath string) {
 						err := os.MkdirAll(filepath.Dir(destPath), dirPerm)
 						assert.NilError(t, err)
-						err = os.WriteFile(destPath, []byte(""), filePerm)
+						err = filesystem.WriteFile(destPath, []byte(""), filePerm)
 						assert.NilError(t, err)
 					},
 				},
@@ -838,7 +839,7 @@ func cpTestHelper(t *testing.T, tg *testgroup) {
 					// Create file on the host
 					err := os.MkdirAll(filepath.Dir(sourceFile), dirPerm)
 					assert.NilError(t, err)
-					err = os.WriteFile(sourceFile, sourceFileContent, filePerm)
+					err = filesystem.WriteFile(sourceFile, sourceFileContent, filePerm)
 					assert.NilError(t, err)
 				}
 

cmd/nerdctl/container/container_inspect_linux_test.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/containerd/nerdctl/v2/pkg/infoutil"
 	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/labels"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
@@ -535,7 +536,7 @@ RUN groupadd -r test && useradd -r -g test test
 USER test
 `, testutil.UbuntuImage)
 
-			err := os.WriteFile(filepath.Join(data.Temp().Path(), "Dockerfile"), []byte(dockerfile), 0o600)
+			err := filesystem.WriteFile(filepath.Join(data.Temp().Path(), "Dockerfile"), []byte(dockerfile), 0o600)
 			assert.NilError(helpers.T(), err)
 
 			helpers.Ensure("build", "-t", data.Identifier(), data.Temp().Path())

cmd/nerdctl/container/container_run_cgroup_linux_test.go

@@ -38,6 +38,7 @@ import (
 
 	"github.com/containerd/nerdctl/v2/pkg/cmd/container"
 	"github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
@@ -246,7 +247,7 @@ func TestRunDevice(t *testing.T) {
 			assert.NilError(t, err)
 			t.Logf("lo[%d] = %+v", i, lo[i])
 			loContent := fmt.Sprintf("lo%d-content", i)
-			assert.NilError(t, os.WriteFile(lo[i].Device, []byte(loContent), 0o700))
+			assert.NilError(t, filesystem.WriteFile(lo[i].Device, []byte(loContent), 0o700))
 			data.Labels().Set("loContent"+strconv.Itoa(i), loContent)
 		}
 

cmd/nerdctl/container/container_run_linux_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/containerd/nerdctl/mod/tigron/test"
 
 	"github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/strutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
@@ -188,7 +189,7 @@ func TestRunIpcHost(t *testing.T) {
 	base := testutil.NewBase(t)
 	testFilePath := filepath.Join("/dev/shm",
 		fmt.Sprintf("%s-%d-%s", testutil.Identifier(t), os.Geteuid(), base.Target))
-	err := os.WriteFile(testFilePath, []byte(""), 0644)
+	err := filesystem.WriteFile(testFilePath, []byte(""), 0644)
 	assert.NilError(base.T, err)
 	defer os.Remove(testFilePath)
 
@@ -691,7 +692,7 @@ func TestRunDeviceCDIWithNerdctlConfig(t *testing.T) {
 	writeTestCDISpec(t, cdiSpecDir)
 
 	tomlPath := filepath.Join(t.TempDir(), "nerdctl.toml")
-	err := os.WriteFile(tomlPath, []byte(fmt.Sprintf(`
+	err := filesystem.WriteFile(tomlPath, []byte(fmt.Sprintf(`
 cdi_spec_dirs = ["%s"]
 `, cdiSpecDir)), 0400)
 	assert.NilError(t, err)
@@ -719,6 +720,6 @@ devices:
 	err := os.MkdirAll(cdiSpecDir, 0700)
 	assert.NilError(t, err)
 	cdiSpecPath := filepath.Join(cdiSpecDir, "vendor1.yaml")
-	err = os.WriteFile(cdiSpecPath, []byte(testCDIVendor1), 0400)
+	err = filesystem.WriteFile(cdiSpecPath, []byte(testCDIVendor1), 0400)
 	assert.NilError(t, err)
 }

cmd/nerdctl/helpers/testing.go

@@ -26,11 +26,13 @@
 	"testing"
 
 	"gotest.tools/v3/assert"
+
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 )
 
 func CreateBuildContext(t *testing.T, dockerfile string) string {
 	tmpDir := t.TempDir()
-	err := os.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(dockerfile), 0644)
+	err := filesystem.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(dockerfile), 0644)
 	assert.NilError(t, err)
 	return tmpDir
 }

cmd/nerdctl/image/image_list_test.go

@@ -19,7 +19,6 @@
 import (
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
 	"runtime"
 	"slices"
@@ -32,6 +31,7 @@
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
 
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/tabutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
@@ -149,7 +149,7 @@
 RUN echo "actually creating a layer so that docker sets the createdAt time"
 `, testutil.CommonImage)
 			buildCtx := data.Temp().Path()
-			err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
+			err := filesystem.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 			assert.NilError(helpers.T(), err)
 			data.Labels().Set("buildCtx", buildCtx)
 		},
@@ -296,7 +296,7 @@
 CMD ["echo", "nerdctl-build-notag-string"]
 	`, testutil.CommonImage)
 			buildCtx := data.Temp().Path()
-			err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
+			err := filesystem.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 			assert.NilError(helpers.T(), err)
 			data.Labels().Set("buildCtx", buildCtx)
 		},

cmd/nerdctl/image/image_prune_test.go

@@ -18,7 +18,6 @@ package image
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -30,6 +29,7 @@ import (
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
 
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
@@ -72,7 +72,7 @@ func TestImagePrune(t *testing.T) {
 					`, testutil.CommonImage)
 
 				buildCtx := data.Temp().Path()
-				err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
+				err := filesystem.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 				assert.NilError(helpers.T(), err)
 				helpers.Ensure("build", buildCtx)
 				// After we rebuild with tag, docker will no longer show the <none> version from above
@@ -120,7 +120,7 @@ func TestImagePrune(t *testing.T) {
 					`, testutil.CommonImage)
 
 				buildCtx := data.Temp().Path()
-				err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
+				err := filesystem.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 				assert.NilError(helpers.T(), err)
 				helpers.Ensure("build", buildCtx)
 				helpers.Ensure("build", "-t", identifier, buildCtx)
@@ -164,7 +164,7 @@ CMD ["echo", "nerdctl-test-image-prune-filter-label"]
 LABEL foo=bar
 LABEL version=0.1`, testutil.CommonImage)
 				buildCtx := data.Temp().Path()
-				err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
+				err := filesystem.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 				assert.NilError(helpers.T(), err)
 				helpers.Ensure("build", "-t", data.Identifier(), buildCtx)
 				imgList := helpers.Capture("images")
@@ -204,7 +204,7 @@ LABEL version=0.1`, testutil.CommonImage)
 RUN echo "Anything, so that we create actual content for docker to set the current time for CreatedAt"
 CMD ["echo", "nerdctl-test-image-prune-until"]`, testutil.CommonImage)
 				buildCtx := data.Temp().Path()
-				err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
+				err := filesystem.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 				assert.NilError(helpers.T(), err)
 				helpers.Ensure("build", "-t", data.Identifier(), buildCtx)
 				imgList := helpers.Capture("images")

cmd/nerdctl/ipfs/ipfs_registry_linux_test.go

@@ -31,6 +31,7 @@
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
 
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
@@ -138,7 +139,7 @@
 	`, data.Labels().Get(ipfsImageURLKey))
 
 				buildCtx := data.Temp().Path()
-				err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
+				err := filesystem.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 				assert.NilError(helpers.T(), err)
 
 				helpers.Ensure("build", "-t", data.Identifier("built-image"), buildCtx)

cmd/nerdctl/main.go

@@ -47,9 +47,9 @@
 	"github.com/containerd/nerdctl/v2/pkg/config"
 	ncdefaults "github.com/containerd/nerdctl/v2/pkg/defaults"
 	"github.com/containerd/nerdctl/v2/pkg/errutil"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/logging"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
-	"github.com/containerd/nerdctl/v2/pkg/store"
 	"github.com/containerd/nerdctl/v2/pkg/version"
 )
 

cmd/nerdctl/volume/volume_inspect_test.go

@@ -33,13 +33,14 @@
 	"github.com/containerd/nerdctl/mod/tigron/tig"
 
 	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/native"
+	"github.com/containerd/nerdctl/v2/pkg/internal/filesystem"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
 
 func createFileWithSize(mountPoint string, size int64) error {
 	token := make([]byte, size)
 	_, _ = rand.Read(token)
-	err := os.WriteFile(filepath.Join(mountPoint, "test-file"), token, 0644)
+	err := filesystem.WriteFile(filepath.Join(mountPoint, "test-file"), token, 0644)
 	return err
 }
 

docs/dev/auditing_dockerfile.md

@@ -255,7 +255,7 @@ On a warm cache, it is still over 150MB and 30+ seconds.
 In and of itself, this is hard to reduce, as we need these...
 
 Actions:
-- [ ] we could cache the module download location to reduce round-trips on modules that are shared accross
+- [ ] we could cache the module download location to reduce round-trips on modules that are shared across
 different projects
 - [ ] we are likely installing nerdctl modules six times - (once per architecture during the build phase, then once per
 ubuntu version and architecture during the tests runs (this is not even accounted for in the audit above)) - it should

docs/dev/store.md

@@ -23,7 +23,7 @@ containers can be named the same), etc.
 However, storing data on the filesystem in a reliable way comes with challenges:
 - incomplete writes may happen (because of a system restart, or an application crash), leaving important structured files
 in a broken state
-- concurrent writes, or reading while writing would obviously be a problem as well, be it accross goroutines, or between
+- concurrent writes, or reading while writing would obviously be a problem as well, be it across goroutines, or between
 concurrent executions of the nerdctl binary, or embedded in a third-party application that does concurrently access resources
 
 The `pkg/store` package does provide a "storage" abstraction that takes care of these issues, generally providing

docs/dir.md

@@ -65,7 +65,7 @@ Data volume
 
 Can be overridden with `nerdctl --cni-netconfpath=<NETCONFPATH>` flag and environment variable `$NETCONFPATH`.
 
-At the top-level of <NETCONFPATH>, network (files) are shared accross all namespaces.
+At the top-level of <NETCONFPATH>, network (files) are shared across all namespaces.
 Sub-folders inside <NETCONFPATH> are only available to the namespace bearing the same name,
 and its networks definitions are private.