Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

go.mod: experimental integration of gomodjail (library sandbox) #4012

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ jobs:
test-integration-rootless:
needs: build-dependencies
timeout-minutes: 40
name: "${{ matrix.target }} | ${{ matrix.containerd }} | ${{ matrix.rootlesskit }} | ${{ matrix.ubuntu }}"
name: "${{ matrix.target }} | ${{ matrix.containerd }} | ${{ matrix.rootlesskit }} | ${{ matrix.ubuntu }} || ${{ matrix.nerdctl }}"
runs-on: "${{ matrix.runner }}"
strategy:
fail-fast: false
Expand All @@ -226,30 +226,47 @@ jobs:
target: rootless
runner: "ubuntu-22.04"
arch: amd64
nerdctl: ""
comment: ""
- ubuntu: 24.04
containerd: v2.0.4
rootlesskit: v2.3.4
target: rootless
arch: amd64
runner: "ubuntu-24.04"
nerdctl: ""
comment: ""
- ubuntu: 24.04
containerd: v2.0.4
rootlesskit: v2.3.4
target: rootless
arch: arm64
runner: "ubuntu-24.04-arm"
nerdctl: ""
comment: ""
- ubuntu: 24.04
containerd: v2.0.4
rootlesskit: v2.3.4
target: rootless-port-slirp4netns
arch: amd64
runner: "ubuntu-24.04"
nerdctl: ""
comment: ""
- ubuntu: 24.04
containerd: v2.0.3
rootlesskit: v2.3.4
target: rootless
arch: arm64
runner: "ubuntu-24.04"
nerdctl: "nerdctl.gomodjail"
comment: "Flaky, not a blocker for merging PRs"
env:
CONTAINERD_VERSION: "${{ matrix.containerd }}"
ARCH: "${{ matrix.arch }}"
UBUNTU_VERSION: "${{ matrix.ubuntu }}"
ROOTLESSKIT_VERSION: "${{ matrix.rootlesskit }}"
TEST_TARGET: "test-integration-${{ matrix.target }}"
NERDCTL: "${{ matrix.nerdctl }}"
steps:
- name: "Set up AppArmor"
if: matrix.ubuntu == '24.04'
Expand Down Expand Up @@ -296,9 +313,10 @@ jobs:
fi
echo "WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622}" >> "$GITHUB_ENV"
- name: "Test (network driver=slirp4netns, port driver=builtin)"
run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false
run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} -e NERDCTL=${NERDCTL} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(This is not working because test-integration-rootless.sh does not propagate the env var across the SSH session)

- name: "Test (network driver=slirp4netns, port driver=builtin) (flaky)"
run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true
if: matrix.nerdctl != 'nerdctl.gomodjail'
run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} -e NERDCTL=${NERDCTL} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true

build:
timeout-minutes: 5
Expand Down
27 changes: 26 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ ARG CONTAINERD_FUSE_OVERLAYFS_VERSION=v2.1.2@BINARY
ARG TINI_VERSION=v0.19.0@BINARY
# Extra deps: Debug
ARG BUILDG_VERSION=v0.4.1@BINARY
# Extra deps: gomodjail
# TODO: pin the deps
ARG MAKESELF_VERSION=v2.5.0
ARG GOMODJAIL_VERSION=master

# Test deps
# Currently, the Docker Official Images and the test deps are not pinned by the hash
Expand Down Expand Up @@ -118,7 +122,8 @@ RUN xx-go --wrap && \
xx-verify --static cmd/ipfs/ipfs && cp -a cmd/ipfs/ipfs /out/${TARGETARCH}

FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS build-base
RUN apk add --no-cache make git curl
# tar: required by makeself (executed by `gomodjail unpack`)
RUN apk add --no-cache make git curl tar
RUN git config --global advice.detachedHead false
ADD hack/git-checkout-tag-with-hash.sh /usr/local/bin/

Expand Down Expand Up @@ -231,6 +236,24 @@ RUN ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION/@BINARY}; \
tar xzf "${fname}" -C /out/bin && \
rm -f "${fname}" /out/bin/rootlesskit-docker-proxy && \
echo "- RootlessKit: ${ROOTLESSKIT_VERSION}" >> /out/share/doc/nerdctl-full/README.md
# makeself is needed by `gomodjail pack`.
# TODO: remove dependency on makeself https://github.com/AkihiroSuda/gomodjail/issues/31
Copy link
Member Author

@AkihiroSuda AkihiroSuda Mar 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems to be a blocker 😞

$ ./nerdctl.gomodjail --namespace=nerdctl-test info
Unrecognized flag : --namespace=nerdctl-test

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIRC static linking didn't work well with glibc, at least in the past.
Basically we should rather try to use Alpine (musl) for building the binaries

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do not build with CGO at all on alpine, though - (and whatever needs CGO is built on Debian - containerd and the rest, because they do need glibc anyhow).
So, the choice of libc is not relevant to make binaries and should not impact static in any way?
(also, you can build against musl on debian as well).

Anyhow, different discussion (can happen in #4021).

Will look into gomodjail and see if I can help with getting rid of makeself.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, left an idea in

(The blocker is about the arg parser of makeself, not about any Alpine stuff)

ARG MAKESELF_VERSION
RUN fname="makeself-${MAKESELF_VERSION/v}.run" && \
curl -o "${fname}" -fsSL --proto '=https' --tlsv1.2 "https://github.com/megastep/makeself/releases/download/release-${MAKESELF_VERSION/v}/${fname}" && \
grep "${fname}" "/SHA256SUMS.d/makeself-${MAKESELF_VERSION}" | sha256sum -c && \
dirname="makeself-${MAKESELF_VERSION/v}" && \
sh "${fname}" && \
cp -a "${dirname}"/makeself-header.sh /usr/local/bin/ && \
cp -a "${dirname}"/makeself.sh /usr/local/bin/makeself && \
rm -rf "${fname}" "${dirname}"
ARG GOMODJAIL_VERSION
RUN git clone https://github.com/AkihiroSuda/gomodjail.git /go/src/github.com/AkihiroSuda/gomodjail && \
cd /go/src/github.com/AkihiroSuda/gomodjail && \
git checkout "${GOMODJAIL_VERSION}" && \
make STATIC=1 && \
cp -a _output/bin/gomodjail /out/bin/ && \
echo "- gomodjail: ${GOMODJAIL_VERSION}" >> /out/share/doc/nerdctl-full/README.md

RUN echo "" >> /out/share/doc/nerdctl-full/README.md && \
echo "## License" >> /out/share/doc/nerdctl-full/README.md && \
Expand All @@ -245,6 +268,8 @@ COPY . /go/src/github.com/containerd/nerdctl
RUN { echo "# nerdctl (full distribution)"; echo "- nerdctl: $(cd /go/src/github.com/containerd/nerdctl && git describe --tags)"; cat /out/share/doc/nerdctl-full/README.md; } > /out/share/doc/nerdctl-full/README.md.new; mv /out/share/doc/nerdctl-full/README.md.new /out/share/doc/nerdctl-full/README.md
WORKDIR /go/src/github.com/containerd/nerdctl
RUN BINDIR=/out/bin make binaries install
RUN /out/bin/gomodjail pack --go-mod=/go/src/github.com/containerd/nerdctl/go.mod /out/bin/nerdctl && \
cp -a nerdctl.gomodjail /out/bin/
COPY README.md /out/share/doc/nerdctl/
COPY docs /out/share/doc/nerdctl/docs
RUN (cd /out && find ! -type d | sort | xargs sha256sum > /tmp/SHA256SUMS ) && \
Expand Down
1 change: 1 addition & 0 deletions Dockerfile.d/SHA256SUMS.d/makeself-v2.5.0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4d2fa9d898be22c63bb3c6bb7cc3dc97237700dea6d6ad898dcbec0289df0bc4 makeself-2.5.0.run
5 changes: 3 additions & 2 deletions Dockerfile.d/test-integration-rootless.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# limitations under the License.

set -eux -o pipefail
: "${NERDCTL:=}"
if [[ "$(id -u)" = "0" ]]; then
# Ensure securityfs is mounted for apparmor to work
if ! mountpoint -q /sys/kernel/security; then
Expand All @@ -32,7 +33,7 @@ if [[ "$(id -u)" = "0" ]]; then

# Switch to the rootless user via SSH
systemctl start ssh
exec ssh -o StrictHostKeyChecking=no rootless@localhost "$0" "$@"
exec ssh -o StrictHostKeyChecking=no rootless@localhost NERDCTL="$NERDCTL" "$0" "$@"
else
containerd-rootless-setuptool.sh install
if grep -q "options use-vc" /etc/resolv.conf; then
Expand Down Expand Up @@ -63,5 +64,5 @@ EOF
# Once ssh-ed, we lost the Dockerfile working dir, so, get back in the nerdctl checkout
cd /go/src/github.com/containerd/nerdctl
# We also lose the PATH (and SendEnv=PATH would require sshd config changes)
exec env PATH="/usr/local/go/bin:$PATH" "$@"
exec env PATH="/usr/local/go/bin:$PATH" NERDCTL="$NERDCTL" "$@"
fi
43 changes: 24 additions & 19 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//gomodjail:confined
module github.com/containerd/nerdctl/v2

go 1.23.0
Expand All @@ -8,53 +9,53 @@ require (
github.com/Microsoft/hcsshim v0.12.9
github.com/compose-spec/compose-go/v2 v2.4.9
github.com/containerd/accelerated-container-image v1.3.0
github.com/containerd/cgroups/v3 v3.0.5
github.com/containerd/console v1.0.4
github.com/containerd/cgroups/v3 v3.0.5 //gomodjail:unconfined
github.com/containerd/console v1.0.4 //gomodjail:unconfined
github.com/containerd/containerd/api v1.8.0
github.com/containerd/containerd/v2 v2.0.4
github.com/containerd/continuity v0.4.5
github.com/containerd/containerd/v2 v2.0.4 //gomodjail:unconfined
github.com/containerd/continuity v0.4.5 //gomodjail:unconfined
github.com/containerd/errdefs v1.0.0
github.com/containerd/fifo v1.1.0
github.com/containerd/fifo v1.1.0 //gomodjail:unconfined
github.com/containerd/go-cni v1.1.12
github.com/containerd/imgcrypt/v2 v2.0.1
github.com/containerd/log v0.1.0
github.com/containerd/nerdctl/mod/tigron v0.0.0
github.com/containerd/nydus-snapshotter v0.15.0
github.com/containerd/platforms v1.0.0-rc.1
github.com/containerd/platforms v1.0.0-rc.1 //gomodjail:unconfined
github.com/containerd/stargz-snapshotter v0.16.3
github.com/containerd/stargz-snapshotter/estargz v0.16.3
github.com/containerd/stargz-snapshotter/ipfs v0.16.3
github.com/containerd/typeurl/v2 v2.2.3
github.com/containernetworking/cni v1.2.3
github.com/containernetworking/cni v1.2.3 //gomodjail:unconfined
github.com/containernetworking/plugins v1.6.2
github.com/coreos/go-iptables v0.8.0
github.com/coreos/go-systemd/v22 v22.5.0
github.com/cyphar/filepath-securejoin v0.4.1
github.com/distribution/reference v0.6.0
github.com/docker/cli v28.0.2+incompatible
github.com/docker/docker v28.0.2+incompatible
github.com/docker/docker v28.0.2+incompatible //gomodjail:unconfined
github.com/docker/go-connections v0.5.0
github.com/docker/go-units v0.5.0
github.com/fahedouch/go-logrotate v0.2.1
github.com/fatih/color v1.18.0
github.com/fahedouch/go-logrotate v0.2.1 //gomodjail:unconfined
github.com/fatih/color v1.18.0 //gomodjail:unconfined
github.com/fluent/fluent-logger-golang v1.9.0
github.com/fsnotify/fsnotify v1.8.0
github.com/go-viper/mapstructure/v2 v2.2.1
github.com/ipfs/go-cid v0.5.0
github.com/klauspost/compress v1.18.0
github.com/mattn/go-isatty v0.0.20
github.com/mattn/go-isatty v0.0.20 //gomodjail:unconfined
github.com/moby/sys/mount v0.3.4
github.com/moby/sys/signal v0.7.1
github.com/moby/sys/userns v0.1.0
github.com/moby/term v0.5.2
github.com/muesli/cancelreader v0.2.2
github.com/moby/sys/userns v0.1.0 //gomodjail:unconfined
github.com/moby/term v0.5.2 //gomodjail:unconfined
github.com/muesli/cancelreader v0.2.2 //gomodjail:unconfined
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.1
github.com/opencontainers/runtime-spec v1.2.1
github.com/pelletier/go-toml/v2 v2.2.3
github.com/rootless-containers/bypass4netns v0.4.2
github.com/rootless-containers/rootlesskit/v2 v2.3.4
github.com/spf13/cobra v1.9.1
github.com/rootless-containers/bypass4netns v0.4.2 //gomodjail:unconfined
github.com/rootless-containers/rootlesskit/v2 v2.3.4 //gomodjail:unconfined
github.com/spf13/cobra v1.9.1 //gomodjail:unconfined
github.com/spf13/pflag v1.0.6
github.com/vishvananda/netlink v1.3.0
github.com/vishvananda/netns v0.0.5
Expand All @@ -63,8 +64,8 @@ require (
golang.org/x/crypto v0.36.0
golang.org/x/net v0.37.0
golang.org/x/sync v0.12.0
golang.org/x/sys v0.31.0
golang.org/x/term v0.30.0
golang.org/x/sys v0.31.0 //gomodjail:unconfined
golang.org/x/term v0.30.0 //gomodjail:unconfined
golang.org/x/text v0.23.0
gopkg.in/yaml.v3 v3.0.1
gotest.tools/v3 v3.5.2
Expand Down Expand Up @@ -105,6 +106,7 @@ require (
github.com/moby/sys/mountinfo v0.7.2 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/symlink v0.3.0 // indirect
//gomodjail:unconfined
github.com/moby/sys/user v0.3.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
Expand All @@ -118,6 +120,7 @@ require (
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/sasha-s/go-deadlock v0.3.5 // indirect
//gomodjail:unconfined
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/smallstep/pkcs7 v0.1.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
Expand All @@ -135,7 +138,9 @@ require (
go.opentelemetry.io/otel/trace v1.31.0 // indirect
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 // indirect
//gomodjail:unconfined
google.golang.org/grpc v1.69.4 // indirect
//gomodjail:unconfined
google.golang.org/protobuf v1.36.2 // indirect
lukechampine.com/blake3 v1.3.0 // indirect
)
Expand Down
18 changes: 13 additions & 5 deletions pkg/testutil/nerdtest/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,28 @@ func newNerdCommand(conf test.Config, t *testing.T) *nerdCommand {
trgt := getTarget()
switch trgt {
case targetNerdctl:
binary, err = exec.LookPath(trgt)
nerdctl := trgt
if env := os.Getenv("NERDCTL"); env != "" {
nerdctl = env
}
binary, err = exec.LookPath(nerdctl)
if err != nil {
t.Fatalf("unable to find binary %q: %v", trgt, err)
t.Fatalf("unable to find binary %q: %v", nerdctl, err)
}
// Set the default namespace if we do not have something already
if conf.Read(Namespace) == "" {
conf.Write(Namespace, defaultNamespace)
}
case targetDocker:
binary, err = exec.LookPath(trgt)
docker := trgt
if env := os.Getenv("DOCKER"); env != "" {
docker = env
}
binary, err = exec.LookPath(docker)
if err != nil {
t.Fatalf("unable to find binary %q: %v", trgt, err)
t.Fatalf("unable to find binary %q: %v", docker, err)
}
if err = exec.Command("docker", "compose", "version").Run(); err != nil {
if err = exec.Command(binary, "compose", "version").Run(); err != nil {
t.Fatalf("docker does not support compose: %v", err)
}
default:
Expand Down
12 changes: 10 additions & 2 deletions pkg/testutil/testutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -793,13 +793,21 @@ func newBase(t *testing.T, ns string, ipv6Compatible bool, kubernetesCompatible
var err error
switch base.Target {
case Nerdctl:
base.Binary, err = exec.LookPath("nerdctl")
nerdctl := "nerdctl"
if env := os.Getenv("NERDCTL"); env != "" {
nerdctl = env
}
base.Binary, err = exec.LookPath(nerdctl)
if err != nil {
t.Fatal(err)
}
base.Args = []string{"--namespace=" + ns}
case Docker:
base.Binary, err = exec.LookPath("docker")
docker := "docker"
if env := os.Getenv("DOCKER"); env != "" {
docker = env
}
base.Binary, err = exec.LookPath(docker)
if err != nil {
t.Fatal(err)
}
Expand Down
Loading