Implement nerdctl run --annotation (introduced in Docker v24)

An OCI runtime (as well as `nerdctl internal oci-hook`) may consume an
annotation and behave differently.
e.g., https://github.com/opencontainers/runc/blob/v1.1.12/docs/systemd.md#auxiliary-properties

nerdctl v1:
- `nerdctl run --annotation` was not implemented.
- `nerdctl run --label` is set as a containerd label and an OCI annotation.

nerdctl v2:
- `nerdctl run --annotation` is only set as an OCI annotation.
- `nerdctl run --label` is only set as a containerd label.
  A label with the `nerdctl/` prefix can no longer be set manually,
  with an exception for `nerdctl/bypass4netns`.
  The `nerdctl/bypass4netns` label is still allowed and is propagated to
  an OCI annotation, for sake of compatibility.

Docker v23:
- `docker run --annotation` was not implemented.
- `docker run --label` is only set as a Docker label.

Docker v24 (implemented in docker/cli PR 4156, moby/moby PR 45025):
- `docker run --annotation` is only set as an OCI annotation.
- `docker run --label` is only set as a Docker label.

Signed-off-by: Akihiro Suda <[email protected]>

Author: AkihiroSuda

README.md

@@ -189,7 +189,7 @@ Major:
 - [P2P image distribution using IPFS](./docs/ipfs.md): `nerdctl run ipfs://CID` .
   P2P image distribution (IPFS) is completely optional. Your host is NOT connected to any P2P network, unless you opt in to [install and run IPFS daemon](https://docs.ipfs.io/install/).
 - [Cosign integration](./docs/cosign.md): `nerdctl pull --verify=cosign` and `nerdctl push --sign=cosign`, and [in Compose](./docs/cosign.md#cosign-in-compose)
-- [Accelerated rootless containers using bypass4netns](./docs/rootless.md): `nerdctl run --label nerdctl/bypass4netns=true`
+- [Accelerated rootless containers using bypass4netns](./docs/rootless.md): `nerdctl run --annotation nerdctl/bypass4netns=true`
 
 Minor:
 

cmd/nerdctl/compose_up_linux_test.go

@@ -523,7 +523,7 @@ services:
       WORDPRESS_DB_NAME: exampledb
     volumes:
       - wordpress:/var/www/html
-    labels:
+    annotations:
       - nerdctl/bypass4netns=1
 
   db:
@@ -536,7 +536,7 @@ services:
       MYSQL_RANDOM_ROOT_PASSWORD: '1'
     volumes:
       - db:/var/lib/mysql
-    labels:
+    annotations:
       - nerdctl/bypass4netns=1
 
 volumes:

cmd/nerdctl/container_create.go

@@ -337,6 +337,10 @@ func processContainerCreateOptions(cmd *cobra.Command) (opt types.ContainerCreat
 	if err != nil {
 		return
 	}
+	opt.Annotations, err = cmd.Flags().GetStringArray("annotation")
+	if err != nil {
+		return
+	}
 	opt.CidFile, err = cmd.Flags().GetString("cidfile")
 	if err != nil {
 		return

cmd/nerdctl/container_run.go

@@ -230,8 +230,10 @@ func setCreateFlags(cmd *cobra.Command) {
 	cmd.Flags().String("name", "", "Assign a name to the container")
 	// label needs to be StringArray, not StringSlice, to prevent "foo=foo1,foo2" from being split to {"foo=foo1", "foo2"}
 	cmd.Flags().StringArrayP("label", "l", nil, "Set metadata on container")
-	cmd.RegisterFlagCompletionFunc("label", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-		return labels.ShellCompletions, cobra.ShellCompDirectiveNoFileComp
+	// annotation needs to be StringArray, not StringSlice, to prevent "foo=foo1,foo2" from being split to {"foo=foo1", "foo2"}
+	cmd.Flags().StringArray("annotation", nil, "Add an annotation to the container (passed through to the OCI runtime)")
+	cmd.RegisterFlagCompletionFunc("annotation", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		return labels.AnnotationShellCompletions, cobra.ShellCompDirectiveNoFileComp
 	})
 
 	// label-file is defined as StringSlice, not StringArray, to allow specifying "--env-file=FILE1,FILE2" (compatible with Podman)

docs/command-reference.md

@@ -298,8 +298,9 @@ Env flags:
 Metadata flags:
 
 - :whale: :blue_square: `--name`: Assign a name to the container
-- :whale: :blue_square: `-l, --label`: Set meta data on a container
+- :whale: :blue_square: `-l, --label`: Set meta data on a container (Not passed through the OCI runtime since nerdctl v2.0, with an exception for `nerdctl/bypass4netns`)
 - :whale: :blue_square: `--label-file`: Read in a line delimited file of labels
+- :whale: :blue_square: `--annotation`: Add an annotation to the container (passed through to the OCI runtime)
 - :whale: :blue_square: `--cidfile`: Write the container ID to the file
 - :nerd_face: `--pidfile`: file path to write the task's pid. The CLI syntax conforms to Podman convention.
 

docs/rootless.md

@@ -121,11 +121,15 @@ The performance benchmark with iperf3 on Ubuntu 21.10 on Hyper-V VM is shown bel
 
 This benchmark can be reproduced with [https://github.com/rootless-containers/bypass4netns/blob/f009d96139e9e38ce69a2ea8a9a746349bad273c/Vagrantfile](https://github.com/rootless-containers/bypass4netns/blob/f009d96139e9e38ce69a2ea8a9a746349bad273c/Vagrantfile)
 
-Acceleration with bypass4netns is available with `--label nerdctl/bypass4netns=true`. You also need to have `bypass4netnsd` (bypass4netns daemon) to be running.
+Acceleration with bypass4netns is available with:
+- `--annotation nerdctl/bypass4netns=true` (for nerdctl v2.0 and later)
+- `--label nerdctl/bypass4netns=true` (deprecated form, used in nerdctl prior to v2.0).
+
+You also need to have `bypass4netnsd` (bypass4netns daemon) to be running.
 Example
 ```console
 $ containerd-rootless-setuptool.sh install-bypass4netnsd
-$ nerdctl run -it --rm -p 8080:80 --label nerdctl/bypass4netns=true alpine
+$ nerdctl run -it --rm -p 8080:80 --annotation nerdctl/bypass4netns=true alpine
 ```
 
 More detail is available at [https://github.com/rootless-containers/bypass4netns/blob/master/README.md](https://github.com/rootless-containers/bypass4netns/blob/master/README.md)

extras/rootless/containerd-rootless-setuptool.sh

@@ -365,7 +365,7 @@ cmd_entrypoint_install_bypass4netnsd() {
 		[Install]
 		WantedBy=default.target
 	EOT
-	INFO "To use bypass4netnsd, set the \"nerdctl/bypass4netns=true\" label on containers, e.g., \`nerdctl run --label nerdctl/bypass4netns=true\`"
+	INFO "To use bypass4netnsd, set the \"nerdctl/bypass4netns=true\" annotation on containers, e.g., \`nerdctl run --annotation nerdctl/bypass4netns=true\`"
 }
 
 # CLI subcommand: "install-fuse-overlayfs"

pkg/api/types/container_types.go

@@ -217,9 +217,12 @@ type ContainerCreateOptions struct {
 	// Name assign a name to the container
 	Name string
 	// Label set meta data on a container
+	// (not passed through to the OCI runtime since nerdctl v2.0, with an exception for "nerdctl/bypass4netns")
 	Label []string
 	// LabelFile read in a line delimited file of labels
 	LabelFile []string
+	// Annotations set meta data on a container (passed through to the OCI runtime)
+	Annotations []string
 	// CidFile write the container ID to the file
 	CidFile string
 	// PidFile specifies the file path to write the task's pid. The CLI syntax conforms to Podman convention.

pkg/cmd/container/create.go

@@ -238,6 +238,7 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa
 		return nil, nil, err
 	}
 	cOpts = append(cOpts, lCOpts...)
+	opts = append(opts, oci.WithAnnotations(strutil.ConvertKVStringsToMap(options.Annotations)))
 
 	var containerNameStore namestore.NameStore
 	if options.Name == "" && !options.NameChanged {
@@ -282,7 +283,7 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa
 	}
 	cOpts = append(cOpts, ilOpt)
 
-	opts = append(opts, propagateContainerdLabelsToOCIAnnotations())
+	opts = append(opts, propagateInternalContainerdLabelsToOCIAnnotations())
 
 	var s specs.Spec
 	spec := containerd.WithSpec(&s, opts...)
@@ -506,6 +507,13 @@ func withContainerLabels(label, labelFile []string) ([]containerd.NewContainerOp
 	if err != nil {
 		return nil, err
 	}
+	for k := range labelMap {
+		if k == labels.Bypass4netns {
+			log.L.Warnf("Label %q is deprecated, use an annotation instead", k)
+		} else if strings.HasPrefix(k, labels.Prefix) {
+			return nil, fmt.Errorf("internal label %q must not be specified manually", k)
+		}
+	}
 	o := containerd.WithAdditionalContainerLabels(labelMap)
 	return []containerd.NewContainerOpts{o}, nil
 }
@@ -704,9 +712,15 @@ func processeds(mountPoints []dockercompat.MountPoint) []*mountutil.Processed {
 	return result
 }
 
-func propagateContainerdLabelsToOCIAnnotations() oci.SpecOpts {
+func propagateInternalContainerdLabelsToOCIAnnotations() oci.SpecOpts {
 	return func(ctx context.Context, oc oci.Client, c *containers.Container, s *oci.Spec) error {
-		return oci.WithAnnotations(c.Labels)(ctx, oc, c, s)
+		allowed := make(map[string]string)
+		for k, v := range c.Labels {
+			if strings.Contains(k, labels.Prefix) {
+				allowed[k] = v
+			}
+		}
+		return oci.WithAnnotations(allowed)(ctx, oc, c, s)
 	}
 }
 

pkg/composer/serviceparser/serviceparser.go

@@ -54,6 +54,7 @@ const Separator = "-"
 func warnUnknownFields(svc types.ServiceConfig) {
 	if unknown := reflectutil.UnknownNonEmptyFields(&svc,
 		"Name",
+		"Annotations",
 		"Build",
 		"BlkioConfig",
 		"CapAdd",
@@ -477,6 +478,14 @@ func newContainer(project *types.Project, parsed *Service, i int) (*Container, e
 		"--pull=never", // because image will be ensured before running replicas with `nerdctl run`.
 	}
 
+	for k, v := range svc.Annotations {
+		if v == "" {
+			c.RunArgs = append(c.RunArgs, fmt.Sprintf("--annotation=%s", k))
+		} else {
+			c.RunArgs = append(c.RunArgs, fmt.Sprintf("--annotation=%s=%s", k, v))
+		}
+	}
+
 	if svc.BlkioConfig != nil && svc.BlkioConfig.Weight != 0 {
 		c.RunArgs = append(c.RunArgs, fmt.Sprintf("--blkio-weight=%d", svc.BlkioConfig.Weight))
 	}

pkg/labels/labels.go

@@ -15,7 +15,7 @@
 */
 
 // Package labels defines labels that are set to containerd containers as labels.
-// The labels are also passed to OCI containers as annotations.
+// The labels defined in this package are also passed to OCI containers as annotations.
 package labels
 
 const (
@@ -107,8 +107,8 @@ const (
 	NerdctlDefaultNetwork = Prefix + "default-network"
 )
 
-var ShellCompletions = []string{
+var AnnotationShellCompletions = []string{
 	Bypass4netns + "=true",
 	Bypass4netns + "=false",
-	// Other labels should not be set via CLI
+	// Other annotations should not be set via CLI
 }