Implement nerdctl run --annotation (introduced in Docker v24)

An OCI runtime (as well as `nerdctl internal oci-hook`) may consume an
annotation and behave differently.
e.g., https://github.com/opencontainers/runc/blob/v1.1.12/docs/systemd.md#auxiliary-properties

nerdctl v1:
- `nerdctl run --annotation` was not implemented.
- `nerdctl run --label` is set as a containerd label and an OCI annotation.

nerdctl v2:
- `nerdctl run --annotation` is only set as an OCI annotation.
- `nerdctl run --label` is only set as a containerd label.
  A label with the `nerdctl/` prefix can no longer be set manually,
  with an exception for `nerdctl/bypass4netns`.
  The `nerdctl/bypass4netns` label is still allowed and is propagated to
  an OCI annotation, for sake of compatibility.

Docker v23:
- `docker run --annotation` was not implemented.
- `docker run --label` is only set as a Docker label.

Docker v24 (implemented in docker/cli PR 4156, moby/moby PR 45025):
- `docker run --annotation` is only set as an OCI annotation.
- `docker run --label` is only set as a Docker label.

Signed-off-by: Akihiro Suda <[email protected]>

Author: AkihiroSuda

README.md

@@ -189,7 +189,7 @@ Major:
 - [P2P image distribution using IPFS](./docs/ipfs.md): `nerdctl run ipfs://CID` .
   P2P image distribution (IPFS) is completely optional. Your host is NOT connected to any P2P network, unless you opt in to [install and run IPFS daemon](https://docs.ipfs.io/install/).
 - [Cosign integration](./docs/cosign.md): `nerdctl pull --verify=cosign` and `nerdctl push --sign=cosign`, and [in Compose](./docs/cosign.md#cosign-in-compose)
-- [Accelerated rootless containers using bypass4netns](./docs/rootless.md): `nerdctl run --label nerdctl/bypass4netns=true`
+- [Accelerated rootless containers using bypass4netns](./docs/rootless.md): `nerdctl run --annotation nerdctl/bypass4netns=true`
 
 Minor:
 

cmd/nerdctl/compose_up_linux_test.go

@@ -523,7 +523,7 @@ services:
       WORDPRESS_DB_NAME: exampledb
     volumes:
       - wordpress:/var/www/html
-    labels:
+    annotations:
       - nerdctl/bypass4netns=1
 
   db:
@@ -536,7 +536,7 @@ services:
       MYSQL_RANDOM_ROOT_PASSWORD: '1'
     volumes:
       - db:/var/lib/mysql
-    labels:
+    annotations:
       - nerdctl/bypass4netns=1
 
 volumes:

cmd/nerdctl/container_create.go

@@ -337,6 +337,10 @@ func processContainerCreateOptions(cmd *cobra.Command) (opt types.ContainerCreat
 	if err != nil {
 		return
 	}
+	opt.Annotations, err = cmd.Flags().GetStringArray("annotation")
+	if err != nil {
+		return
+	}
 	opt.CidFile, err = cmd.Flags().GetString("cidfile")
 	if err != nil {
 		return

cmd/nerdctl/container_run.go

@@ -230,8 +230,10 @@ func setCreateFlags(cmd *cobra.Command) {
 	cmd.Flags().String("name", "", "Assign a name to the container")
 	// label needs to be StringArray, not StringSlice, to prevent "foo=foo1,foo2" from being split to {"foo=foo1", "foo2"}
 	cmd.Flags().StringArrayP("label", "l", nil, "Set metadata on container")
-	cmd.RegisterFlagCompletionFunc("label", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-		return labels.ShellCompletions, cobra.ShellCompDirectiveNoFileComp
+	// annotation needs to be StringArray, not StringSlice, to prevent "foo=foo1,foo2" from being split to {"foo=foo1", "foo2"}
+	cmd.Flags().StringArray("annotation", nil, "Add an annotation to the container (passed through to the OCI runtime)")
+	cmd.RegisterFlagCompletionFunc("annotation", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		return labels.AnnotationShellCompletions, cobra.ShellCompDirectiveNoFileComp
 	})
 
 	// label-file is defined as StringSlice, not StringArray, to allow specifying "--env-file=FILE1,FILE2" (compatible with Podman)

docs/command-reference.md

@@ -298,8 +298,9 @@ Env flags:
 Metadata flags:
 
 - :whale: :blue_square: `--name`: Assign a name to the container
-- :whale: :blue_square: `-l, --label`: Set meta data on a container
+- :whale: :blue_square: `-l, --label`: Set meta data on a container (Not passed through the OCI runtime since nerdctl v2.0, with an exception for `nerdctl/bypass4netns`)
 - :whale: :blue_square: `--label-file`: Read in a line delimited file of labels
+- :whale: :blue_square: `--annotation`: Add an annotation to the container (passed through to the OCI runtime)
 - :whale: :blue_square: `--cidfile`: Write the container ID to the file
 - :nerd_face: `--pidfile`: file path to write the task's pid. The CLI syntax conforms to Podman convention.
 

docs/rootless.md

@@ -121,11 +121,15 @@ The performance benchmark with iperf3 on Ubuntu 21.10 on Hyper-V VM is shown bel
 
 This benchmark can be reproduced with [https://github.com/rootless-containers/bypass4netns/blob/f009d96139e9e38ce69a2ea8a9a746349bad273c/Vagrantfile](https://github.com/rootless-containers/bypass4netns/blob/f009d96139e9e38ce69a2ea8a9a746349bad273c/Vagrantfile)
 
-Acceleration with bypass4netns is available with `--label nerdctl/bypass4netns=true`. You also need to have `bypass4netnsd` (bypass4netns daemon) to be running.
+Acceleration with bypass4netns is available with:
+- `--annotation nerdctl/bypass4netns=true` (for nerdctl v2.0 and later)
+- `--label nerdctl/bypass4netns=true` (deprecated form, used in nerdctl prior to v2.0).
+
+You also need to have `bypass4netnsd` (bypass4netns daemon) to be running.
 Example
 ```console
 $ containerd-rootless-setuptool.sh install-bypass4netnsd
-$ nerdctl run -it --rm -p 8080:80 --label nerdctl/bypass4netns=true alpine
+$ nerdctl run -it --rm -p 8080:80 --annotation nerdctl/bypass4netns=true alpine
 ```
 
 More detail is available at [https://github.com/rootless-containers/bypass4netns/blob/master/README.md](https://github.com/rootless-containers/bypass4netns/blob/master/README.md)

extras/rootless/containerd-rootless-setuptool.sh

@@ -365,7 +365,7 @@ cmd_entrypoint_install_bypass4netnsd() {
 		[Install]
 		WantedBy=default.target
 	EOT
-	INFO "To use bypass4netnsd, set the \"nerdctl/bypass4netns=true\" label on containers, e.g., \`nerdctl run --label nerdctl/bypass4netns=true\`"
+	INFO "To use bypass4netnsd, set the \"nerdctl/bypass4netns=true\" annotation on containers, e.g., \`nerdctl run --annotation nerdctl/bypass4netns=true\`"
 }
 
 # CLI subcommand: "install-fuse-overlayfs"

pkg/api/types/container_types.go

@@ -217,9 +217,12 @@ type ContainerCreateOptions struct {
 	// Name assign a name to the container
 	Name string
 	// Label set meta data on a container
+	// (not passed through to the OCI runtime since nerdctl v2.0, with an exception for "nerdctl/bypass4netns")
 	Label []string
 	// LabelFile read in a line delimited file of labels
 	LabelFile []string
+	// Annotations set meta data on a container (passed through to the OCI runtime)
+	Annotations []string
 	// CidFile write the container ID to the file
 	CidFile string
 	// PidFile specifies the file path to write the task's pid. The CLI syntax conforms to Podman convention.

pkg/bypass4netnsutil/bypass4netnsutil.go

@@ -46,8 +46,8 @@ func generateSecurityOpt(listenerPath string) (oci.SpecOpts, error) {
 	return opt, nil
 }
 
-func GenerateBypass4netnsOpts(securityOptsMaps map[string]string, labelMaps map[string]string, id string) ([]oci.SpecOpts, error) {
-	b4nn, ok := labelMaps[labels.Bypass4netns]
+func GenerateBypass4netnsOpts(securityOptsMaps map[string]string, annotations map[string]string, id string) ([]oci.SpecOpts, error) {
+	b4nn, ok := annotations[labels.Bypass4netns]
 	if !ok {
 		return nil, nil
 	}

pkg/cmd/container/create.go

@@ -276,13 +276,15 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa
 		}
 	}
 
+	// TODO: abolish internal labels and only use annotations
 	ilOpt, err := withInternalLabels(internalLabels)
 	if err != nil {
 		return nil, nil, err
 	}
 	cOpts = append(cOpts, ilOpt)
 
-	opts = append(opts, propagateContainerdLabelsToOCIAnnotations())
+	opts = append(opts, propagateInternalContainerdLabelsToOCIAnnotations(),
+		oci.WithAnnotations(strutil.ConvertKVStringsToMap(options.Annotations)))
 
 	var s specs.Spec
 	spec := containerd.WithSpec(&s, opts...)
@@ -506,6 +508,13 @@ func withContainerLabels(label, labelFile []string) ([]containerd.NewContainerOp
 	if err != nil {
 		return nil, err
 	}
+	for k := range labelMap {
+		if k == labels.Bypass4netns {
+			log.L.Warnf("Label %q is deprecated, use an annotation instead", k)
+		} else if strings.HasPrefix(k, labels.Prefix) {
+			return nil, fmt.Errorf("internal label %q must not be specified manually", k)
+		}
+	}
 	o := containerd.WithAdditionalContainerLabels(labelMap)
 	return []containerd.NewContainerOpts{o}, nil
 }
@@ -704,9 +713,15 @@ func processeds(mountPoints []dockercompat.MountPoint) []*mountutil.Processed {
 	return result
 }
 
-func propagateContainerdLabelsToOCIAnnotations() oci.SpecOpts {
+func propagateInternalContainerdLabelsToOCIAnnotations() oci.SpecOpts {
 	return func(ctx context.Context, oc oci.Client, c *containers.Container, s *oci.Spec) error {
-		return oci.WithAnnotations(c.Labels)(ctx, oc, c, s)
+		allowed := make(map[string]string)
+		for k, v := range c.Labels {
+			if strings.Contains(k, labels.Prefix) {
+				allowed[k] = v
+			}
+		}
+		return oci.WithAnnotations(allowed)(ctx, oc, c, s)
 	}
 }
 

pkg/cmd/container/run_linux.go

@@ -59,10 +59,7 @@ func setPlatformOptions(ctx context.Context, client *containerd.Client, id, uts
 	}
 	opts = append(opts, cgOpts...)
 
-	labelsMap, err := readKVStringsMapfFromLabel(options.Label, options.LabelFile)
-	if err != nil {
-		return nil, err
-	}
+	annotations := strutil.ConvertKVStringsToMap(options.Annotations)
 
 	capOpts, err := generateCapOpts(
 		strutil.DedupeStrSlice(options.CapAdd),
@@ -78,7 +75,7 @@ func setPlatformOptions(ctx context.Context, client *containerd.Client, id, uts
 	}
 	opts = append(opts, secOpts...)
 
-	b4nnOpts, err := bypass4netnsutil.GenerateBypass4netnsOpts(securityOptsMaps, labelsMap, id)
+	b4nnOpts, err := bypass4netnsutil.GenerateBypass4netnsOpts(securityOptsMaps, annotations, id)
 	if err != nil {
 		return nil, err
 	}

pkg/composer/serviceparser/serviceparser.go

@@ -54,6 +54,7 @@ const Separator = "-"
 func warnUnknownFields(svc types.ServiceConfig) {
 	if unknown := reflectutil.UnknownNonEmptyFields(&svc,
 		"Name",
+		"Annotations",
 		"Build",
 		"BlkioConfig",
 		"CapAdd",
@@ -477,6 +478,14 @@ func newContainer(project *types.Project, parsed *Service, i int) (*Container, e
 		"--pull=never", // because image will be ensured before running replicas with `nerdctl run`.
 	}
 
+	for k, v := range svc.Annotations {
+		if v == "" {
+			c.RunArgs = append(c.RunArgs, fmt.Sprintf("--annotation=%s", k))
+		} else {
+			c.RunArgs = append(c.RunArgs, fmt.Sprintf("--annotation=%s=%s", k, v))
+		}
+	}
+
 	if svc.BlkioConfig != nil && svc.BlkioConfig.Weight != 0 {
 		c.RunArgs = append(c.RunArgs, fmt.Sprintf("--blkio-weight=%d", svc.BlkioConfig.Weight))
 	}

pkg/labels/labels.go

@@ -15,7 +15,7 @@
 */
 
 // Package labels defines labels that are set to containerd containers as labels.
-// The labels are also passed to OCI containers as annotations.
+// The labels defined in this package are also passed to OCI containers as annotations.
 package labels
 
 const (
@@ -107,8 +107,8 @@ const (
 	NerdctlDefaultNetwork = Prefix + "default-network"
 )
 
-var ShellCompletions = []string{
+var AnnotationShellCompletions = []string{
 	Bypass4netns + "=true",
 	Bypass4netns + "=false",
-	// Other labels should not be set via CLI
+	// Other annotations should not be set via CLI
 }