Support gzip compressed image layers

Update buildpack support to allow gzip compressed image layers to be
used when returned by the Docker engine. This update is restores
buildpack support when using Docker Desktop with the "Use containerd
for pulling and storing images" option enabled.

This commit introduces a new `ExportedImageTar` class to deal with the
intricacies of determining the mimetype of a layer. The class deals with
the parsing of `index.json' and related manifest blobs in order to
obtain layer information. The legacy `manifest.json` format is also
supported should `index.json` be missing.

Tests have been added to ensure that export archives from Docker Engine,
Docker Desktop (with and without containerd), and Podman can be used.

Fixes gh-40100

Co-authored-by: Moritz Halbritter <[email protected]>
Co-authored-by: Scott Frederick <[email protected]>

Author: 3 people

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/Builder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2023 the original author or authors.
+ * Copyright 2012-2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@
 package org.springframework.boot.buildpack.platform.build;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.List;
 import java.util.function.Consumer;
 
@@ -33,6 +32,7 @@
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
+import org.springframework.boot.buildpack.platform.io.TarArchive;
 import org.springframework.util.Assert;
 import org.springframework.util.StringUtils;
 
@@ -273,8 +273,9 @@ public Image fetchImage(ImageReference reference, ImageType imageType) throws IO
 		}
 
 		@Override
-		public void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
-			Builder.this.docker.image().exportLayerFiles(reference, exports);
+		public void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
+				throws IOException {
+			Builder.this.docker.image().exportLayers(reference, exports);
 		}
 
 	}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/BuildpackResolverContext.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2023 the original author or authors.
+ * Copyright 2012-2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,12 +17,12 @@
 package org.springframework.boot.buildpack.platform.build;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.List;
 
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
+import org.springframework.boot.buildpack.platform.io.TarArchive;
 
 /**
  * Context passed to a {@link BuildpackResolver}.
@@ -52,6 +52,6 @@ interface BuildpackResolverContext {
 	 * during the callback)
 	 * @throws IOException on IO error
 	 */
-	void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException;
+	void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports) throws IOException;
 
 }

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/ImageBuildpack.java

@@ -36,6 +36,7 @@
 import org.springframework.boot.buildpack.platform.docker.type.Layer;
 import org.springframework.boot.buildpack.platform.docker.type.LayerId;
 import org.springframework.boot.buildpack.platform.io.IOConsumer;
+import org.springframework.boot.buildpack.platform.io.TarArchive;
 import org.springframework.util.StreamUtils;
 
 /**
@@ -115,31 +116,31 @@ private static class ExportedLayers {
 
 		ExportedLayers(BuildpackResolverContext context, ImageReference imageReference) throws IOException {
 			List<Path> layerFiles = new ArrayList<>();
-			context.exportImageLayers(imageReference, (name, path) -> layerFiles.add(copyToTemp(path)));
+			context.exportImageLayers(imageReference,
+					(name, tarArchive) -> layerFiles.add(createLayerFile(tarArchive)));
 			this.layerFiles = Collections.unmodifiableList(layerFiles);
 		}
 
-		private Path copyToTemp(Path path) throws IOException {
-			Path outputPath = Files.createTempFile("create-builder-scratch-", null);
-			try (OutputStream out = Files.newOutputStream(outputPath)) {
-				copyLayerTar(path, out);
+		private Path createLayerFile(TarArchive tarArchive) throws IOException {
+			Path sourceTarFile = Files.createTempFile("create-builder-scratch-source-", null);
+			try (OutputStream out = Files.newOutputStream(sourceTarFile)) {
+				tarArchive.writeTo(out);
 			}
-			return outputPath;
-		}
-
-		private void copyLayerTar(Path path, OutputStream out) throws IOException {
-			try (TarArchiveInputStream tarIn = new TarArchiveInputStream(Files.newInputStream(path));
-					TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
-				tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
-				TarArchiveEntry entry = tarIn.getNextTarEntry();
-				while (entry != null) {
-					tarOut.putArchiveEntry(entry);
-					StreamUtils.copy(tarIn, tarOut);
-					tarOut.closeArchiveEntry();
-					entry = tarIn.getNextTarEntry();
+			Path layerFile = Files.createTempFile("create-builder-scratch-", null);
+			try (TarArchiveOutputStream out = new TarArchiveOutputStream(Files.newOutputStream(layerFile))) {
+				try (TarArchiveInputStream in = new TarArchiveInputStream(Files.newInputStream(sourceTarFile))) {
+					out.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+					TarArchiveEntry entry = in.getNextTarEntry();
+					while (entry != null) {
+						out.putArchiveEntry(entry);
+						StreamUtils.copy(in, out);
+						out.closeArchiveEntry();
+						entry = in.getNextTarEntry();
+					}
+					out.finish();
 				}
-				tarOut.finish();
 			}
+			return layerFile;
 		}
 
 		void apply(IOConsumer<Layer> layers) throws IOException {

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/DockerApi.java

@@ -16,27 +16,18 @@
 
 package org.springframework.boot.buildpack.platform.docker;
 
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
-import java.util.stream.Collectors;
 
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.hc.core5.net.URIBuilder;
 
 import org.springframework.boot.buildpack.platform.docker.configuration.DockerConfiguration.DockerHostConfiguration;
@@ -48,15 +39,13 @@
 import org.springframework.boot.buildpack.platform.docker.type.ContainerStatus;
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageArchive;
-import org.springframework.boot.buildpack.platform.docker.type.ImageArchiveManifest;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.docker.type.VolumeName;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
 import org.springframework.boot.buildpack.platform.io.TarArchive;
 import org.springframework.boot.buildpack.platform.json.JsonStream;
 import org.springframework.boot.buildpack.platform.json.SharedObjectMapper;
 import org.springframework.util.Assert;
-import org.springframework.util.StreamUtils;
 import org.springframework.util.StringUtils;
 
 /**
@@ -263,49 +252,50 @@ public void load(ImageArchive archive, UpdateListener<LoadImageUpdateEvent> list
 		}
 
 		/**
-		 * Export the layers of an image as {@link TarArchive}s.
+		 * Export the layers of an image as paths to layer tar files.
 		 * @param reference the reference to export
-		 * @param exports a consumer to receive the layers (contents can only be accessed
-		 * during the callback)
+		 * @param exports a consumer to receive the layer tar file paths (file can only be
+		 * accessed during the callback)
 		 * @throws IOException on IO error
+		 * @since 2.7.10
+		 * @deprecated since 3.2.6 for removal in 3.5.0 in favor of
+		 * {@link #exportLayers(ImageReference, IOBiConsumer)}
 		 */
-		public void exportLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
-				throws IOException {
-			exportLayerFiles(reference, (name, path) -> {
-				try (InputStream in = Files.newInputStream(path)) {
-					TarArchive archive = (out) -> StreamUtils.copy(in, out);
-					exports.accept(name, archive);
+		@Deprecated(since = "3.2.6", forRemoval = true)
+		public void exportLayerFiles(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
+			Assert.notNull(reference, "Reference must not be null");
+			Assert.notNull(exports, "Exports must not be null");
+			exportLayers(reference, (name, archive) -> {
+				Path path = Files.createTempFile("docker-export-layer-files-", null);
+				try {
+					try (OutputStream out = Files.newOutputStream(path)) {
+						archive.writeTo(out);
+						exports.accept(name, path);
+					}
+				}
+				finally {
+					Files.delete(path);
 				}
 			});
 		}
 
 		/**
-		 * Export the layers of an image as paths to layer tar files.
+		 * Export the layers of an image as {@link TarArchive TarArchives}.
 		 * @param reference the reference to export
-		 * @param exports a consumer to receive the layer tar file paths (file can only be
-		 * accessed during the callback)
+		 * @param exports a consumer to receive the layers (contents can only be accessed
+		 * during the callback)
 		 * @throws IOException on IO error
-		 * @since 2.7.10
 		 */
-		public void exportLayerFiles(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
+		public void exportLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
+				throws IOException {
 			Assert.notNull(reference, "Reference must not be null");
 			Assert.notNull(exports, "Exports must not be null");
-			URI saveUri = buildUrl("/images/" + reference + "/get");
-			Response response = http().get(saveUri);
-			Path exportFile = copyToTemp(response.getContent());
-			ImageArchiveManifest manifest = getManifest(reference, exportFile);
-			try (TarArchiveInputStream tar = new TarArchiveInputStream(new FileInputStream(exportFile.toFile()))) {
-				TarArchiveEntry entry = tar.getNextTarEntry();
-				while (entry != null) {
-					if (manifestContainsLayerEntry(manifest, entry.getName())) {
-						Path layerFile = copyToTemp(tar);
-						exports.accept(entry.getName(), layerFile);
-						Files.delete(layerFile);
-					}
-					entry = tar.getNextTarEntry();
+			URI uri = buildUrl("/images/" + reference + "/get");
+			try (Response response = http().get(uri)) {
+				try (ExportedImageTar exportedImageTar = new ExportedImageTar(reference, response.getContent())) {
+					exportedImageTar.exportLayers(exports);
 				}
 			}
-			Files.delete(exportFile);
 		}
 
 		/**
@@ -345,37 +335,6 @@ public void tag(ImageReference sourceReference, ImageReference targetReference)
 			http().post(uri).close();
 		}
 
-		private ImageArchiveManifest getManifest(ImageReference reference, Path exportFile) throws IOException {
-			try (TarArchiveInputStream tar = new TarArchiveInputStream(new FileInputStream(exportFile.toFile()))) {
-				TarArchiveEntry entry = tar.getNextTarEntry();
-				while (entry != null) {
-					if (entry.getName().equals("manifest.json")) {
-						return readManifest(tar);
-					}
-					entry = tar.getNextTarEntry();
-				}
-			}
-			throw new IllegalArgumentException("Manifest not found in image " + reference);
-		}
-
-		private ImageArchiveManifest readManifest(TarArchiveInputStream tar) throws IOException {
-			String manifestContent = new BufferedReader(new InputStreamReader(tar, StandardCharsets.UTF_8)).lines()
-				.collect(Collectors.joining());
-			return ImageArchiveManifest.of(new ByteArrayInputStream(manifestContent.getBytes(StandardCharsets.UTF_8)));
-		}
-
-		private Path copyToTemp(InputStream in) throws IOException {
-			Path path = Files.createTempFile("create-builder-scratch-", null);
-			try (OutputStream out = Files.newOutputStream(path)) {
-				StreamUtils.copy(in, out);
-			}
-			return path;
-		}
-
-		private boolean manifestContainsLayerEntry(ImageArchiveManifest manifest, String layerId) {
-			return manifest.getEntries().stream().anyMatch((content) -> content.getLayers().contains(layerId));
-		}
-
 	}
 
 	/**
@@ -458,8 +417,9 @@ public void logs(ContainerReference reference, UpdateListener<LogUpdateEvent> li
 		public ContainerStatus wait(ContainerReference reference) throws IOException {
 			Assert.notNull(reference, "Reference must not be null");
 			URI uri = buildUrl("/containers/" + reference + "/wait");
-			Response response = http().post(uri);
-			return ContainerStatus.of(response.getContent());
+			try (Response response = http().post(uri)) {
+				return ContainerStatus.of(response.getContent());
+			}
 		}
 
 		/**