archive/zip: add File.OpenRaw, Writer.CreateRaw, Writer.Copy

These new methods provide support for cases where performance is a
primary concern. For example, copying files from an existing zip to a
new zip without incurring the decompression and compression overhead.
Using an optimized, external compression method and writing the output
to a zip archive. And compressing file contents in parallel and then
sequentially writing the compressed bytes to a zip archive.

TestWriterCopy is copied verbatim from https://github.com/rsc/zipmerge

Fixes #34974

Change-Id: Iade5bc245ba34cdbb86364bf59f79f38bb9e2eb6
Reviewed-on: https://go-review.googlesource.com/c/go/+/312310
Run-TryBot: Ian Lance Taylor <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Ian Lance Taylor <[email protected]>
Trust: Carlos Amedee <[email protected]>

Author: escholtz

src/archive/zip/reader.go

@@ -52,12 +52,9 @@ type File struct {
 	FileHeader
 	zip          *Reader
 	zipr         io.ReaderAt
-	zipsize      int64
 	headerOffset int64
-}
-
-func (f *File) hasDataDescriptor() bool {
-	return f.Flags&0x8 != 0
+	zip64        bool  // zip64 extended information extra field presence
+	descErr      error // error reading the data descriptor during init
 }
 
 // OpenReader will open the Zip file specified by name and return a ReadCloser.
@@ -112,14 +109,15 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
 	// a bad one, and then only report an ErrFormat or UnexpectedEOF if
 	// the file count modulo 65536 is incorrect.
 	for {
-		f := &File{zip: z, zipr: r, zipsize: size}
+		f := &File{zip: z, zipr: r}
 		err = readDirectoryHeader(f, buf)
 		if err == ErrFormat || err == io.ErrUnexpectedEOF {
 			break
 		}
 		if err != nil {
 			return err
 		}
+		f.readDataDescriptor()
 		z.File = append(z.File, f)
 	}
 	if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here
@@ -180,26 +178,68 @@ func (f *File) Open() (io.ReadCloser, error) {
 		return nil, ErrAlgorithm
 	}
 	var rc io.ReadCloser = dcomp(r)
-	var desr io.Reader
-	if f.hasDataDescriptor() {
-		desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen)
-	}
 	rc = &checksumReader{
 		rc:   rc,
 		hash: crc32.NewIEEE(),
 		f:    f,
-		desr: desr,
 	}
 	return rc, nil
 }
 
+// OpenRaw returns a Reader that provides access to the File's contents without
+// decompression.
+func (f *File) OpenRaw() (io.Reader, error) {
+	bodyOffset, err := f.findBodyOffset()
+	if err != nil {
+		return nil, err
+	}
+	r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, int64(f.CompressedSize64))
+	return r, nil
+}
+
+func (f *File) readDataDescriptor() {
+	if !f.hasDataDescriptor() {
+		return
+	}
+
+	bodyOffset, err := f.findBodyOffset()
+	if err != nil {
+		f.descErr = err
+		return
+	}
+
+	// In section 4.3.9.2 of the spec: "However ZIP64 format MAY be used
+	// regardless of the size of a file.  When extracting, if the zip64
+	// extended information extra field is present for the file the
+	// compressed and uncompressed sizes will be 8 byte values."
+	//
+	// Historically, this package has used the compressed and uncompressed
+	// sizes from the central directory to determine if the package is
+	// zip64.
+	//
+	// For this case we allow either the extra field or sizes to determine
+	// the data descriptor length.
+	zip64 := f.zip64 || f.isZip64()
+	n := int64(dataDescriptorLen)
+	if zip64 {
+		n = dataDescriptor64Len
+	}
+	size := int64(f.CompressedSize64)
+	r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, n)
+	dd, err := readDataDescriptor(r, zip64)
+	if err != nil {
+		f.descErr = err
+		return
+	}
+	f.CRC32 = dd.crc32
+}
+
 type checksumReader struct {
 	rc    io.ReadCloser
 	hash  hash.Hash32
 	nread uint64 // number of bytes read so far
 	f     *File
-	desr  io.Reader // if non-nil, where to read the data descriptor
-	err   error     // sticky error
+	err   error // sticky error
 }
 
 func (r *checksumReader) Stat() (fs.FileInfo, error) {
@@ -220,12 +260,12 @@ func (r *checksumReader) Read(b []byte) (n int, err error) {
 		if r.nread != r.f.UncompressedSize64 {
 			return 0, io.ErrUnexpectedEOF
 		}
-		if r.desr != nil {
-			if err1 := readDataDescriptor(r.desr, r.f); err1 != nil {
-				if err1 == io.EOF {
+		if r.f.hasDataDescriptor() {
+			if r.f.descErr != nil {
+				if r.f.descErr == io.EOF {
 					err = io.ErrUnexpectedEOF
 				} else {
-					err = err1
+					err = r.f.descErr
 				}
 			} else if r.hash.Sum32() != r.f.CRC32 {
 				err = ErrChecksum
@@ -336,6 +376,8 @@ parseExtras:
 
 		switch fieldTag {
 		case zip64ExtraID:
+			f.zip64 = true
+
 			// update directory values from the zip64 extra block.
 			// They should only be consulted if the sizes read earlier
 			// are maxed out.
@@ -435,8 +477,9 @@ parseExtras:
 	return nil
 }
 
-func readDataDescriptor(r io.Reader, f *File) error {
-	var buf [dataDescriptorLen]byte
+func readDataDescriptor(r io.Reader, zip64 bool) (*dataDescriptor, error) {
+	// Create enough space for the largest possible size
+	var buf [dataDescriptor64Len]byte
 
 	// The spec says: "Although not originally assigned a
 	// signature, the value 0x08074b50 has commonly been adopted
@@ -446,10 +489,9 @@ func readDataDescriptor(r io.Reader, f *File) error {
 	// descriptors and should account for either case when reading
 	// ZIP files to ensure compatibility."
 	//
-	// dataDescriptorLen includes the size of the signature but
-	// first read just those 4 bytes to see if it exists.
+	// First read just those 4 bytes to see if the signature exists.
 	if _, err := io.ReadFull(r, buf[:4]); err != nil {
-		return err
+		return nil, err
 	}
 	off := 0
 	maybeSig := readBuf(buf[:4])
@@ -458,21 +500,28 @@ func readDataDescriptor(r io.Reader, f *File) error {
 		// bytes.
 		off += 4
 	}
-	if _, err := io.ReadFull(r, buf[off:12]); err != nil {
-		return err
+
+	end := dataDescriptorLen - 4
+	if zip64 {
+		end = dataDescriptor64Len - 4
 	}
-	b := readBuf(buf[:12])
-	if b.uint32() != f.CRC32 {
-		return ErrChecksum
+	if _, err := io.ReadFull(r, buf[off:end]); err != nil {
+		return nil, err
 	}
+	b := readBuf(buf[:end])
 
-	// The two sizes that follow here can be either 32 bits or 64 bits
-	// but the spec is not very clear on this and different
-	// interpretations has been made causing incompatibilities. We
-	// already have the sizes from the central directory so we can
-	// just ignore these.
+	out := &dataDescriptor{
+		crc32: b.uint32(),
+	}
 
-	return nil
+	if zip64 {
+		out.compressedSize = b.uint64()
+		out.uncompressedSize = b.uint64()
+	} else {
+		out.compressedSize = uint64(b.uint32())
+		out.uncompressedSize = uint64(b.uint32())
+	}
+	return out, nil
 }
 
 func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error) {