format: idxfile, support for >2Gb packfiles

plumbing/format/idxfile/decoder.go

@@ -123,6 +123,7 @@ func readCRC32(idx *Idxfile, r io.Reader) error {
 
 func readOffsets(idx *Idxfile, r io.Reader) error {
 	c := int(idx.ObjectCount)
+
 	for i := 0; i < c; i++ {
 		o, err := binary.ReadUint32(r)
 		if err != nil {
@@ -132,6 +133,19 @@ func readOffsets(idx *Idxfile, r io.Reader) error {
 		idx.Entries[i].Offset = uint64(o)
 	}
 
+	for i := 0; i < c; i++ {
+		if idx.Entries[i].Offset <= offsetLimit {
+			continue
+		}
+
+		o, err := binary.ReadUint64(r)
+		if err != nil {
+			return err
+		}
+
+		idx.Entries[i].Offset = o
+	}
+
 	return nil
 }
 

plumbing/format/idxfile/decoder_test.go

@@ -2,6 +2,7 @@ package idxfile_test
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
 	"testing"
 
@@ -65,3 +66,76 @@ func (s *IdxfileSuite) TestDecodeCRCs(c *C) {
 
 	c.Assert(idx.Entries, DeepEquals, i.Entries)
 }
+
+func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) {
+	f := bytes.NewBufferString(fixtureLarge4GB)
+
+	idx := &Idxfile{}
+
+	d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+	err := d.Decode(idx)
+	c.Assert(err, IsNil)
+
+	expected := map[string]uint64{
+		"303953e5aa461c203a324821bc1717f9b4fff895": 12,
+		"5296768e3d9f661387ccbff18c4dea6c997fd78c": 142,
+		"03fc8d58d44267274edef4585eaeeb445879d33f": 1601322837,
+		"8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772": 2646996529,
+		"e0d1d625010087f79c9e01ad9d8f95e1628dda02": 3452385606,
+		"90eba326cdc4d1d61c5ad25224ccbf08731dd041": 3707047470,
+		"bab53055add7bc35882758a922c54a874d6b1272": 5323223332,
+		"1b8995f51987d8a449ca5ea4356595102dc2fbd4": 5894072943,
+		"35858be9c6f5914cbe6768489c41eb6809a2bceb": 5924278919,
+	}
+
+	for _, e := range idx.Entries {
+		c.Assert(expected[e.Hash.String()], Equals, e.Offset)
+	}
+}
+
+func (s *IdxfileSuite) TestDecode64bitsOffsetsIdempotent(c *C) {
+	f := bytes.NewBufferString(fixtureLarge4GB)
+
+	expected := &Idxfile{}
+
+	d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+	err := d.Decode(expected)
+	c.Assert(err, IsNil)
+
+	buf := bytes.NewBuffer(nil)
+	_, err = NewEncoder(buf).Encode(expected)
+	c.Assert(err, IsNil)
+
+	idx := &Idxfile{}
+	err = NewDecoder(buf).Decode(idx)
+	c.Assert(err, IsNil)
+
+	c.Assert(idx.Entries, DeepEquals, expected.Entries)
+}
+
+const fixtureLarge4GB = `/3RPYwAAAAIAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEA
+AAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAA
+AAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAA
+AgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAQAAAAE
+AAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQA
+AAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABQAA
+AAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAA
+BQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAF
+AAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUA
+AAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAA
+AAUAAAAFAAAABQAAAAYAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAA
+BwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAH
+AAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcA
+AAAHAAAABwAAAAcAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAA
+AAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAA
+CAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAkAAAAJ
+AAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkA
+AAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAA
+AAkAAAAJA/yNWNRCZydO3vRYXq7rRFh50z8biZX1GYfYpEnKXqQ1ZZUQLcL71DA5U+WqRhwgOjJI
+IbwXF/m0//iVNYWL6cb1kUy+Z2hInEHraAmivOtSlnaOPZ9mE4fMv/GMTepsmX/XjI88606ky55K
+D3UXletByaTwe+dykOujJs3E0dYcWtJSJMy/CHMd0EG6tTBVrde8NYgnWKkixUqHTWsScuDR1iUB
+AIf3nJ4BrZ2PleFijdoCkp36qiGHwFa8NHxMnInZ0s3CKEKmHe+KcZPzuqwmm44GvqGAX3I/VYAA
+AAAAAAAMgAAAAQAAAI6AAAACgAAAA4AAAASAAAAFAAAAAV9Qam8AAAABYR1ShwAAAACdxfYxAAAA
+ANz1Di4AAAABPUnxJAAAAADNxzlGr6vCJpIFz4XaG/fi/f9C9zgQ8ptKSQpfQ1NMJBGTDTxxYGGp
+ch2xUA==
+`

plumbing/format/idxfile/encoder.go

@@ -98,13 +98,28 @@ func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
 
 func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) {
 	sz := 0
+
+	var o64bits []uint64
 	for _, ent := range idx.Entries {
-		if err := binary.WriteUint32(e, uint32(ent.Offset)); err != nil {
+		o := ent.Offset
+		if o > offsetLimit {
+			o64bits = append(o64bits, o)
+			o = offsetLimit + uint64(len(o64bits))
+		}
+
+		if err := binary.WriteUint32(e, uint32(o)); err != nil {
 			return sz, err
 		}
 
 		sz += 4
+	}
+
+	for _, o := range o64bits {
+		if err := binary.WriteUint64(e, o); err != nil {
+			return sz, err
+		}
 
+		sz += 8
 	}
 
 	return sz, nil

plumbing/format/idxfile/idxfile.go

@@ -5,6 +5,8 @@ import "gopkg.in/src-d/go-git.v4/plumbing"
 const (
 	// VersionSupported is the only idx version supported.
 	VersionSupported = 2
+
+	offsetLimit = 0x7fffffff
 )
 
 var (

utils/binary/read.go

@@ -94,7 +94,17 @@ const (
 	lengthBits   = uint8(7)   // subsequent bytes has 7 bits to store the length
 )
 
-// ReadUint32 reads 4 bytes and returns them as a Big ndian uint32
+// ReadUint64 reads 8 bytes and returns them as a BigEndian uint32
+func ReadUint64(r io.Reader) (uint64, error) {
+	var v uint64
+	if err := binary.Read(r, binary.BigEndian, &v); err != nil {
+		return 0, err
+	}
+
+	return v, nil
+}
+
+// ReadUint32 reads 4 bytes and returns them as a BigEndian uint32
 func ReadUint32(r io.Reader) (uint32, error) {
 	var v uint32
 	if err := binary.Read(r, binary.BigEndian, &v); err != nil {

utils/binary/write.go

@@ -31,6 +31,12 @@ func WriteVariableWidthInt(w io.Writer, n int64) error {
 	return err
 }
 
+// WriteUint64 writes the binary representation of a uint64 into w, in BigEndian
+// order
+func WriteUint64(w io.Writer, value uint64) error {
+	return binary.Write(w, binary.BigEndian, value)
+}
+
 // WriteUint32 writes the binary representation of a uint32 into w, in BigEndian
 // order
 func WriteUint32(w io.Writer, value uint32) error {