Skip to content

Commit 0a87fa5

Browse files
authored
upgrade deps + interoperable uvarint delimited writer/reader. (#20)
1 parent 808ef98 commit 0a87fa5

File tree

6 files changed

+411
-3
lines changed

6 files changed

+411
-3
lines changed

go.mod

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
11
module github.com/libp2p/go-msgio
22

3-
require github.com/libp2p/go-buffer-pool v0.0.2
3+
go 1.14
4+
5+
require (
6+
github.com/gogo/protobuf v1.3.1
7+
github.com/libp2p/go-buffer-pool v0.0.2
8+
github.com/multiformats/go-varint v0.0.6
9+
)

go.sum

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
github.com/libp2p/go-buffer-pool v0.0.1 h1:9Rrn/H46cXjaA2HQ5Y8lyhOS1NhTkZ4yuEs2r3Eechg=
2-
github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
1+
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
2+
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
3+
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
4+
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
35
github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs=
46
github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
7+
github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
8+
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
9+
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

protoio/interfaces.go

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
//
2+
// Adapted from gogo/protobuf to use multiformats/go-varint for
3+
// efficient, interoperable length-prefixing.
4+
//
5+
// Protocol Buffers for Go with Gadgets
6+
//
7+
// Copyright (c) 2013, The GoGo Authors. All rights reserved.
8+
// http://github.com/gogo/protobuf
9+
//
10+
// Redistribution and use in source and binary forms, with or without
11+
// modification, are permitted provided that the following conditions are
12+
// met:
13+
//
14+
// * Redistributions of source code must retain the above copyright
15+
// notice, this list of conditions and the following disclaimer.
16+
// * Redistributions in binary form must reproduce the above
17+
// copyright notice, this list of conditions and the following disclaimer
18+
// in the documentation and/or other materials provided with the
19+
// distribution.
20+
//
21+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32+
//
33+
package protoio
34+
35+
import (
36+
"io"
37+
38+
"github.com/gogo/protobuf/proto"
39+
)
40+
41+
type Writer interface {
42+
WriteMsg(proto.Message) error
43+
}
44+
45+
type WriteCloser interface {
46+
Writer
47+
io.Closer
48+
}
49+
50+
type Reader interface {
51+
ReadMsg(msg proto.Message) error
52+
}
53+
54+
type ReadCloser interface {
55+
Reader
56+
io.Closer
57+
}
58+
59+
func getSize(v interface{}) (int, bool) {
60+
if sz, ok := v.(interface {
61+
Size() (n int)
62+
}); ok {
63+
return sz.Size(), true
64+
} else if sz, ok := v.(interface {
65+
ProtoSize() (n int)
66+
}); ok {
67+
return sz.ProtoSize(), true
68+
} else {
69+
return 0, false
70+
}
71+
}

protoio/uvarint_reader.go

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
//
2+
// Adapted from gogo/protobuf to use multiformats/go-varint for
3+
// efficient, interoperable length-prefixing.
4+
//
5+
// Protocol Buffers for Go with Gadgets
6+
//
7+
// Copyright (c) 2013, The GoGo Authors. All rights reserved.
8+
// http://github.com/gogo/protobuf
9+
//
10+
// Redistribution and use in source and binary forms, with or without
11+
// modification, are permitted provided that the following conditions are
12+
// met:
13+
//
14+
// * Redistributions of source code must retain the above copyright
15+
// notice, this list of conditions and the following disclaimer.
16+
// * Redistributions in binary form must reproduce the above
17+
// copyright notice, this list of conditions and the following disclaimer
18+
// in the documentation and/or other materials provided with the
19+
// distribution.
20+
//
21+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32+
//
33+
package protoio
34+
35+
import (
36+
"bufio"
37+
"io"
38+
39+
"github.com/gogo/protobuf/proto"
40+
41+
"github.com/multiformats/go-varint"
42+
)
43+
44+
type uvarintReader struct {
45+
r *bufio.Reader
46+
buf []byte
47+
maxSize int
48+
closer io.Closer
49+
}
50+
51+
func NewDelimitedReader(r io.Reader, maxSize int) ReadCloser {
52+
var closer io.Closer
53+
if c, ok := r.(io.Closer); ok {
54+
closer = c
55+
}
56+
return &uvarintReader{bufio.NewReader(r), nil, maxSize, closer}
57+
}
58+
59+
func (ur *uvarintReader) ReadMsg(msg proto.Message) error {
60+
length64, err := varint.ReadUvarint(ur.r)
61+
if err != nil {
62+
return err
63+
}
64+
length := int(length64)
65+
if length < 0 || length > ur.maxSize {
66+
return io.ErrShortBuffer
67+
}
68+
if len(ur.buf) < length {
69+
ur.buf = make([]byte, length)
70+
}
71+
buf := ur.buf[:length]
72+
if _, err := io.ReadFull(ur.r, buf); err != nil {
73+
return err
74+
}
75+
return proto.Unmarshal(buf, msg)
76+
}
77+
78+
func (ur *uvarintReader) Close() error {
79+
if ur.closer != nil {
80+
return ur.closer.Close()
81+
}
82+
return nil
83+
}

protoio/uvarint_test.go

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
//
2+
// Adapted from gogo/protobuf to use multiformats/go-varint for
3+
// efficient, interoperable length-prefixing.
4+
//
5+
// Protocol Buffers for Go with Gadgets
6+
//
7+
// Copyright (c) 2013, The GoGo Authors. All rights reserved.
8+
// http://github.com/gogo/protobuf
9+
//
10+
// Redistribution and use in source and binary forms, with or without
11+
// modification, are permitted provided that the following conditions are
12+
// met:
13+
//
14+
// * Redistributions of source code must retain the above copyright
15+
// notice, this list of conditions and the following disclaimer.
16+
// * Redistributions in binary form must reproduce the above
17+
// copyright notice, this list of conditions and the following disclaimer
18+
// in the documentation and/or other materials provided with the
19+
// distribution.
20+
//
21+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32+
//
33+
package protoio_test
34+
35+
import (
36+
"bytes"
37+
"io"
38+
"math/rand"
39+
"testing"
40+
"time"
41+
42+
"github.com/gogo/protobuf/test"
43+
44+
"github.com/libp2p/go-msgio/protoio"
45+
"github.com/multiformats/go-varint"
46+
)
47+
48+
func TestVarintNormal(t *testing.T) {
49+
buf := newBuffer()
50+
writer := protoio.NewDelimitedWriter(buf)
51+
reader := protoio.NewDelimitedReader(buf, 1024*1024)
52+
if err := iotest(writer, reader); err != nil {
53+
t.Error(err)
54+
}
55+
if !buf.closed {
56+
t.Fatalf("did not close buffer")
57+
}
58+
}
59+
60+
func TestVarintNoClose(t *testing.T) {
61+
buf := bytes.NewBuffer(nil)
62+
writer := protoio.NewDelimitedWriter(buf)
63+
reader := protoio.NewDelimitedReader(buf, 1024*1024)
64+
if err := iotest(writer, reader); err != nil {
65+
t.Error(err)
66+
}
67+
}
68+
69+
// https://github.com/gogo/protobuf/issues/32
70+
func TestVarintMaxSize(t *testing.T) {
71+
buf := newBuffer()
72+
writer := protoio.NewDelimitedWriter(buf)
73+
reader := protoio.NewDelimitedReader(buf, 20)
74+
if err := iotest(writer, reader); err != io.ErrShortBuffer {
75+
t.Error(err)
76+
} else {
77+
t.Logf("%s", err)
78+
}
79+
}
80+
81+
func TestVarintError(t *testing.T) {
82+
buf := newBuffer()
83+
// beyond uvarint63 capacity.
84+
buf.Write([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff})
85+
reader := protoio.NewDelimitedReader(buf, 1024*1024)
86+
msg := &test.NinOptNative{}
87+
err := reader.ReadMsg(msg)
88+
if err != varint.ErrOverflow {
89+
t.Fatalf("expected varint.ErrOverflow error")
90+
}
91+
}
92+
93+
type buffer struct {
94+
*bytes.Buffer
95+
closed bool
96+
}
97+
98+
func (b *buffer) Close() error {
99+
b.closed = true
100+
return nil
101+
}
102+
103+
func newBuffer() *buffer {
104+
return &buffer{bytes.NewBuffer(nil), false}
105+
}
106+
107+
func iotest(writer protoio.WriteCloser, reader protoio.ReadCloser) error {
108+
size := 1000
109+
msgs := make([]*test.NinOptNative, size)
110+
r := rand.New(rand.NewSource(time.Now().UnixNano()))
111+
for i := range msgs {
112+
msgs[i] = test.NewPopulatedNinOptNative(r, true)
113+
// https://github.com/gogo/protobuf/issues/31
114+
if i == 5 {
115+
msgs[i] = &test.NinOptNative{}
116+
}
117+
// https://github.com/gogo/protobuf/issues/31
118+
if i == 999 {
119+
msgs[i] = &test.NinOptNative{}
120+
}
121+
err := writer.WriteMsg(msgs[i])
122+
if err != nil {
123+
return err
124+
}
125+
}
126+
if err := writer.Close(); err != nil {
127+
return err
128+
}
129+
i := 0
130+
for {
131+
msg := &test.NinOptNative{}
132+
if err := reader.ReadMsg(msg); err != nil {
133+
if err == io.EOF {
134+
break
135+
}
136+
return err
137+
}
138+
if err := msg.VerboseEqual(msgs[i]); err != nil {
139+
return err
140+
}
141+
i++
142+
}
143+
if i != size {
144+
panic("not enough messages read")
145+
}
146+
if err := reader.Close(); err != nil {
147+
return err
148+
}
149+
return nil
150+
}

0 commit comments

Comments
 (0)