Skip to content

Commit 2a93f90

Browse files
taowentaowen
committed
fix #244 use BinaryAsStringExtension to make []byte pretty, while the output is valid json, but it can not be decoded by other json codec, as \x01 is decoded as \x01 by them, which is not original input
1 parent 9472474 commit 2a93f90

File tree

3 files changed

+275
-0
lines changed

3 files changed

+275
-0
lines changed

Diff for: ‎extra/binary_as_string_codec.go

+238
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
package extra
2+
3+
import (
4+
"github.com/json-iterator/go"
5+
"unsafe"
6+
"unicode/utf8"
7+
"github.com/v2pro/plz/reflect2"
8+
)
9+
10+
// safeSet holds the value true if the ASCII character with the given array
11+
// position can be represented inside a JSON string without any further
12+
// escaping.
13+
//
14+
// All values are true except for the ASCII control characters (0-31), the
15+
// double quote ("), and the backslash character ("\").
16+
var safeSet = [utf8.RuneSelf]bool{
17+
' ': true,
18+
'!': true,
19+
'"': false,
20+
'#': true,
21+
'$': true,
22+
'%': true,
23+
'&': true,
24+
'\'': true,
25+
'(': true,
26+
')': true,
27+
'*': true,
28+
'+': true,
29+
',': true,
30+
'-': true,
31+
'.': true,
32+
'/': true,
33+
'0': true,
34+
'1': true,
35+
'2': true,
36+
'3': true,
37+
'4': true,
38+
'5': true,
39+
'6': true,
40+
'7': true,
41+
'8': true,
42+
'9': true,
43+
':': true,
44+
';': true,
45+
'<': true,
46+
'=': true,
47+
'>': true,
48+
'?': true,
49+
'@': true,
50+
'A': true,
51+
'B': true,
52+
'C': true,
53+
'D': true,
54+
'E': true,
55+
'F': true,
56+
'G': true,
57+
'H': true,
58+
'I': true,
59+
'J': true,
60+
'K': true,
61+
'L': true,
62+
'M': true,
63+
'N': true,
64+
'O': true,
65+
'P': true,
66+
'Q': true,
67+
'R': true,
68+
'S': true,
69+
'T': true,
70+
'U': true,
71+
'V': true,
72+
'W': true,
73+
'X': true,
74+
'Y': true,
75+
'Z': true,
76+
'[': true,
77+
'\\': false,
78+
']': true,
79+
'^': true,
80+
'_': true,
81+
'`': true,
82+
'a': true,
83+
'b': true,
84+
'c': true,
85+
'd': true,
86+
'e': true,
87+
'f': true,
88+
'g': true,
89+
'h': true,
90+
'i': true,
91+
'j': true,
92+
'k': true,
93+
'l': true,
94+
'm': true,
95+
'n': true,
96+
'o': true,
97+
'p': true,
98+
'q': true,
99+
'r': true,
100+
's': true,
101+
't': true,
102+
'u': true,
103+
'v': true,
104+
'w': true,
105+
'x': true,
106+
'y': true,
107+
'z': true,
108+
'{': true,
109+
'|': true,
110+
'}': true,
111+
'~': true,
112+
'\u007f': true,
113+
}
114+
115+
var binaryType = reflect2.TypeOfPtr((*[]byte)(nil)).Elem()
116+
117+
type BinaryAsStringExtension struct {
118+
jsoniter.DummyExtension
119+
}
120+
121+
func (extension *BinaryAsStringExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncoder {
122+
if typ == binaryType {
123+
return &binaryAsStringCodec{}
124+
}
125+
return nil
126+
}
127+
128+
func (extension *BinaryAsStringExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder {
129+
if typ == binaryType {
130+
return &binaryAsStringCodec{}
131+
}
132+
return nil
133+
}
134+
135+
type binaryAsStringCodec struct {
136+
}
137+
138+
func (codec *binaryAsStringCodec) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
139+
rawBytes := iter.ReadStringAsSlice()
140+
bytes := make([]byte, 0, len(rawBytes))
141+
for i := 0; i < len(rawBytes); i++ {
142+
b := rawBytes[i]
143+
if b == '\\' {
144+
b2 := rawBytes[i+1]
145+
if b2 != '\\' {
146+
iter.ReportError("decode binary as string", `\\x is only supported escape`)
147+
return
148+
}
149+
b3 := rawBytes[i+2]
150+
if b3 != 'x' {
151+
iter.ReportError("decode binary as string", `\\x is only supported escape`)
152+
return
153+
}
154+
b4 := rawBytes[i+3]
155+
b5 := rawBytes[i+4]
156+
i = i + 4
157+
b = readHex(iter, b4, b5)
158+
}
159+
bytes = append(bytes, b)
160+
}
161+
*(*[]byte)(ptr) = bytes
162+
}
163+
func (codec *binaryAsStringCodec) IsEmpty(ptr unsafe.Pointer) bool {
164+
return len(*((*[]byte)(ptr))) == 0
165+
}
166+
func (codec *binaryAsStringCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
167+
newBuffer := writeBytes(stream.Buffer(), *(*[]byte)(ptr))
168+
stream.SetBuffer(newBuffer)
169+
}
170+
171+
func readHex(iter *jsoniter.Iterator, b1, b2 byte) byte {
172+
var ret byte
173+
if b1 >= '0' && b1 <= '9' {
174+
ret = b1-'0'
175+
} else if b1 >= 'a' && b1 <= 'f' {
176+
ret = b1-'a'+10
177+
} else {
178+
iter.ReportError("read hex", "expects 0~9 or a~f, but found "+string([]byte{b1}))
179+
return 0
180+
}
181+
ret = ret * 16
182+
if b2 >= '0' && b2 <= '9' {
183+
ret = b2-'0'
184+
} else if b2 >= 'a' && b2 <= 'f' {
185+
ret = b2-'a'+10
186+
} else {
187+
iter.ReportError("read hex", "expects 0~9 or a~f, but found "+string([]byte{b2}))
188+
return 0
189+
}
190+
return ret
191+
}
192+
193+
var hex = "0123456789abcdef"
194+
195+
func writeBytes(space []byte, s []byte) []byte {
196+
space = append(space, '"')
197+
// write string, the fast path, without utf8 and escape support
198+
var i int
199+
var c byte
200+
for i, c = range s {
201+
if c < utf8.RuneSelf && safeSet[c] {
202+
space = append(space, c)
203+
} else {
204+
break
205+
}
206+
}
207+
if i == len(s)-1 {
208+
space = append(space, '"')
209+
return space
210+
}
211+
return writeBytesSlowPath(space, s[i:])
212+
}
213+
214+
func writeBytesSlowPath(space []byte, s []byte) []byte {
215+
start := 0
216+
// for the remaining parts, we process them char by char
217+
var i int
218+
var b byte
219+
for i, b = range s {
220+
if b >= utf8.RuneSelf {
221+
space = append(space, '\\', '\\', 'x', hex[b>>4], hex[b&0xF])
222+
start = i + 1
223+
continue
224+
}
225+
if safeSet[b] {
226+
continue
227+
}
228+
if start < i {
229+
space = append(space, s[start:i]...)
230+
}
231+
space = append(space, '\\', '\\', 'x', hex[b>>4], hex[b&0xF])
232+
start = i + 1
233+
}
234+
if start < len(s) {
235+
space = append(space, s[start:]...)
236+
}
237+
return append(space, '"')
238+
}

Diff for: ‎extra/binary_as_string_codec_test.go

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package extra
2+
3+
import (
4+
"testing"
5+
"github.com/stretchr/testify/require"
6+
"github.com/json-iterator/go"
7+
)
8+
9+
func init() {
10+
jsoniter.RegisterExtension(&BinaryAsStringExtension{})
11+
}
12+
13+
func TestBinaryAsStringCodec(t *testing.T) {
14+
t.Run("safe set", func(t *testing.T) {
15+
should := require.New(t)
16+
output, err := jsoniter.Marshal([]byte("hello"))
17+
should.NoError(err)
18+
should.Equal(`"hello"`, string(output))
19+
var val []byte
20+
should.NoError(jsoniter.Unmarshal(output, &val))
21+
should.Equal(`hello`, string(val))
22+
})
23+
t.Run("non safe set", func(t *testing.T) {
24+
should := require.New(t)
25+
output, err := jsoniter.Marshal([]byte{1, 2, 3, 15})
26+
should.NoError(err)
27+
should.Equal(`"\\x01\\x02\\x03\\x0f"`, string(output))
28+
var val []byte
29+
should.NoError(jsoniter.Unmarshal(output, &val))
30+
should.Equal([]byte{1, 2, 3, 15}, val)
31+
})
32+
}

Diff for: ‎stream.go

+5
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ func (stream *Stream) Buffer() []byte {
5555
return stream.buf
5656
}
5757

58+
// SetBuffer allows to append to the internal buffer directly
59+
func (stream *Stream) SetBuffer(buf []byte) {
60+
stream.buf = buf
61+
}
62+
5863
// Write writes the contents of p into the buffer.
5964
// It returns the number of bytes written.
6065
// If nn < len(p), it also returns an error explaining

0 commit comments

Comments
 (0)