Skip to content

Commit cf57414

Browse files
sandwwraithsandwwraith
authored
Add a flag to allow parser to accept trailing commas. (#2480)
This is one of the popular community requests and one of the main reasons people ask for Json5 support. Implementing this flag separately will allow for alleviating large paint points quickly without waiting for full Json5 support. Fixes #1812 Relates to: #797, #2221
1 parent 6ac4902 commit cf57414

File tree

11 files changed

+173
-19
lines changed

11 files changed

+173
-19
lines changed

‎formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonErrorMessagesTest.kt

+1-7
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package kotlinx.serialization.json
77

88
import kotlinx.serialization.*
9+
import kotlinx.serialization.test.*
910
import kotlin.test.*
1011

1112

@@ -155,11 +156,4 @@ class JsonErrorMessagesTest : JsonTestBase() {
155156
})
156157

157158
}
158-
159-
private fun checkSerializationException(action: () -> Unit, assertions: SerializationException.(String) -> Unit) {
160-
val e = assertFailsWith(SerializationException::class, action)
161-
assertNotNull(e.message)
162-
e.assertions(e.message!!)
163-
}
164-
165159
}

‎formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonParserTest.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ class JsonParserTest : JsonTestBase() {
8383
}
8484

8585
private fun testTrailingComma(content: String) {
86-
assertFailsWithSerialMessage("JsonDecodingException", "Unexpected trailing") { Json.parseToJsonElement(content) }
86+
assertFailsWithSerialMessage("JsonDecodingException", "Trailing comma before the end of JSON object") { Json.parseToJsonElement(content) }
8787
}
8888

8989
@Test

‎formats/json-tests/commonTest/src/kotlinx/serialization/json/TrailingCommaTest.kt

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* Copyright 2017-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package kotlinx.serialization.json
6+
7+
import kotlinx.serialization.*
8+
import kotlinx.serialization.test.*
9+
import kotlin.test.*
10+
11+
class TrailingCommaTest : JsonTestBase() {
12+
val tj = Json { allowTrailingComma = true }
13+
14+
@Serializable
15+
data class Optional(val data: String = "")
16+
17+
@Serializable
18+
data class MultipleFields(val a: String, val b: String, val c: String)
19+
20+
private val multipleFields = MultipleFields("1", "2", "3")
21+
22+
@Serializable
23+
data class WithMap(val m: Map<String, String>)
24+
25+
private val withMap = WithMap(mapOf("a" to "1", "b" to "2", "c" to "3"))
26+
27+
@Serializable
28+
data class WithList(val l: List<Int>)
29+
30+
private val withList = WithList(listOf(1, 2, 3))
31+
32+
@Test
33+
fun basic() = parametrizedTest { mode ->
34+
val sd = """{"data":"str",}"""
35+
assertEquals(Optional("str"), tj.decodeFromString<Optional>(sd, mode))
36+
}
37+
38+
@Test
39+
fun trailingCommaNotAllowedByDefaultForObjects() = parametrizedTest { mode ->
40+
val sd = """{"data":"str",}"""
41+
checkSerializationException({
42+
default.decodeFromString<Optional>(sd, mode)
43+
}, { message ->
44+
assertContains(
45+
message,
46+
"""Unexpected JSON token at offset 13: Trailing comma before the end of JSON object"""
47+
)
48+
})
49+
}
50+
51+
@Test
52+
fun trailingCommaNotAllowedByDefaultForLists() = parametrizedTest { mode ->
53+
val sd = """{"l":[1,]}"""
54+
checkSerializationException({
55+
default.decodeFromString<WithList>(sd, mode)
56+
}, { message ->
57+
assertContains(
58+
message,
59+
"""Unexpected JSON token at offset 7: Trailing comma before the end of JSON array"""
60+
)
61+
})
62+
}
63+
64+
@Test
65+
fun trailingCommaNotAllowedByDefaultForMaps() = parametrizedTest { mode ->
66+
val sd = """{"m":{"a": "b",}}"""
67+
checkSerializationException({
68+
default.decodeFromString<WithMap>(sd, mode)
69+
}, { message ->
70+
assertContains(
71+
message,
72+
"""Unexpected JSON token at offset 14: Trailing comma before the end of JSON object"""
73+
)
74+
})
75+
}
76+
77+
@Test
78+
fun emptyObjectNotAllowed() = parametrizedTest { mode ->
79+
assertFailsWithMessage<SerializationException>("Unexpected leading comma") {
80+
tj.decodeFromString<Optional>("""{,}""", mode)
81+
}
82+
}
83+
84+
@Test
85+
fun emptyListNotAllowed() = parametrizedTest { mode ->
86+
assertFailsWithMessage<SerializationException>("Unexpected leading comma") {
87+
tj.decodeFromString<WithList>("""{"l":[,]}""", mode)
88+
}
89+
}
90+
91+
@Test
92+
fun emptyMapNotAllowed() = parametrizedTest { mode ->
93+
assertFailsWithMessage<SerializationException>("Unexpected leading comma") {
94+
tj.decodeFromString<WithMap>("""{"m":{,}}""", mode)
95+
}
96+
}
97+
98+
@Test
99+
fun testMultipleFields() = parametrizedTest { mode ->
100+
val input = """{"a":"1","b":"2","c":"3", }"""
101+
assertEquals(multipleFields, tj.decodeFromString(input, mode))
102+
}
103+
104+
@Test
105+
fun testWithMap() = parametrizedTest { mode ->
106+
val input = """{"m":{"a":"1","b":"2","c":"3", }}"""
107+
108+
assertEquals(withMap, tj.decodeFromString(input, mode))
109+
}
110+
111+
@Test
112+
fun testWithList() = parametrizedTest { mode ->
113+
val input = """{"l":[1, 2, 3, ]}"""
114+
assertEquals(withList, tj.decodeFromString(input, mode))
115+
}
116+
117+
@Serializable
118+
data class Mixed(val mf: MultipleFields, val wm: WithMap, val wl: WithList)
119+
120+
@Test
121+
fun testMixed() = parametrizedTest { mode ->
122+
//language=JSON5
123+
val input = """{"mf":{"a":"1","b":"2","c":"3",},
124+
"wm":{"m":{"a":"1","b":"2","c":"3",},},
125+
"wl":{"l":[1, 2, 3,],},}"""
126+
assertEquals(Mixed(multipleFields, withMap, withList), tj.decodeFromString(input, mode))
127+
}
128+
}

‎formats/json-tests/commonTest/src/kotlinx/serialization/test/TestingFramework.kt

+6
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,9 @@ inline fun <reified T : Throwable> assertFailsWithMessage(
9292
"expected:<$message> but was:<${exception.message}>"
9393
)
9494
}
95+
96+
inline fun checkSerializationException(action: () -> Unit, assertions: SerializationException.(String) -> Unit) {
97+
val e = assertFailsWith(SerializationException::class, action)
98+
assertNotNull(e.message)
99+
e.assertions(e.message!!)
100+
}

‎formats/json/api/kotlinx-serialization-json.api

+3
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ public final class kotlinx/serialization/json/JsonArraySerializer : kotlinx/seri
8787
public final class kotlinx/serialization/json/JsonBuilder {
8888
public final fun getAllowSpecialFloatingPointValues ()Z
8989
public final fun getAllowStructuredMapKeys ()Z
90+
public final fun getAllowTrailingComma ()Z
9091
public final fun getClassDiscriminator ()Ljava/lang/String;
9192
public final fun getCoerceInputValues ()Z
9293
public final fun getDecodeEnumsCaseInsensitive ()Z
@@ -102,6 +103,7 @@ public final class kotlinx/serialization/json/JsonBuilder {
102103
public final fun isLenient ()Z
103104
public final fun setAllowSpecialFloatingPointValues (Z)V
104105
public final fun setAllowStructuredMapKeys (Z)V
106+
public final fun setAllowTrailingComma (Z)V
105107
public final fun setClassDiscriminator (Ljava/lang/String;)V
106108
public final fun setCoerceInputValues (Z)V
107109
public final fun setDecodeEnumsCaseInsensitive (Z)V
@@ -130,6 +132,7 @@ public final class kotlinx/serialization/json/JsonConfiguration {
130132
public fun <init> ()V
131133
public final fun getAllowSpecialFloatingPointValues ()Z
132134
public final fun getAllowStructuredMapKeys ()Z
135+
public final fun getAllowTrailingComma ()Z
133136
public final fun getClassDiscriminator ()Ljava/lang/String;
134137
public final fun getCoerceInputValues ()Z
135138
public final fun getDecodeEnumsCaseInsensitive ()Z

‎formats/json/commonMain/src/kotlinx/serialization/json/Json.kt

+11-1
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,16 @@ public class JsonBuilder internal constructor(json: Json) {
364364
@ExperimentalSerializationApi
365365
public var decodeEnumsCaseInsensitive: Boolean = json.configuration.decodeEnumsCaseInsensitive
366366

367+
/**
368+
* Allows parser to accept trailing (ending) commas in JSON objects and arrays,
369+
* making inputs like `[1, 2, 3,]` valid.
370+
*
371+
* Does not affect encoding.
372+
* `false` by default.
373+
*/
374+
@ExperimentalSerializationApi
375+
public var allowTrailingComma: Boolean = json.configuration.allowTrailingComma
376+
367377
/**
368378
* Module with contextual and polymorphic serializers to be used in the resulting [Json] instance.
369379
*
@@ -396,7 +406,7 @@ public class JsonBuilder internal constructor(json: Json) {
396406
allowStructuredMapKeys, prettyPrint, explicitNulls, prettyPrintIndent,
397407
coerceInputValues, useArrayPolymorphism,
398408
classDiscriminator, allowSpecialFloatingPointValues, useAlternativeNames,
399-
namingStrategy, decodeEnumsCaseInsensitive
409+
namingStrategy, decodeEnumsCaseInsensitive, allowTrailingComma
400410
)
401411
}
402412
}

‎formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt

+4-2
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ public class JsonConfiguration @OptIn(ExperimentalSerializationApi::class) inter
3232
@ExperimentalSerializationApi
3333
public val namingStrategy: JsonNamingStrategy? = null,
3434
@ExperimentalSerializationApi
35-
public val decodeEnumsCaseInsensitive: Boolean = false
35+
public val decodeEnumsCaseInsensitive: Boolean = false,
36+
@ExperimentalSerializationApi
37+
public val allowTrailingComma: Boolean = false,
3638
) {
3739

3840
/** @suppress Dokka **/
@@ -42,6 +44,6 @@ public class JsonConfiguration @OptIn(ExperimentalSerializationApi::class) inter
4244
"allowStructuredMapKeys=$allowStructuredMapKeys, prettyPrint=$prettyPrint, explicitNulls=$explicitNulls, " +
4345
"prettyPrintIndent='$prettyPrintIndent', coerceInputValues=$coerceInputValues, useArrayPolymorphism=$useArrayPolymorphism, " +
4446
"classDiscriminator='$classDiscriminator', allowSpecialFloatingPointValues=$allowSpecialFloatingPointValues, useAlternativeNames=$useAlternativeNames, " +
45-
"namingStrategy=$namingStrategy, decodeEnumsCaseInsensitive=$decodeEnumsCaseInsensitive)"
47+
"namingStrategy=$namingStrategy, decodeEnumsCaseInsensitive=$decodeEnumsCaseInsensitive, allowTrailingComma=$allowTrailingComma)"
4648
}
4749
}

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonExceptions.kt

+7
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,13 @@ internal fun AbstractJsonLexer.throwInvalidFloatingPointDecoded(result: Number):
4646
hint = specialFlowingValuesHint)
4747
}
4848

49+
internal fun AbstractJsonLexer.invalidTrailingComma(entity: String = "object"): Nothing {
50+
fail("Trailing comma before the end of JSON $entity",
51+
position = currentPosition - 1,
52+
hint = "Trailing commas are non-complaint JSON and not allowed by default. Use 'allowTrailingCommas = true' in 'Json {}' builder to support them."
53+
)
54+
}
55+
4956
@OptIn(ExperimentalSerializationApi::class)
5057
internal fun InvalidKeyKindException(keyDescriptor: SerialDescriptor) = JsonEncodingException(
5158
"Value of type '${keyDescriptor.serialName}' can't be used in JSON as a key in the map. " +

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonTreeReader.kt

+6-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ internal class JsonTreeReader(
1313
private val lexer: AbstractJsonLexer
1414
) {
1515
private val isLenient = configuration.isLenient
16+
private val trailingCommaAllowed = configuration.allowTrailingComma
1617
private var stackDepth = 0
1718

1819
private fun readObject(): JsonElement = readObjectImpl {
@@ -44,8 +45,9 @@ internal class JsonTreeReader(
4445
if (lastToken == TC_BEGIN_OBJ) { // Case of empty object
4546
lexer.consumeNextToken(TC_END_OBJ)
4647
} else if (lastToken == TC_COMMA) { // Trailing comma
47-
lexer.fail("Unexpected trailing comma")
48-
}
48+
if (!trailingCommaAllowed) lexer.invalidTrailingComma()
49+
lexer.consumeNextToken(TC_END_OBJ)
50+
} // else unexpected token?
4951
return JsonObject(result)
5052
}
5153

@@ -66,7 +68,8 @@ internal class JsonTreeReader(
6668
if (lastToken == TC_BEGIN_LIST) { // Case of empty object
6769
lexer.consumeNextToken(TC_END_LIST)
6870
} else if (lastToken == TC_COMMA) { // Trailing comma
69-
lexer.fail("Unexpected trailing comma")
71+
if (!trailingCommaAllowed) lexer.invalidTrailingComma("array")
72+
lexer.consumeNextToken(TC_END_LIST)
7073
}
7174
return JsonArray(result)
7275
}

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt

+5-4
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ internal open class StreamingJsonDecoder(
122122
if (json.configuration.ignoreUnknownKeys && descriptor.elementsCount == 0) {
123123
skipLeftoverElements(descriptor)
124124
}
125+
if (lexer.tryConsumeComma() && !json.configuration.allowTrailingComma) lexer.invalidTrailingComma("")
125126
// First consume the object so we know it's correct
126127
lexer.consumeNextToken(mode.end)
127128
// Then cleanup the path
@@ -195,12 +196,12 @@ internal open class StreamingJsonDecoder(
195196

196197
return if (lexer.canConsumeValue()) {
197198
if (decodingKey) {
198-
if (currentIndex == -1) lexer.require(!hasComma) { "Unexpected trailing comma" }
199+
if (currentIndex == -1) lexer.require(!hasComma) { "Unexpected leading comma" }
199200
else lexer.require(hasComma) { "Expected comma after the key-value pair" }
200201
}
201202
++currentIndex
202203
} else {
203-
if (hasComma) lexer.fail("Expected '}', but had ',' instead")
204+
if (hasComma && !json.configuration.allowTrailingComma) lexer.invalidTrailingComma()
204205
CompositeDecoder.DECODE_DONE
205206
}
206207
}
@@ -239,7 +240,7 @@ internal open class StreamingJsonDecoder(
239240
hasComma = handleUnknown(key)
240241
}
241242
}
242-
if (hasComma) lexer.fail("Unexpected trailing comma")
243+
if (hasComma && !json.configuration.allowTrailingComma) lexer.invalidTrailingComma()
243244

244245
return elementMarker?.nextUnmarkedIndex() ?: CompositeDecoder.DECODE_DONE
245246
}
@@ -262,7 +263,7 @@ internal open class StreamingJsonDecoder(
262263
if (currentIndex != -1 && !hasComma) lexer.fail("Expected end of the array or comma")
263264
++currentIndex
264265
} else {
265-
if (hasComma) lexer.fail("Unexpected trailing comma")
266+
if (hasComma && !json.configuration.allowTrailingComma) lexer.invalidTrailingComma("array")
266267
CompositeDecoder.DECODE_DONE
267268
}
268269
}

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/AbstractJsonLexer.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ internal abstract class AbstractJsonLexer {
149149
protected abstract val source: CharSequence
150150

151151
@JvmField
152-
protected var currentPosition: Int = 0 // position in source
152+
internal var currentPosition: Int = 0 // position in source
153153

154154
@JvmField
155155
val path = JsonPath()

0 commit comments

Comments
 (0)