Fix for issue 48: JSONValue.parse does not correctly decode UTF-8 bytes from an inputstream

json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java

@@ -15,19 +15,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static net.minidev.json.parser.ParseException.ERROR_UNEXPECTED_EOF;
-
-import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
 
 /**
  * Parser for JSON text. Please note that JSONParser is NOT thread-safe.
  * 
  * @author Uriel Chemouni <[email protected]>
  */
-class JSONParserInputStream extends JSONParserStream {
-	private InputStream in;
-
+class JSONParserInputStream extends JSONParserReader {
 	// len
 	public JSONParserInputStream(int permissiveMode) {
 		super(permissiveMode);
@@ -40,50 +37,55 @@ public JSONParserInputStream(int permissiveMode) {
 	public Object parse(InputStream in) throws ParseException {
 		return parse(in, ContainerFactory.FACTORY_SIMPLE, ContentHandlerDumy.HANDLER);
 	}
+
+	/**
+	 * use to return Primitive Type, or String, Or JsonObject or JsonArray
+	 * generated by a ContainerFactory
+	 * 
+	 * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader
+	 */
+	public Object parse(InputStream in, String charset) throws ParseException, UnsupportedEncodingException {
+		return parse(in, ContainerFactory.FACTORY_SIMPLE, ContentHandlerDumy.HANDLER, charset);
+	}
 
 	/**
 	 * use to return Primitive Type, or String, Or JsonObject or JsonArray
 	 * generated by a ContainerFactory
+	 * 
+	 * NB: This uses the default charset
 	 */
 	public Object parse(InputStream in, ContainerFactory containerFactory) throws ParseException {
 		return parse(in, containerFactory, ContentHandlerDumy.HANDLER);
 	}
+
+	/**
+	 * use to return Primitive Type, or String, Or JsonObject or JsonArray
+	 * generated by a ContainerFactory
+	 * 
+	 * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader
+	 */
+	public Object parse(InputStream in, ContainerFactory containerFactory, String charset) throws ParseException, UnsupportedEncodingException {
+		return parse(in, containerFactory, ContentHandlerDumy.HANDLER, charset);
+	}
 
 	/**
 	 * use to return Primitive Type, or String, Or JsonObject or JsonArray
 	 * generated by a ContainerFactory
+	 * 
+	 * NB: This uses the default charset
 	 */
 	public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler)
 			throws ParseException {
-		//
-		this.in = in;
-		this.pos = -1;
-		return super.parse(containerFactory, handler);
-	}
-
-	protected void read() throws IOException {
-		int i = in.read();
-		c = (i == -1) ? (char) EOI : (char) i;
-		pos++;
-		//
-	}
-
-	protected void readS() throws IOException {
-		sb.append(c);
-		int i = in.read();
-		if (i == -1) {
-			c = EOI;
-		} else {
-			c = (char) i;
-			pos++;
-		}
+	   return super.parse(new InputStreamReader(in));
 	}
-
-	protected void readNoEnd() throws ParseException, IOException {
-		int i = in.read();
-		if (i == -1)
-			throw new ParseException(pos - 1, ERROR_UNEXPECTED_EOF, "EOF");
-		c = (char) i;
-		//
+
+	/**
+	 * use to return Primitive Type, or String, Or JsonObject or JsonArray
+	 * generated by a ContainerFactory
+	 * 
+	 * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader
+	 */
+	public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler, String charset) throws ParseException, UnsupportedEncodingException {
+		return super.parse(new InputStreamReader(in, charset));
 	}
 }

json-smart/src/test/java/net/minidev/json/test/TestIssue48.java

@@ -0,0 +1,99 @@
+package net.minidev.json.test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+
+import junit.framework.TestCase;
+import net.minidev.json.JSONObject;
+import net.minidev.json.JSONValue;
+import net.minidev.json.parser.ContentHandler;
+import net.minidev.json.parser.ParseException;
+
+/**
+ * Catches issue 48: "JSONValue.parse does not correctly decode UTF-8 bytes from an inputstream"
+ * mentioned in https://code.google.com/p/json-smart/issues/detail?id=48
+ */
+public class TestIssue48 extends TestCase {
+	/**
+	 * Covers the case described in https://code.google.com/p/json-smart/issues/detail?id=48
+	 */
+	public void testIssue48() throws UnsupportedEncodingException { 
+		final String testJsonString = "{\"balance\":1000.21,\"num\":100,\"nickname\":null,\"is_vip\":true,\"Sinhalese\":\"සිංහල ජාතිය\",\"name\":\"foo\"}";
+
+		final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8"));
+
+		final JSONObject obj = (JSONObject) JSONValue.parse(bis);
+
+		final String actual = (String) obj.get("Sinhalese");
+
+		assertEquals("සිංහල ජාතිය",  actual);
+	}
+
+	/**
+	 * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-8
+	 */
+	public void testIssue48_SAXParse_InputStream_UTF8() throws Exception {
+		final String originalText = "僧伽罗人";
+		final String testJsonString = "[ \"" + originalText + "\" ]";
+
+		final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8"));
+
+		JSONValue.SAXParse(bis, new AssertingHandler(originalText));
+	}
+
+	/**
+	 * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-16
+	 */
+	public void testIssue48_SAXParse_InputStream_UTF16() throws Exception {
+		final String originalText = "僧伽罗人";
+		final String testJsonString = "[ \"" + originalText + "\" ]";
+
+		final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-16"));
+
+		JSONValue.SAXParse(bis, new AssertingHandler(originalText));
+	}
+
+	/**
+	 * Covers the same issue seen when using SAXParse with a Reader source.
+	 * 
+	 * A Reader doesn't seem to be affected by the issue, probably because it reads 
+	 * a character at a time, rather than a byte at a time. 
+	 */
+	public void testIssue48_SAXParse_Reader() throws Exception {
+		final String originalText = "僧伽罗人";
+		final String testJsonString = "[ \"" + originalText + "\" ]";
+
+		final Reader source = new StringReader(testJsonString);
+
+		JSONValue.SAXParse(source, new AssertingHandler(originalText));
+	}
+
+	/**
+	 * A ContentHandler that checks any parsed primitive against a single expected 
+	 * value and fails the test if it's not the same
+	 */
+	private static class AssertingHandler implements ContentHandler {
+		final String expectedPrimitive;
+
+		public AssertingHandler(final String expectedPrimitive) {
+			this.expectedPrimitive = expectedPrimitive;
+		}
+
+		public void startJSON() throws ParseException, IOException { }
+		public void endJSON() throws ParseException, IOException { }
+		public boolean startObject() throws ParseException, IOException { return true; }
+		public boolean endObject() throws ParseException, IOException { return true; }
+		public boolean startObjectEntry(String key) throws ParseException, IOException { return true; }
+		public boolean endObjectEntry() throws ParseException, IOException { return true; }
+		public boolean startArray() throws ParseException, IOException { return true; }
+		public boolean endArray() throws ParseException, IOException { return true; }
+
+		public boolean primitive(final Object value) throws ParseException, IOException {
+			assertEquals(this.expectedPrimitive, value);
+			return true;
+		}
+	}
+}