Add Indent Support in to_json (#28130)

Author: WillAyd

Diff for: doc/source/whatsnew/v1.0.0.rst

@@ -36,7 +36,7 @@ Other enhancements
   when using the ``pyarrow`` engine. It is currently not yet supported when converting back to
   pandas (so it will become an integer or float dtype depending on the presence of missing data).
   (:issue:`28368`)
--
+- :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`)
 
 
 Build Changes
@@ -217,6 +217,7 @@ I/O
 - Improve infinity parsing. :meth:`read_csv` now interprets ``Infinity``, ``+Infinity``, ``-Infinity`` as floating point values (:issue:`10065`)
 - Bug in :meth:`DataFrame.to_csv` where values were truncated when the length of ``na_rep`` was shorter than the text input data. (:issue:`25099`)
 - Bug in :func:`DataFrame.to_string` where values were truncated using display options instead of outputting the full content (:issue:`9784`)
+- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in ISO format with ``orient="table"`` (:issue:`28130`)
 
 Plotting
 ^^^^^^^^

Diff for: pandas/_libs/src/ujson/lib/ultrajson.h

@@ -244,6 +244,10 @@ typedef struct __JSONObjectEncoder {
   If true, '<', '>', and '&' characters will be encoded as \u003c, \u003e, and \u0026, respectively. If false, no special encoding will be used. */
   int encodeHTMLChars;
 
+  /*
+  Configuration for spaces of indent */
+  int indent;
+
   /*
   Set to an error message if error occurred */
   const char *errorMsg;

Diff for: pandas/_libs/src/ujson/lib/ultrajsonenc.c

@@ -728,6 +728,22 @@ FASTCALL_ATTR INLINE_PREFIX void FASTCALL_MSVC strreverse(char *begin,
     while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux;
 }
 
+void Buffer_AppendIndentNewlineUnchecked(JSONObjectEncoder *enc)
+{
+  if (enc->indent > 0) Buffer_AppendCharUnchecked(enc, '\n');
+}
+
+// This function could be refactored to only accept enc as an argument,
+// but this is a straight vendor from ujson source
+void Buffer_AppendIndentUnchecked(JSONObjectEncoder *enc, JSINT32 value)
+{
+  int i;
+  if (enc->indent > 0)
+    while (value-- > 0)
+      for (i = 0; i < enc->indent; i++)
+        Buffer_AppendCharUnchecked(enc, ' ');
+}
+
 void Buffer_AppendIntUnchecked(JSONObjectEncoder *enc, JSINT32 value) {
     char *wstr;
     JSUINT32 uvalue = (value < 0) ? -value : value;
@@ -960,24 +976,28 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name,
             enc->iterBegin(obj, &tc);
 
             Buffer_AppendCharUnchecked(enc, '[');
+            Buffer_AppendIndentNewlineUnchecked (enc);
 
             while (enc->iterNext(obj, &tc)) {
                 if (count > 0) {
                     Buffer_AppendCharUnchecked(enc, ',');
 #ifndef JSON_NO_EXTRA_WHITESPACE
                     Buffer_AppendCharUnchecked(buffer, ' ');
 #endif
+                    Buffer_AppendIndentNewlineUnchecked (enc);
                 }
 
                 iterObj = enc->iterGetValue(obj, &tc);
 
                 enc->level++;
+                Buffer_AppendIndentUnchecked (enc, enc->level);
                 encode(iterObj, enc, NULL, 0);
                 count++;
             }
 
             enc->iterEnd(obj, &tc);
-            Buffer_Reserve(enc, 2);
+            Buffer_AppendIndentNewlineUnchecked (enc);
+            Buffer_AppendIndentUnchecked (enc, enc->level);
             Buffer_AppendCharUnchecked(enc, ']');
             break;
         }
@@ -987,25 +1007,29 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name,
             enc->iterBegin(obj, &tc);
 
             Buffer_AppendCharUnchecked(enc, '{');
+            Buffer_AppendIndentNewlineUnchecked (enc);
 
             while (enc->iterNext(obj, &tc)) {
                 if (count > 0) {
                     Buffer_AppendCharUnchecked(enc, ',');
 #ifndef JSON_NO_EXTRA_WHITESPACE
                     Buffer_AppendCharUnchecked(enc, ' ');
 #endif
+                    Buffer_AppendIndentNewlineUnchecked (enc);
                 }
 
                 iterObj = enc->iterGetValue(obj, &tc);
                 objName = enc->iterGetName(obj, &tc, &szlen);
 
                 enc->level++;
+                Buffer_AppendIndentUnchecked (enc, enc->level);
                 encode(iterObj, enc, objName, szlen);
                 count++;
             }
 
             enc->iterEnd(obj, &tc);
-            Buffer_Reserve(enc, 2);
+            Buffer_AppendIndentNewlineUnchecked (enc);
+            Buffer_AppendIndentUnchecked (enc, enc->level);
             Buffer_AppendCharUnchecked(enc, '}');
             break;
         }

Diff for: pandas/_libs/src/ujson/python/objToJSON.c

@@ -2373,10 +2373,16 @@ char *Object_iterGetName(JSOBJ obj, JSONTypeContext *tc, size_t *outLen) {
 }
 
 PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) {
-    static char *kwlist[] = {
-        "obj",    "ensure_ascii", "double_precision", "encode_html_chars",
-        "orient", "date_unit",    "iso_dates",        "default_handler",
-        NULL};
+    static char *kwlist[] = {"obj",
+                             "ensure_ascii",
+                             "double_precision",
+                             "encode_html_chars",
+                             "orient",
+                             "date_unit",
+                             "iso_dates",
+                             "default_handler",
+                             "indent",
+                             NULL};
 
     char buffer[65536];
     char *ret;
@@ -2389,6 +2395,7 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) {
     char *sdateFormat = NULL;
     PyObject *oisoDates = 0;
     PyObject *odefHandler = 0;
+    int indent = 0;
 
     PyObjectEncoder pyEncoder = {{
         Object_beginTypeContext,
@@ -2410,6 +2417,7 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) {
         idoublePrecision,
         1, // forceAscii
         0, // encodeHTMLChars
+        0, // indent
     }};
     JSONObjectEncoder *encoder = (JSONObjectEncoder *)&pyEncoder;
 
@@ -2434,10 +2442,10 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) {
 
     PRINTMARK();
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OiOssOO", kwlist, &oinput,
-                                     &oensureAscii, &idoublePrecision,
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OiOssOOi", kwlist,
+                                     &oinput, &oensureAscii, &idoublePrecision,
                                      &oencodeHTMLChars, &sOrient, &sdateFormat,
-                                     &oisoDates, &odefHandler)) {
+                                     &oisoDates, &odefHandler, &indent)) {
         return NULL;
     }
 
@@ -2503,6 +2511,8 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) {
         pyEncoder.defaultHandler = odefHandler;
     }
 
+    encoder->indent = indent;
+
     pyEncoder.originalOutputFormat = pyEncoder.outputFormat;
     PRINTMARK();
     ret = JSON_EncodeObject(oinput, encoder, buffer, sizeof(buffer));

Diff for: pandas/_typing.py

@@ -22,7 +22,7 @@
 FilePathOrBuffer = Union[str, Path, IO[AnyStr]]
 
 FrameOrSeries = TypeVar("FrameOrSeries", bound="NDFrame")
-Scalar = Union[str, int, float]
+Scalar = Union[str, int, float, bool]
 Axis = Union[str, int]
 Ordered = Optional[bool]
 

Diff for: pandas/core/generic.py

@@ -8,6 +8,7 @@
 import re
 from textwrap import dedent
 from typing import (
+    Any,
     Callable,
     Dict,
     FrozenSet,
@@ -60,7 +61,7 @@
 from pandas.core.dtypes.missing import isna, notna
 
 import pandas as pd
-from pandas._typing import Dtype, FilePathOrBuffer
+from pandas._typing import Dtype, FilePathOrBuffer, Scalar
 from pandas.core import missing, nanops
 import pandas.core.algorithms as algos
 from pandas.core.base import PandasObject, SelectionMixin
@@ -2245,17 +2246,18 @@ def to_excel(
 
     def to_json(
         self,
-        path_or_buf=None,
-        orient=None,
-        date_format=None,
-        double_precision=10,
-        force_ascii=True,
-        date_unit="ms",
-        default_handler=None,
-        lines=False,
-        compression="infer",
-        index=True,
-    ):
+        path_or_buf: Optional[FilePathOrBuffer] = None,
+        orient: Optional[str] = None,
+        date_format: Optional[str] = None,
+        double_precision: int = 10,
+        force_ascii: bool_t = True,
+        date_unit: str = "ms",
+        default_handler: Optional[Callable[[Any], Union[Scalar, List, Dict]]] = None,
+        lines: bool_t = False,
+        compression: Optional[str] = "infer",
+        index: bool_t = True,
+        indent: Optional[int] = None,
+    ) -> Optional[str]:
         """
         Convert the object to a JSON string.
 
@@ -2335,6 +2337,11 @@ def to_json(
 
             .. versionadded:: 0.23.0
 
+        indent : integer, optional
+           Length of whitespace used to indent each record.
+
+           .. versionadded:: 1.0.0
+
         Returns
         -------
         None or str
@@ -2345,6 +2352,13 @@ def to_json(
         --------
         read_json
 
+        Notes
+        -----
+        The behavior of ``indent=0`` varies from the stdlib, which does not
+        indent the output but does insert newlines. Currently, ``indent=0``
+        and the default ``indent=None`` are equivalent in pandas, though this
+        may change in a future release.
+
         Examples
         --------
 
@@ -2395,6 +2409,10 @@ def to_json(
             date_format = "iso"
         elif date_format is None:
             date_format = "epoch"
+
+        config.is_nonnegative_int(indent)
+        indent = indent or 0
+
         return json.to_json(
             path_or_buf=path_or_buf,
             obj=self,
@@ -2407,6 +2425,7 @@ def to_json(
             lines=lines,
             compression=compression,
             index=index,
+            indent=indent,
         )
 
     def to_hdf(self, path_or_buf, key, **kwargs):