BUG: Categorical doesn't show tzinfo properly

Author: sinhrks

Diff for: doc/source/whatsnew/v0.17.0.txt

@@ -606,6 +606,9 @@ Bug Fixes
 
 
 - Bug in ``read_stata`` when reading a file with a different order set in ``columns`` (:issue:`10757`)
+- Bug in ``Categorical`` may not representing properly when category contains ``tz`` or ``Period`` (:issue:`10713`)
+- Bug in ``Categorical.__iter__`` may not returning correct ``datetime`` and ``Period`` (:issue:`10713`)
+
 
 - Reading "famafrench" data via ``DataReader`` results in HTTP 404 error because of the website url is changed (:issue:`10591`).
 - Bug in ``read_msgpack`` where DataFrame to decode has duplicate column names (:issue:`9618`)

Diff for: pandas/core/categorical.py

@@ -12,7 +12,7 @@
 import pandas.core.common as com
 from pandas.util.decorators import cache_readonly, deprecate_kwarg
 
-from pandas.core.common import (CategoricalDtype, ABCSeries, ABCIndexClass, ABCPeriodIndex, ABCCategoricalIndex,
+from pandas.core.common import (CategoricalDtype, ABCSeries, ABCIndexClass, ABCCategoricalIndex,
                                 isnull, notnull, is_dtype_equal,
                                 is_categorical_dtype, is_integer_dtype, is_object_dtype,
                                 _possibly_infer_to_datetimelike, get_dtype_kinds,
@@ -1053,15 +1053,12 @@ def get_values(self):
         Returns
         -------
         values : numpy array
-            A numpy array of the same dtype as categorical.categories.dtype or dtype string if
-            periods
+            A numpy array of the same dtype as categorical.categories.dtype or
+            Index if datetime / periods
         """
-
-        # if we are a period index, return a string repr
-        if isinstance(self.categories, ABCPeriodIndex):
-            return take_1d(np.array(self.categories.to_native_types(), dtype=object),
-                           self._codes)
-
+        # if we are a datetime and period index, return Index to keep metadata
+        if com.is_datetimelike(self.categories):
+            return self.categories.take(self._codes)
         return np.array(self)
 
     def check_for_ordered(self, op):
@@ -1308,7 +1305,7 @@ def __len__(self):
 
     def __iter__(self):
         """Returns an Iterator over the values of this Categorical."""
-        return iter(np.array(self))
+        return iter(self.get_values())
 
     def _tidy_repr(self, max_vals=10, footer=True):
         """ a short repr displaying only max_vals and an optional (but default footer) """
@@ -1328,7 +1325,7 @@ def _repr_categories(self):
         max_categories = (10 if get_option("display.max_categories") == 0
                     else get_option("display.max_categories"))
         from pandas.core import format as fmt
-        category_strs = fmt.format_array(self.categories.get_values(), None)
+        category_strs = fmt.format_array(self.categories, None)
         if len(category_strs) > max_categories:
             num = max_categories // 2
             head = category_strs[:num]
@@ -1343,22 +1340,24 @@ def _repr_categories_info(self):
         """ Returns a string representation of the footer."""
 
         category_strs = self._repr_categories()
-        levheader = "Categories (%d, %s): " % (len(self.categories),
-                                               self.categories.dtype)
+        dtype = getattr(self.categories, 'dtype_str', str(self.categories.dtype))
+
+        levheader = "Categories (%d, %s): " % (len(self.categories), dtype)
         width, height = get_terminal_size()
         max_width = get_option("display.width") or width
         if com.in_ipython_frontend():
             # 0 = no breaks
             max_width = 0
         levstring = ""
         start = True
-        cur_col_len = len(levheader)
+        cur_col_len = len(levheader) # header
         sep_len, sep = (3, " < ") if self.ordered else (2, ", ")
+        linesep = sep.rstrip() + "\n" # remove whitespace
         for val in category_strs:
             if max_width != 0 and cur_col_len + sep_len + len(val) > max_width:
-                levstring += "\n" + (" "* len(levheader))
-                cur_col_len = len(levheader)
-            if not start:
+                levstring += linesep + (" " * (len(levheader) + 1))
+                cur_col_len = len(levheader) + 1 # header + a whitespace
+            elif not start:
                 levstring += sep
                 cur_col_len += len(val)
             levstring += val

Diff for: pandas/core/format.py

@@ -207,7 +207,7 @@ def _get_formatted_index(self):
         return fmt_index, have_header
 
     def _get_formatted_values(self):
-        return format_array(self.tr_series.get_values(), None,
+        return format_array(self.tr_series.values, None,
                             float_format=self.float_format,
                             na_rep=self.na_rep)
 
@@ -681,7 +681,7 @@ def _format_col(self, i):
         frame = self.tr_frame
         formatter = self._get_formatter(i)
         return format_array(
-            (frame.iloc[:, i]).get_values(),
+            frame.iloc[:, i].values,
             formatter, float_format=self.float_format, na_rep=self.na_rep,
             space=self.col_space
         )
@@ -1895,8 +1895,13 @@ def get_formatted_cells(self):
 
 def format_array(values, formatter, float_format=None, na_rep='NaN',
                  digits=None, space=None, justify='right'):
-    if com.is_float_dtype(values.dtype):
+
+    if com.is_categorical_dtype(values):
+        fmt_klass = CategoricalArrayFormatter
+    elif com.is_float_dtype(values.dtype):
         fmt_klass = FloatArrayFormatter
+    elif com.is_period_arraylike(values):
+        fmt_klass = PeriodArrayFormatter
     elif com.is_integer_dtype(values.dtype):
         fmt_klass = IntArrayFormatter
     elif com.is_datetime64_dtype(values.dtype):
@@ -1963,6 +1968,8 @@ def _format(x):
                 return '%s' % formatter(x)
 
         vals = self.values
+        if isinstance(vals, Index):
+            vals = vals.values
 
         is_float = lib.map_infer(vals, com.is_float) & notnull(vals)
         leading_space = is_float.any()
@@ -2076,8 +2083,30 @@ def _format_strings(self):
             values = values.asobject
             is_dates_only = _is_dates_only(values)
             formatter = (self.formatter or _get_format_datetime64(is_dates_only, values, date_format=self.date_format))
-            fmt_values = [ formatter(x) for x in self.values ]
+            fmt_values = [ formatter(x) for x in values ]
+
+        return fmt_values
+
 
+class PeriodArrayFormatter(IntArrayFormatter):
+
+    def _format_strings(self):
+        values = np.array(self.values.to_native_types(), dtype=object)
+        formatter = self.formatter or (lambda x: '%s' % x)
+        fmt_values = [formatter(x) for x in values]
+        return fmt_values
+
+
+class CategoricalArrayFormatter(GenericArrayFormatter):
+
+    def __init__(self, values, *args, **kwargs):
+        GenericArrayFormatter.__init__(self, values, *args, **kwargs)
+
+    def _format_strings(self):
+        fmt_values = format_array(self.values.get_values(), self.formatter,
+                                  float_format=self.float_format,
+                                  na_rep=self.na_rep, digits=self.digits,
+                                  space=self.space, justify=self.justify)
         return fmt_values
 
 

Diff for: pandas/core/index.py

@@ -276,6 +276,11 @@ def dtype(self):
         """ return the dtype object of the underlying data """
         return self._data.dtype
 
+    @cache_readonly
+    def dtype_str(self):
+        """ return the dtype str of the underlying data """
+        return str(self.dtype)
+
     @property
     def values(self):
         """ return the underlying data as an ndarray """
@@ -2994,6 +2999,10 @@ def equals(self, other):
 
         return False
 
+    @property
+    def _formatter_func(self):
+        return self.categories._formatter_func
+
     def _format_attrs(self):
         """
         Return a list of tuples of the (attr,formatted_value)