serializers.py

from rest_framework import serializers
from pandas import DataFrame
from pandas.api.types import is_numeric_dtype
from django.core.exceptions import ImproperlyConfigured
from django.utils.functional import cached_property
import datetime
from collections import OrderedDict
from . import settings


def get_label(field, name):
    if field.label == "ID":
        return "id"
    elif field.label:
        return field.label
    else:
        return name


class PandasSerializer(serializers.ListSerializer):
    """
    Transforms dataset into a dataframe and applies an index
    """

    read_only = True
    apply_field_labels = settings.APPLY_FIELD_LABELS
    index_none_value = settings.INDEX_NONE_VALUE
    wq_chart_type = None

    def get_index(self, dataframe):
        return self.get_index_fields()

    @cached_property
    def field_labels(self):
        return {
            name: get_label(field, name)
            for name, field in self.child.get_fields().items()
        }

    def get_dataframe(self, data):
        dataframe = DataFrame(data)
        if self.apply_field_labels:
            dataframe.rename(columns=self.field_labels, inplace=True)
        index = self.get_index(dataframe)
        if index:
            if self.index_none_value is not None:
                for key in index:
                    try:
                        dataframe[key].fillna(
                            self.index_none_value, inplace=True
                        )
                    except ValueError:
                        pass
            dataframe.set_index(index, inplace=True)
        else:
            # Name auto-index column to ensure valid CSV output
            dataframe.index.name = "row"
        return dataframe

    def transform_dataframe(self, dataframe):
        view = self.context.get("view", None)
        if view and hasattr(view, "transform_dataframe"):
            return self.context["view"].transform_dataframe(dataframe)
        return dataframe

    @property
    def data(self):
        data = super(serializers.ListSerializer, self).data
        if isinstance(data, DataFrame) or data:
            dataframe = self.get_dataframe(data)
            return self.transform_dataframe(dataframe)
        else:
            return DataFrame([])

    def to_representation(self, data):
        if isinstance(data, DataFrame):
            return data
        return super().to_representation(data)

    @property
    def model_serializer(self):
        serializer = type(self.child)
        if serializer.__name__ == "SerializerWithListSerializer":
            return serializer.__bases__[0]
        return serializer

    @property
    def model_serializer_meta(self):
        return getattr(self.model_serializer, "Meta", object())

    def get_index_fields(self):
        """
        List of fields to use for index
        """
        index_fields = self.get_meta_option("index", [], True)
        if index_fields:
            return index_fields

        model = getattr(self.model_serializer_meta, "model", None)
        if model:
            pk_name = model._meta.pk.name
            if pk_name in self.child.get_fields():
                if self.apply_field_labels:
                    pk_name = self.field_labels.get(pk_name, pk_name)
                return [pk_name]

        return []

    def get_meta_option(self, name, default=None, apply_field_labels=False):
        meta_name = "pandas_" + name
        value = getattr(self.model_serializer_meta, meta_name, None)

        if value is None:
            if default is not None:
                return default
            else:
                raise ImproperlyConfigured(
                    "%s should be specified on %s.Meta"
                    % (meta_name, self.model_serializer.__name__)
                )
        elif apply_field_labels and self.apply_field_labels:
            return [self.field_labels.get(field, field) for field in value]
        else:
            return value


class PandasUnstackedSerializer(PandasSerializer):
    """
    Pivots dataframe so commonly-repeating values are across the top in a
    multi-row header.  Intended for use with e.g. time series data, where the
    header includes metadata applicable to each time series.
    (Use with wq/chart.js' timeSeries() function)
    """

    index_none_value = "-"
    wq_chart_type = "timeSeries"

    def get_index(self, dataframe):
        """
        Include header fields in initial index for later unstacking
        """
        return self.get_index_fields() + self.get_header_fields()

    def transform_dataframe(self, dataframe):
        """
        Unstack the dataframe so header fields are across the top.
        """
        dataframe.columns.name = ""

        for i in range(len(self.get_header_fields())):
            dataframe = dataframe.unstack()

        # Remove blank rows / columns
        dataframe = dataframe.dropna(axis=0, how="all").dropna(
            axis=1, how="all"
        )
        return dataframe

    def get_header_fields(self):
        """
        Series metadata fields for header (first few rows)
        """
        return self.get_meta_option("unstacked_header", None, True)


class PandasScatterSerializer(PandasSerializer):
    """
    Pivots dataframe into a format suitable for plotting two series
    against each other as x vs y on a scatter plot.
    (Use with wq/chart.js' scatter() function)
    """

    index_none_value = "-"
    wq_chart_type = "scatter"

    def get_index(self, dataframe):
        """
        Include scatter & header fields in initial index for later unstacking
        """
        return (
            self.get_index_fields()
            + self.get_header_fields()
            + self.get_coord_fields()
        )

    def transform_dataframe(self, dataframe):
        """
        Unstack the dataframe so header consists of a composite 'value' header
        plus any other header fields.
        """
        coord_fields = self.get_coord_fields()
        header_fields = self.get_header_fields()

        # Remove any pairs that don't have data for both x & y
        for i in range(len(coord_fields)):
            dataframe = dataframe.unstack()
        dataframe = dataframe.dropna(axis=1, how="all")
        dataframe = dataframe.dropna(axis=0, how="any")

        # Unstack series header
        for i in range(len(header_fields)):
            dataframe = dataframe.unstack()

        # Compute new column headers
        columns = []
        for i in range(len(header_fields) + 1):
            columns.append([])

        for col in dataframe.columns:
            value_name = col[0]
            coord_names = list(col[1 : len(coord_fields) + 1])
            header_names = list(col[len(coord_fields) + 1 :])
            coord_name = ""
            for name in coord_names:
                if name != self.index_none_value:
                    coord_name += name + "-"
            coord_name += value_name
            columns[0].append(coord_name)
            for i, header_name in enumerate(header_names):
                columns[1 + i].append(header_name)

        dataframe.columns = columns
        dataframe.columns.names = [""] + header_fields

        return dataframe

    def get_coord_fields(self):
        """
        Fields that will be collapsed into a single header with the name of
        each coordinate.
        """
        return self.get_meta_option("scatter_coord", None, True)

    def get_header_fields(self):
        """
        Other header fields, if any
        """
        return self.get_meta_option("scatter_header", [], True)


class PandasBoxplotSerializer(PandasSerializer):
    """
    Compute boxplot statistics on dataframe columns, creating a new unstacked
    dataframe where each row describes a boxplot.
    (Use with wq/chart.js' boxplot() function)
    """

    index_none_value = "-"
    wq_chart_type = "boxplot"

    def get_index(self, dataframe):
        group_field = self.get_group_field()
        date_field = self.get_date_field()
        header_fields = self.get_header_fields()
        extra_index_fields = self.get_extra_index_fields()

        index = []
        if date_field:
            index.append(date_field)
        index += extra_index_fields
        index.append(group_field)
        index += header_fields
        return index

    def transform_dataframe(self, dataframe):
        """
        Use matplotlib to compute boxplot statistics on e.g. timeseries data.
        """
        grouping = self.get_grouping(dataframe)
        group_field = self.get_group_field()
        header_fields = self.get_header_fields()

        if "series" in grouping:
            # Unstack so each series is a column
            for i in range(len(header_fields) + 1):
                dataframe = dataframe.unstack()

        groups = {col: dataframe[col] for col in dataframe.columns}

        if "year" in grouping:
            interval = "year"
        elif "month" in grouping:
            interval = "month"
        else:
            interval = None

        # Compute stats for each column, potentially grouped by year
        series_infos = OrderedDict()
        for header, series in groups.items():
            if interval:
                series_stats = self.boxplots_for_interval(series, interval)
            else:
                series_stats = [self.compute_boxplot(series)]

            for series_stat in series_stats:
                if isinstance(header, tuple):
                    value_name = header[0]
                    col_values = header[1:]
                else:
                    value_name = header
                    col_values = []
                col_names = tuple(zip(dataframe.columns.names[1:], col_values))
                if interval in series_stat:
                    col_names += ((interval, series_stat[interval]),)
                series_infos.setdefault(col_names, dict(col_names))
                series_info = series_infos[col_names]
                for stat_name, val in series_stat.items():
                    if stat_name != interval:
                        series_info[value_name + "-" + stat_name] = val

        dataframe = DataFrame(list(series_infos.values()))
        if "series" in grouping:
            index = header_fields + [group_field]
            unstack = len(header_fields)
            if interval:
                index = [interval] + index
                unstack += 1
        else:
            index = [interval]
            unstack = 0

        dataframe.set_index(index, inplace=True)
        dataframe.columns.name = ""
        for i in range(unstack):
            dataframe = dataframe.unstack()

        # Remove blank columns
        dataframe = dataframe.dropna(axis=1, how="all")
        return dataframe

    def get_grouping(self, dataframe):
        request = self.context.get("request", None)
        datasets = len(dataframe.columns)
        if request:
            group = request.GET.get("group", None)
            if group:
                return group
        return default_grouping(datasets, self.get_date_field())

    def boxplots_for_interval(self, series, interval):
        def get_interval_name(date):
            if isinstance(date, tuple):
                date = date[0]
            if hasattr(date, "count") and date.count("-") == 2:
                date = datetime.datetime.strptime(date, "%Y-%m-%d")
            return getattr(date, interval)

        interval_stats = []
        groups = series.groupby(get_interval_name).groups
        for interval_name, group in groups.items():
            stats = self.compute_boxplot(series[group])
            stats[interval] = interval_name
            interval_stats.append(stats)
        return interval_stats

    def compute_boxplot(self, series):
        """
        Compute boxplot for given pandas Series.
        """
        from matplotlib.cbook import boxplot_stats

        series = series[series.notnull()]
        if len(series.values) == 0:
            return {}
        elif not is_numeric_dtype(series):
            return self.non_numeric_stats(series)
        stats = boxplot_stats(list(series.values))[0]
        stats["count"] = len(series.values)
        stats["fliers"] = "|".join(map(str, stats["fliers"]))
        return stats

    def non_numeric_stats(self, series):
        return {
            "count": len(series),
            "mode": series.mode()[0],
        }

    def get_group_field(self):
        """
        Categorical field to group datasets by.
        """
        return self.get_meta_option("boxplot_group")

    def get_date_field(self):
        """
        Date field to group datasets by year or month.
        """
        return self.get_meta_option("boxplot_date", False)

    def get_header_fields(self):
        """
        Additional series metadata for boxplot column headers
        """
        return self.get_meta_option("boxplot_header", [], True)

    def get_extra_index_fields(self):
        """
        Fields that identify each row but don't need to be considered for plot
        """
        return self.get_meta_option("boxplot_extra_index", [], True)


class SimpleSerializer(serializers.Serializer):
    """
    Simple serializer for non-model (simple) views
    """

    def to_representation(self, obj):
        return obj


def default_grouping(datasets, date_field=None):
    """
    Heuristic for default boxplot grouping
    """
    if datasets > 20 and date_field:
        # Group all data by year
        return "year"
    elif datasets > 10 or not date_field:
        # Compare series but don't break down by year
        return "series"
    else:
        # 10 or fewer datasets, break down by both series and year
        return "series-year"