boxplot: handle non-numeric cols & duplicate rows

Author: sheppard

rest_pandas/serializers.py

@@ -1,7 +1,9 @@
 from rest_framework import serializers
 from pandas import DataFrame
+from pandas.api.types import is_numeric_dtype
 from django.core.exceptions import ImproperlyConfigured
 import datetime
+from collections import OrderedDict
 
 
 class PandasSerializer(serializers.ListSerializer):
@@ -222,12 +224,15 @@ def get_index(self, dataframe):
         group_field = self.get_group_field()
         date_field = self.get_date_field()
         header_fields = self.get_header_fields()
+        extra_index_fields = self.get_extra_index_fields()
 
+        index = []
         if date_field:
-            group_fields = [date_field, group_field]
-        else:
-            group_fields = [group_field]
-        return group_fields + header_fields
+            index.append(date_field)
+        index += extra_index_fields
+        index.append(group_field)
+        index += header_fields
+        return index
 
     def transform_dataframe(self, dataframe):
         """
@@ -255,35 +260,30 @@ def transform_dataframe(self, dataframe):
             interval = None
 
         # Compute stats for each column, potentially grouped by year
-        all_stats = []
+        series_infos = OrderedDict()
         for header, series in groups.items():
             if interval:
                 series_stats = self.boxplots_for_interval(series, interval)
             else:
-                interval = None
                 series_stats = [self.compute_boxplot(series)]
 
-            series_infos = []
             for series_stat in series_stats:
-                series_info = {}
                 if isinstance(header, tuple):
                     value_name = header[0]
                     col_values = header[1:]
                 else:
                     value_name = header
                     col_values = []
-                col_names = zip(dataframe.columns.names[1:], col_values)
-                for col_name, value in col_names:
-                    series_info[col_name] = value
+                col_names = tuple(zip(dataframe.columns.names[1:], col_values))
+                if interval in series_stat:
+                    col_names += ((interval, series_stat[interval]),)
+                series_infos.setdefault(col_names, dict(col_names))
+                series_info = series_infos[col_names]
                 for stat_name, val in series_stat.items():
-                    if stat_name == interval:
-                        series_info[stat_name] = val
-                    else:
+                    if stat_name != interval:
                         series_info[value_name + '-' + stat_name] = val
-                series_infos.append(series_info)
-            all_stats += series_infos
 
-        dataframe = DataFrame(all_stats)
+        dataframe = DataFrame(list(series_infos.values()))
         if 'series' in grouping:
             index = header_fields + [group_field]
             unstack = len(header_fields)
@@ -336,11 +336,19 @@ def compute_boxplot(self, series):
         series = series[series.notnull()]
         if len(series.values) == 0:
             return {}
+        elif not is_numeric_dtype(series):
+            return self.non_numeric_stats(series)
         stats = boxplot_stats(list(series.values))[0]
         stats['count'] = len(series.values)
         stats['fliers'] = "|".join(map(str, stats['fliers']))
         return stats
 
+    def non_numeric_stats(self, series):
+        return {
+            'count': len(series),
+            'mode': series.mode()[0],
+        }
+
     def get_group_field(self):
         """
         Categorical field to group datasets by.
@@ -359,6 +367,12 @@ def get_header_fields(self):
         """
         return self.get_meta_option('boxplot_header', [])
 
+    def get_extra_index_fields(self):
+        """
+        Fields that identify each row but don't need to be considered for plot
+        """
+        return self.get_meta_option('boxplot_extra_index', [])
+
 
 class SimpleSerializer(serializers.Serializer):
     """

tests/test_complex.py

@@ -30,16 +30,19 @@ def setUp(self):
             ('site2', 'flow', 'cfs', '2015-01-04', 'routine', 0.3, None),
             ('site2', 'flow', 'cfs', '2015-01-05', 'routine', 0.8, None),
         )
-        for site, parameter, units, date, type, value, flag in data:
-            ComplexTimeSeries.objects.create(
-                site=site,
-                parameter=parameter,
-                units=units,
-                date=date,
-                type=type,
-                value=value,
-                flag=flag,
-            )
+        for row in data:
+            self.create_row(*row)
+
+    def create_row(self, site, parameter, units, date, type, value, flag):
+        ComplexTimeSeries.objects.create(
+            site=site,
+            parameter=parameter,
+            units=units,
+            date=date,
+            type=type,
+            value=value,
+            flag=flag,
+        )
 
     def test_complex_series(self):
         response = self.client.get("/complextimeseries.csv")
@@ -285,5 +288,15 @@ def test_complex_boxplot_year(self):
         self.assertEqual(round(stats['value-mean'], 5), 0.56111)
         self.assertEqual(stats['value-whishi'], 1.5)
 
+    @unittest.skipUnless(HAS_MATPLOTLIB, "requires matplotlib")
+    def test_complex_boxplot_extra(self):
+        self.create_row(
+            'site1', 'flow', 'cfs', '2015-01-01', 'routine', 0.3, None
+        )
+        with self.assertRaises(ValueError):
+            response = self.client.get("/complexboxplot.csv")
+        response = self.client.get("/complexboxplotextra.csv")
+        self.assertEqual(1, len(response.data))
+
     def parse_csv(self, response):
         return parse_csv(response.content.decode('utf-8'))

tests/testapp/serializers.py

@@ -57,6 +57,16 @@ class Meta(ComplexTimeSeriesSerializer.Meta):
         pandas_boxplot_header = ['units', 'parameter']
 
 
+class ComplexBoxplotExtraSerializer(ComplexTimeSeriesSerializer):
+    class Meta:
+        model = ComplexTimeSeries
+        fields = '__all__'
+        pandas_boxplot_group = 'site'
+        pandas_boxplot_date = 'date'
+        pandas_boxplot_extra_index = ['id']  # Ensure row uniqueness
+        pandas_boxplot_header = ['units', 'parameter']
+
+
 class NotUnstackableSerializer(ModelSerializer):
     class Meta:
         model = MultiTimeSeries

tests/testapp/urls.py

@@ -7,7 +7,8 @@
     TimeSeriesMixedRendererView, TimeSeriesMixinView, TimeSeriesNoMixinView,
     DjangoPandasView, TimeSeriesViewSet,
     MultiTimeSeriesView, MultiScatterView, MultiBoxplotView,
-    ComplexTimeSeriesView, ComplexScatterView, ComplexBoxplotView,
+    ComplexTimeSeriesView, ComplexScatterView,
+    ComplexBoxplotView, ComplexBoxplotExtraView,
     CustomIndexSeriesView,
 )
 
@@ -29,6 +30,7 @@
     url(r'^complextimeseries$', ComplexTimeSeriesView.as_view()),
     url(r'^complexscatter$', ComplexScatterView.as_view()),
     url(r'^complexboxplot$', ComplexBoxplotView.as_view()),
+    url(r'^complexboxplotextra$', ComplexBoxplotExtraView.as_view()),
     url(r'^customindex$', CustomIndexSeriesView.as_view()),
 ]
 urlpatterns = format_suffix_patterns(urlpatterns)

tests/testapp/views.py

@@ -12,7 +12,7 @@
     TimeSeriesSerializer, TimeSeriesNoIdSerializer,
     MultiTimeSeriesSerializer,
     ComplexTimeSeriesSerializer, ComplexScatterSerializer,
-    ComplexBoxplotSerializer,
+    ComplexBoxplotSerializer, ComplexBoxplotExtraSerializer,
     CustomIndexSeriesSerializer,
 )
 import pandas as pd
@@ -127,6 +127,12 @@ class ComplexBoxplotView(PandasView):
     pandas_serializer_class = PandasBoxplotSerializer
 
 
+class ComplexBoxplotExtraView(PandasView):
+    queryset = ComplexTimeSeries.objects.all()
+    serializer_class = ComplexBoxplotExtraSerializer
+    pandas_serializer_class = PandasBoxplotSerializer
+
+
 class CustomIndexSeriesView(PandasView):
     queryset = CustomIndexSeries.objects.all()
     serializer_class = CustomIndexSeriesSerializer