Initial work on new time-series storage

Author: dcramer

src/sentry/tsdb/__init__.py

@@ -0,0 +1,7 @@
+"""
+sentry.tsdb
+~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""

src/sentry/tsdb/manager.py

@@ -0,0 +1,56 @@
+"""
+sentry.tsdb.manager
+~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from django.db.models import F
+from django.utils import timezone
+
+from sentry.manager import BaseManager
+from sentry.utils.models import create_or_update
+
+from .utils import Granularity
+
+
+class PointManager(BaseManager):
+    def fetch(self, key, start_timestamp, end_timestamp):
+        # TODO
+        raise NotImplementedError
+
+    def incr(self, key, amount=1, timestamp=None):
+        """
+        Increments all granularities for ``key`` by ``amount``.
+        """
+        if timestamp is None:
+            timestamp = timezone.now()
+        for granularity, _ in Granularity.get_choices():
+            epoch = Granularity.normalize_to_epoch(granularity, timestamp)
+
+            create_or_update(
+                model=self.model,
+                key=key,
+                epoch=epoch,
+                granularity=granularity,
+                defaults={
+                    'value': F('value') + amount,
+                }
+            )
+
+    def trim(self):
+        """
+        Called periodically to flush the out of bounds points from the
+        database.
+        """
+        now = timezone.now()
+        for granularity, _ in Granularity.get_choices():
+            min_value = Granularity.get_min_value(granularity, now)
+            if min_value is None:
+                continue
+
+            self.filter(
+                granularity=granularity,
+                value__lt=min_value,
+            ).delete()

src/sentry/tsdb/models.py

@@ -0,0 +1,54 @@
+"""
+sentry.tsdb.models
+~~~~~~~~~~~~~~~~~~
+
+Get a key:
+
+>>> key = Key.objects.get_or_create(
+>>>     name='events.group.{}'.format(group_id)
+>>> )
+
+Increment the key for the current time:
+
+>>> now = timezone.now()
+>>> Point.objects.incr(key, timestamp=now)
+
+Periodically flush unused data:
+
+>>> if random.random() > 0.9:
+>>>     Point.objects.trim()
+
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+
+from django.db import models
+
+from sentry.manager import BaseManager
+
+from .manager import PointManager
+from .utils import Granularity
+
+
+class Key(models.Model):
+    name = models.CharField(max_length=1000, unique=True)
+    label = models.CharField(max_length=1000, null=True)
+
+    objects = BaseManager(cache_fields=['name'])
+
+
+class Point(models.Model):
+    key = models.ForeignKey(Key)
+    value = models.PositiveIntegerField(default=0)
+    epoch = models.PositiveIntegerField()
+    granularity = models.PositiveIntegerField(choices=Granularity.get_choices())
+
+    objects = PointManager()
+
+    class Meta:
+        index_together = (
+            ('key', 'granularity', 'value'),
+            ('granularity', 'value'),
+        )

src/sentry/tsdb/utils.py

@@ -0,0 +1,82 @@
+"""
+sentry.tsdb.utils
+~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from datetime import timedelta
+
+ONE_MINUTE = 60
+ONE_HOUR = ONE_MINUTE * 60
+ONE_DAY = ONE_HOUR * 24
+ONE_WEEK = ONE_DAY * 7
+
+
+class Granularity(object):
+    SECONDS = 0
+    MINUTES = 1
+    HOURS = 2
+    DAYS = 3
+    WEEKS = 4
+    MONTHS = 5
+    YEARS = 6
+    ALL_TIME = 7
+
+    @classmethod
+    def get_choices(cls):
+        if hasattr(cls, '__choice_cache'):
+            return cls.__choice_cache
+
+        results = []
+        for name in dir(cls):
+            if name.startswith('_'):
+                continue
+            if not name.upper() == name:
+                continue
+            results.append((getattr(cls, name), name.replace('_', ' ').title()))
+        cls.__choice_cache = results
+        return results
+
+    @classmethod
+    def normalize_to_epoch(cls, granularity, timestamp):
+        timestamp = timestamp.replace(microsecond=0)
+        if granularity == cls.ALL_TIME:
+            return 0
+        if granularity == cls.SECONDS:
+            return timestamp
+        timestamp = timestamp.replace(seconds=0)
+        if granularity == cls.MINUTES:
+            return timestamp
+        timestamp = timestamp.replace(minutes=0)
+        if granularity == cls.HOURS:
+            return timestamp
+        timestamp = timestamp.replace(hours=0)
+        if granularity == cls.WEEKS:
+            timestamp -= timedelta(days=timestamp.weekday())
+        elif granularity in (cls.MONTHS, cls.YEARS):
+            timestamp = timestamp.replace(day=1)
+        elif granularity == cls.YEARS:
+            timestamp = timestamp.replace(month=1)
+        return int(timestamp.strftime('%s'))
+
+    @classmethod
+    def get_min_timestamp(cls, granularity, timestamp):
+        if granularity in (cls.ALL_TIME, cls.YEARS):
+            return None
+
+        if granularity == cls.SECONDS:
+            timestamp -= timedelta(minutes=1)
+        elif granularity == cls.MINUTES:
+            timestamp -= timedelta(hours=1)
+        elif granularity == cls.HOURS:
+            timestamp -= timedelta(days=1)
+        elif granularity == cls.DAYS:
+            # days are stored for a full month
+            timestamp -= timedelta(months=1)
+        elif granularity == cls.WEEKS:
+            # weeks are stored for a full year
+            timestamp -= timedelta(years=1)
+
+        return cls.normalize_to_epoch(timestamp, granularity)