[ON HOLD] Fetch Data Statistics

contrib/start.sh

@@ -2,12 +2,19 @@
 python manage.py migrate --noinput                # Apply database migrations
 python manage.py collectstatic --clear --noinput  # Collect static files
 
+# recreate the cached static json files
+python manage.py cache_static_json_data --interval 5m
+python manage.py cache_static_json_data --interval 1h
+python manage.py cache_static_json_data --interval 24h
+
 # Prepare log files and start outputting logs to stdout
 touch /src/logs/celery.log
 touch /src/logs/gunicorn.log
 touch /src/logs/access.log
 tail -n 0 -f /src/logs/*.log &
 
+#purge all configured task queues
+celery -A sensorsafrica purge
 celery -A sensorsafrica beat -l info &> /src/logs/celery.log  &
 celery -A sensorsafrica worker --hostname=$DOKKU_APP_NAME -l info &> /src/logs/celery.log  &
 celery -A sensorsafrica flower --basic_auth=$SENSORSAFRICA_FLOWER_ADMIN_USERNAME:$SENSORSAFRICA_FLOWER_ADMIN_PASSWORD &> /src/logs/celery.log  &

sensorsafrica/admin.py

@@ -2,7 +2,7 @@
 from django.utils.html import format_html
 from django.conf.urls import include, url
 from django.template.response import TemplateResponse
-from .api.models import LastActiveNodes, SensorDataStat, City
+from .api.models import LastActiveNodes, City
 from django.db.models import Q
 
 from feinstaub.sensors.admin import (
@@ -78,56 +78,6 @@ def save_related(self, request, form, formsets, change):
         pass
 
 
-@admin.register(SensorDataStat)
-class SensorDataStatAdmin(admin.ModelAdmin):
-    readonly_fields = [
-        "node",
-        "sensor",
-        "location",
-        "city_slug",
-        "value_type",
-        "average",
-        "maximum",
-        "minimum",
-        "timestamp",
-    ]
-    search_fields = ["city_slug", "value_type"]
-    list_display = [
-        "node",
-        "sensor",
-        "location",
-        "city_slug",
-        "value_type",
-        "average",
-        "maximum",
-        "minimum",
-        "timestamp",
-        "created",
-        "modified",
-    ]
-    list_filter = ["timestamp", "node", "sensor", "location"]
-
-    def get_actions(self, request):
-        actions = super(SensorDataStatAdmin, self).get_actions(request)
-        del actions["delete_selected"]
-        return actions
-
-    def has_add_permission(self, request):
-        return False
-
-    def has_delete_permission(self, request, obj=None):
-        return False
-
-    def save_model(self, request, obj, form, change):
-        pass
-
-    def delete_model(self, request, obj):
-        pass
-
-    def save_related(self, request, form, formsets, change):
-        pass
-
-
 @admin.register(City)
 class CityAdmin(admin.ModelAdmin):
     search_fields = ["slug", "name", "country"]

sensorsafrica/api/models.py

@@ -5,12 +5,18 @@
 
 
 class City(TimeStampedModel):
-    slug = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    name = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    country = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    location = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    latitude = models.DecimalField(max_digits=14, decimal_places=11, null=True, blank=True)
-    longitude = models.DecimalField(max_digits=14, decimal_places=11, null=True, blank=True)
+    slug = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    name = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    country = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    location = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    latitude = models.DecimalField(
+        max_digits=14, decimal_places=11, null=True, blank=True)
+    longitude = models.DecimalField(
+        max_digits=14, decimal_places=11, null=True, blank=True)
 
     class Meta:
         verbose_name_plural = "Cities"
@@ -20,36 +26,6 @@ def save(self, *args, **kwargs):
         return super(City, self).save(*args, **kwargs)
 
 
-class SensorDataStat(TimeStampedModel):
-    node = models.ForeignKey(Node)
-    sensor = models.ForeignKey(Sensor)
-    location = models.ForeignKey(SensorLocation)
-
-    city_slug = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    value_type = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-
-    average = models.FloatField(null=False, blank=False)
-    maximum = models.FloatField(null=False, blank=False)
-    minimum = models.FloatField(null=False, blank=False)
-
-    # Number of data points averaged
-    sample_size = models.IntegerField(null=False, blank=False)
-    # Last datetime of calculated stats
-    last_datetime = models.DateTimeField()
-
-    timestamp = models.DateTimeField()
-
-    def __str__(self):
-        return "%s %s %s avg=%s min=%s max=%s" % (
-            self.timestamp,
-            self.city_slug,
-            self.value_type,
-            self.average,
-            self.minimum,
-            self.maximum,
-        )
-
-
 class LastActiveNodes(TimeStampedModel):
     node = models.ForeignKey(Node)
     location = models.ForeignKey(SensorLocation)

sensorsafrica/api/v2/router.py

@@ -5,7 +5,7 @@
 
 data_router = routers.DefaultRouter()
 
-data_router.register(r"", SensorDataStatView)
+data_router.register(r"", SensorDataStatView, basename="sensor_data_stat_view")
 
 city_router = routers.DefaultRouter()
 

sensorsafrica/api/v2/serializers.py

@@ -1,14 +1,35 @@
 from rest_framework import serializers
 
 
-class SensorDataStatSerializer(serializers.Serializer):
-    average = serializers.FloatField()
-    minimum = serializers.FloatField()
-    maximum = serializers.FloatField()
-    value_type = serializers.CharField(max_length=200)
-    start_datetime = serializers.DateTimeField()
-    end_datetime = serializers.DateTimeField()
-    city_slug = serializers.CharField(max_length=200)
+class RawSensorDataStatSerializer(serializers.Serializer):
+    average = serializers.SerializerMethodField()
+    minimum = serializers.SerializerMethodField()
+    maximum = serializers.SerializerMethodField()
+    value_type = serializers.SerializerMethodField()
+    start_datetime = serializers.SerializerMethodField()
+    end_datetime = serializers.SerializerMethodField()
+    city_name = serializers.SerializerMethodField()
+
+    def get_city_name(self, obj):
+        return obj[0]
+
+    def get_start_datetime(self, obj):
+        return obj[1]
+
+    def get_end_datetime(self, obj):
+        return obj[2]
+
+    def get_average(self, obj):
+        return obj[3]
+
+    def get_minimum(self, obj):
+        return obj[4]
+
+    def get_maximum(self, obj):
+        return obj[5]
+
+    def get_value_type(self, obj):
+        return obj[6]
 
 
 class CitySerializer(serializers.Serializer):