Migrate to Grout

Migration instructions:
1. For safety, comment out the migrations task in
  deployment/ansible/roles/driver.app/tasks/main.yml
2. `vagrant provision` (it will probably fail)
3. `./scripts/manage.sh migrate grout 0022 --fake`
4. `./scripts/manage.sh migrate data 0011`
5. `./scripts/manage.sh migrate grout 0025`
6. `./scripts/manage.sh migrate data`
7. `vagrant provision` (it should work)

Author: ddohler

README.md

@@ -46,16 +46,6 @@ DRIVER - Data for Road Incident Visualization, Evaluation, and Reporting
     sudo apt-get install nfs-common nfs-kernel-server
     ```
 
-1. Clone the [Ashlar](https://github.com/azavea/ashlar) project such that it is a sibling of DRIVER (Ashlar is needed for generating dynamic schemas used by DRIVER).
-
-    An example setup might be as follows:
-    ```bash
-    /home/username/
-      git
-        DRIVER
-        ashlar
-    ```
-
 1. Start the Vagrant VM
     ```bash
     vagrant up

Vagrantfile

@@ -90,7 +90,6 @@ Vagrant.configure("2") do |config|
     app.vm.synced_folder "./web", "/opt/web", type: "nfs", mount_options: MOUNT_OPTIONS
     app.vm.synced_folder "./windshaft", "/opt/windshaft", type: "nfs", mount_options: MOUNT_OPTIONS
     app.vm.synced_folder "./schema_editor", "/opt/schema_editor", type: "nfs", mount_options: MOUNT_OPTIONS
-    app.vm.synced_folder "../ashlar", "/opt/ashlar", type: "nfs", mount_options: MOUNT_OPTIONS
 
     # nginx
     app.vm.network "forwarded_port", guest: 80, host: Integer(ENV.fetch("DRIVER_WEB_PORT_80", 7000))
@@ -136,7 +135,6 @@ Vagrant.configure("2") do |config|
     celery.vm.synced_folder "./web", "/opt/web", type: "nfs", mount_options: MOUNT_OPTIONS
     celery.vm.synced_folder "./analysis_tasks", "/opt/analysis_tasks", type: "nfs", mount_options: MOUNT_OPTIONS
     celery.vm.synced_folder "./schema_editor", "/opt/schema_editor", type: "nfs", mount_options: MOUNT_OPTIONS
-    celery.vm.synced_folder "../ashlar", "/opt/ashlar", type: "nfs", mount_options: MOUNT_OPTIONS
     # jar build task on celery vm
     celery.vm.synced_folder "./gradle", "/opt/gradle", type: "nfs", mount_options: MOUNT_OPTIONS
 

app/black_spots/filters.py

@@ -4,8 +4,8 @@
 
 from django.db.models import Q
 
-from ashlar.exceptions import QueryParameterException
-from ashlar.models import BoundaryPolygon
+from grout.exceptions import QueryParameterException
+from grout.models import BoundaryPolygon
 
 from black_spots.models import (BlackSpot, BlackSpotSet)
 

app/black_spots/migrations/0001_initial.py

@@ -9,7 +9,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ashlar', '0021_add_weather_fields'),
+        ('grout', '0021_add_weather_fields'),
     ]
 
     operations = [
@@ -36,7 +36,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(auto_now=True)),
                 ('effective_start', models.DateTimeField()),
                 ('effective_end', models.DateTimeField(null=True, blank=True)),
-                ('record_type', models.ForeignKey(to='ashlar.RecordType')),
+                ('record_type', models.ForeignKey(to='grout.RecordType')),
             ],
             options={
                 'abstract': False,

app/black_spots/models.py

@@ -1,16 +1,16 @@
 from django.conf import settings
 from django.contrib.gis.db import models
 from django.core.validators import MaxValueValidator, MinValueValidator
-from ashlar.models import AshlarModel
+from grout.models import GroutModel
 
 
-class BlackSpot(AshlarModel):
+class BlackSpot(GroutModel):
     """A black spot -- an area where there is an historical/statistical
     concentration of records
     """
 
     #: Buffered road segment polygon where black spot analysis is performed
-    geom = models.PolygonField(srid=settings.ASHLAR['SRID'])
+    geom = models.PolygonField(srid=settings.GROUT['SRID'])
 
     #: Number that determines the severity of this black spot. May be used for color-coding on map.
     severity_score = models.FloatField()
@@ -37,7 +37,7 @@ def longitude(self):
         return self.geom.centroid.x
 
 
-class BlackSpotSet(AshlarModel):
+class BlackSpotSet(GroutModel):
     """A grouping of black spots generated at one time"""
 
     #: DateTime when the black spots become effective
@@ -48,10 +48,10 @@ class BlackSpotSet(AshlarModel):
     effective_end = models.DateTimeField(null=True, blank=True)
 
     #: The record type these black spots are associated with
-    record_type = models.ForeignKey('ashlar.RecordType')
+    record_type = models.ForeignKey('grout.RecordType')
 
 
-class BlackSpotConfig(AshlarModel):
+class BlackSpotConfig(GroutModel):
     """Holds user-configurable settings for how black spot generation should work"""
     #: Blackspot severity percentile cutoff; segments with forecast severity above this threshold
     #: will be considered blackspots.
@@ -60,27 +60,27 @@ class BlackSpotConfig(AshlarModel):
                                                                   MinValueValidator(0.0)])
 
 
-class BlackSpotRecordsFile(AshlarModel):
+class BlackSpotRecordsFile(GroutModel):
     """Model to track blackspot record csvs"""
     #: Path to csvs
     csv = models.FileField(upload_to='blackspot_records/')
 
 
-class RoadSegmentsShapefile(AshlarModel):
+class RoadSegmentsShapefile(GroutModel):
     """Model to track gzipped shapefile for road segments training input"""
 
     #: Path to gzipped shapefile
     shp_tgz = models.FileField(upload_to='road_segments/')
 
 
-class BlackSpotTrainingCsv(AshlarModel):
+class BlackSpotTrainingCsv(GroutModel):
     """Model to track blackspot training csvs"""
 
     #: Path to csvs
     csv = models.FileField(upload_to='training/blackspot/')
 
 
-class LoadForecastTrainingCsv(AshlarModel):
+class LoadForecastTrainingCsv(GroutModel):
     """Model to track forecast training csvs"""
     #: Path to csvs
     csv = models.FileField(upload_to='training/forecast')

app/black_spots/serializers.py

@@ -2,7 +2,7 @@
 from rest_framework.exceptions import ParseError, NotFound
 from rest_framework.serializers import ModelSerializer
 
-from ashlar.models import BoundaryPolygon
+from grout.models import BoundaryPolygon
 from black_spots.models import (BlackSpot, BlackSpotSet, BlackSpotConfig)
 from black_spots.filters import parse_and_validate_dt
 
@@ -37,6 +37,7 @@ class Meta:
         fields = ('black_spot_set', 'geom', 'latitude', 'longitude', 'severity_score',)
         read_only_fields = ('uuid',)
 
+
 class EnforcerAssignmentInputSerializer():
     """Serializer for enforcer assignment inputs"""
 

app/black_spots/tasks/calculate_black_spots.py

@@ -10,7 +10,7 @@
 from celery import shared_task, chain
 from celery.utils.log import get_task_logger
 
-from ashlar.models import RecordType
+from grout.models import RecordType
 
 from black_spots.tasks import (forecast_segment_incidents, load_blackspot_geoms,
                                load_road_network, get_training_noprecip)

app/black_spots/tasks/load_blackspot_geoms.py

@@ -17,7 +17,7 @@
 from django.contrib.gis.geos import GEOSGeometry
 from django.utils import timezone
 
-from ashlar.models import RecordType
+from grout.models import RecordType
 from black_spots.models import BlackSpot, BlackSpotSet
 
 logger = get_task_logger(__name__)
@@ -200,7 +200,7 @@ def blackspot_from_feature(feature, black_spot_set, src_srid, score_prop=None, r
     """
     blackspot = BlackSpot()
     geometry = GEOSGeometry(buffer(shape(feature['geometry']).wkb), srid=src_srid)
-    geometry.transform(settings.ASHLAR['SRID'])
+    geometry.transform(settings.GROUT['SRID'])
     blackspot.geom = geometry
     blackspot.black_spot_set = black_spot_set
     blackspot.severity_score = feature['properties'][score_prop] if score_prop else 0.0

app/black_spots/tests/tests.py

@@ -7,7 +7,7 @@
 
 from rest_framework.test import APIClient, APITestCase, APIRequestFactory
 
-from ashlar.models import RecordType
+from grout.models import RecordType
 
 from black_spots.models import (BlackSpot, BlackSpotSet)
 

app/black_spots/views.py

@@ -4,7 +4,7 @@
 from django_redis import get_redis_connection
 from django.utils import timezone
 
-from ashlar.models import Record
+from data.models import DriverRecord
 from black_spots.models import (BlackSpot, BlackSpotSet, BlackSpotConfig)
 from black_spots.serializers import (BlackSpotSerializer, BlackSpotSetSerializer,
                                      BlackSpotConfigSerializer, EnforcerAssignmentInputSerializer,
@@ -14,9 +14,12 @@
 
 from driver_auth.permissions import IsAdminOrReadOnly
 from driver import mixins
-import datetime, random, uuid
+import datetime
+import random
+import uuid
 from dateutil import rrule
 
+
 class BlackSpotViewSet(viewsets.ModelViewSet, mixins.GenerateViewsetQuery):
     """ViewSet for black spots"""
     queryset = BlackSpot.objects.all()
@@ -132,7 +135,7 @@ def scale_by_toddow(self, assignments, shift_start, shift_end, record_type, geom
         num_days_events = 365
         max_dt = timezone.now()
         min_dt = max_dt - datetime.timedelta(days=num_days_events)
-        records = Record.objects.filter(
+        records = DriverRecord.objects.filter(
             occurred_from__gte=min_dt,
             occurred_to__lte=max_dt,
             schema__record_type_id=record_type