SQLite Import moved to mgmt command for Heroku

ccdb/projects/migrations/0002_initial_grant.py

@@ -7,7 +7,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('projects', '0002_project_data'),
+        ('projects', '0001_initial_project'),
     ]
 
     operations = [

ccdb/projects/migrations/0002_project_data.py

@@ -1,31 +0,0 @@
-from django.db import migrations, models, transaction
-
-from ccdb.utils.data_import import setup_sqlite
-
-
-@transaction.atomic
-def import_projects(apps, schema_editor):
-    Project = apps.get_model('projects', 'Project')
-    c = setup_sqlite()
-    if c:
-        for r in c.execute('SELECT * FROM tbl_lu_projects;'):
-            p = Project(id=r[0], name=r[1], code=r[2], iacuc_number=r[3],
-                description=r[4], sort_order=r[5])
-            p.save()
-
-
-def remove_projects(apps, schema_editor):
-    print("removing projects...")
-    Project = apps.get_model("projects", "Project")
-    Project.objects.all().delete()
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('projects', '0001_initial_project'),
-    ]
-
-    operations = [
-        migrations.RunPython(import_projects, remove_projects),
-    ]

ccdb/projects/migrations/0003_grant_projects.py

@@ -7,7 +7,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('projects', '0004_grant_data'),
+        ('projects', '0002_initial_grant'),
     ]
 
     operations = [

ccdb/projects/migrations/0004_grant_data.py

@@ -1,30 +0,0 @@
-from django.db import migrations, models, transaction
-
-from ccdb.utils.data_import import setup_sqlite
-
-
-@transaction.atomic
-def import_grants(apps, schema_editor):
-    Grant = apps.get_model('projects', 'Grant')
-    c = setup_sqlite()
-    if c:
-        for r in c.execute('SELECT * FROM tbl_lu_grants;'):
-            g = Grant(id=r[0], title=r[1], code=r[2],
-                description=r[3], sort_order=r[4])
-            g.save()
-
-
-def remove_grants(apps, schema_editor):
-    Grant = apps.get_model('projects', 'Grant')
-    Grant.objects.all().delete()
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('projects', '0003_initial_grant'),
-    ]
-
-    operations = [
-        migrations.RunPython(import_grants, remove_grants),
-    ]

ccdb/projects/migrations/0004_initial_grantreport.py

@@ -7,7 +7,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('projects', '0006_project_grant_data'),
+        ('projects', '0003_grant_projects'),
     ]
 
     operations = [

ccdb/projects/migrations/0006_project_grant_data.py

@@ -1,31 +0,0 @@
-from django.db import migrations, models, transaction
-
-from ccdb.utils.data_import import setup_sqlite
-
-
-@transaction.atomic
-def import_project_grant(apps, schema_editor):
-    Project = apps.get_model('projects', 'Project')
-    Grant = apps.get_model('projects', 'Grant')
-    c = setup_sqlite()
-    if c:
-        for r in c.execute('SELECT * FROM tbl_hash_project_grants;'):
-            p = Project.objects.get(id=r[0])
-            g = Grant.objects.get(id=r[1])
-            p.grants.add(g)
-            p.save()
-
-
-def remove_project_grant(apps, schema_editor):
-    pass
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('projects', '0005_grant_projects'),
-    ]
-
-    operations = [
-        migrations.RunPython(import_project_grant, remove_project_grant),
-    ]

ccdb/projects/migrations/0008_grantreport_data.py

@@ -1,37 +0,0 @@
-from django.db import migrations, models, transaction
-
-from ccdb.utils.data_import import setup_sqlite
-
-
-@transaction.atomic
-def import_grantreport(apps, schema_editor):
-    GrantReport = apps.get_model('projects', 'GrantReport')
-    Grant = apps.get_model('projects', 'Grant')
-    c = setup_sqlite()
-    if c:
-        q = '''
-               SELECT *, report_due_date AS "due_date [dtdt]"
-               FROM tbl_lu_grant_reports;
-            '''
-        for r in c.execute(q):
-            g = Grant.objects.get(id=r[0])
-            gr = GrantReport(grant=g, title=r[1], report_type=r[2],
-                description=r[3], due_date=r[8], submitted_date=r[5],
-                attachment=r[6], sort_order=r[7])
-            gr.save()
-
-
-def remove_grantreport(apps, schema_editor):
-    GrantReport = apps.get_model('projects', 'GrantReport')
-    GrantReport.objects.all().delete()
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('projects', '0007_initial_grantreport'),
-    ]
-
-    operations = [
-        migrations.RunPython(import_grantreport, remove_grantreport),
-    ]

ccdb/utils/management/commands/import_data.py

@@ -0,0 +1,73 @@
+import os
+import shutil
+
+from django.core.management.base import BaseCommand
+
+import requests
+
+from ccdb.utils.data_import import setup_sqlite
+from ccdb.projects.models import Project, Grant, GrantReport
+
+
+class Command(BaseCommand):
+    help = 'Imports prior data into the DB'
+
+    def add_arguments(self, parser):
+        parser.add_argument('manifest_url', type=str)
+
+    def handle(self, **options):
+        _fetch_data(options['manifest_url'], self.stdout.write)
+        self.stdout.write('Fetched data')
+        _import_data()
+        self.stdout.write('Imported data')
+
+
+def _fetch_data(url, write):
+    data_dir = 'data/'
+    r = requests.get(url)
+    files = r.json()
+    if not os.path.exists(data_dir):
+        os.makedirs(data_dir)
+    for f in files['files']:
+        p = ''.join([data_dir, f.split('/')[-1]])
+        if not os.path.exists(p):
+            write('Grabbing {}'.format(p))
+            r = requests.get(f, stream=True)
+            with open(p, 'wb') as out_file:
+                for chunk in r:
+                    out_file.write(chunk)
+
+
+def _import_data():
+    c = setup_sqlite()
+    if c:
+        # Projects
+        for r in c.execute('SELECT * FROM tbl_lu_projects;'):
+            p = Project(id=r[0], name=r[1], code=r[2], iacuc_number=r[3],
+                description=r[4], sort_order=r[5])
+            p.save()
+
+        # Grants
+        for r in c.execute('SELECT * FROM tbl_lu_grants;'):
+            g = Grant(id=r[0], title=r[1], code=r[2],
+                description=r[3], sort_order=r[4])
+            g.save()
+
+        # Project-Grants
+        for r in c.execute('SELECT * FROM tbl_hash_project_grants;'):
+            p = Project.objects.get(id=r[0])
+            g = Grant.objects.get(id=r[1])
+            p.grants.add(g)
+            p.save()
+
+        # Grant Reports
+        q = '''
+               SELECT *, report_due_date AS "due_date [dtdt]"
+               FROM tbl_lu_grant_reports;
+            '''
+        for r in c.execute(q):
+            g = Grant.objects.get(id=r[0])
+            gr = GrantReport(grant=g, title=r[1], report_type=r[2],
+                description=r[3], due_date=r[8], submitted_date=r[5],
+                attachment=r[6], sort_order=r[7])
+            gr.save()

config/settings/base.py

@@ -37,6 +37,7 @@ THIRD_PARTY_APPS = (
 
 # Apps specific for this project go here.
 LOCAL_APPS = (
+    'ccdb.utils',
     'ccdb.users',  # custom users app
     'ccdb.projects',
 )

requirements/base.txt

@@ -36,3 +36,6 @@ django-grappelli==2.7.3
 
 # Date/time strings
 python-dateutil==2.4.2
+
+# HTTP
+requests==2.9.1