Import from sqlite

ccdb/projects/migrations/0002_project_data.py

@@ -1,24 +1,17 @@
-# -*- coding: utf-8 -*-
+from django.db import migrations, models, transaction
-from __future__ import unicode_literals
 
-import csv
+from ccdb.utils.data_import import setup_sqlite
-import os
-
-from django.db import migrations, models
 
 
+@transaction.atomic
 def import_projects(apps, schema_editor):
     Project = apps.get_model('projects', 'Project')
-    filename = 'data/tbl_LU_Projects.csv'
+    c = setup_sqlite()
-    if os.path.exists(filename):
+    if c:
-        with open(filename) as f:
+        for r in c.execute('SELECT * FROM tbl_lu_projects;'):
-            fieldnames = ['id', 'name', 'code', 'iacuc_number',
+            p = Project(id=r[0], name=r[1], code=r[2], iacuc_number=r[3],
-                'description', 'sort_order']
+                description=r[4], sort_order=r[5])
-            reader = csv.DictReader(f, fieldnames=fieldnames)
+            p.save()
-            for r in reader:
-                r['sort_order'] = int(float(r['sort_order']))
-                p = Project(**r)
-                p.save()
 
 
 def remove_projects(apps, schema_editor):

ccdb/projects/migrations/0004_grant_data.py

@@ -1,23 +1,17 @@
-# -*- coding: utf-8 -*-
+from django.db import migrations, models, transaction
-from __future__ import unicode_literals
 
-import csv
+from ccdb.utils.data_import import setup_sqlite
-import os
-
-from django.db import migrations, models
 
 
+@transaction.atomic
 def import_grants(apps, schema_editor):
     Grant = apps.get_model('projects', 'Grant')
-    filename = 'data/tbl_LU_Grants.csv'
+    c = setup_sqlite()
-    if os.path.exists(filename):
+    if c:
-        with open(filename) as f:
+        for r in c.execute('SELECT * FROM tbl_lu_grants;'):
-            fieldnames = ['id', 'title', 'code', 'description', 'sort_order']
+            g = Grant(id=r[0], title=r[1], code=r[2],
-            reader = csv.DictReader(f, fieldnames=fieldnames)
+                description=r[3], sort_order=r[4])
-            for r in reader:
+            g.save()
-                r['sort_order'] = None
-                g = Grant(**r)
-                g.save()
 
 
 def remove_grants(apps, schema_editor):

ccdb/projects/migrations/0006_project_grant_data.py

@@ -1,30 +1,23 @@
-# -*- coding: utf-8 -*-
+from django.db import migrations, models, transaction
-from __future__ import unicode_literals
 
-import csv
+from ccdb.utils.data_import import setup_sqlite
-import os
-
-from django.db import migrations, models
 
 
+@transaction.atomic
 def import_project_grant(apps, schema_editor):
     Project = apps.get_model('projects', 'Project')
     Grant = apps.get_model('projects', 'Grant')
-    filename = 'data/tbl_HASH_Project_Grants.csv'
+    c = setup_sqlite()
-    if os.path.exists(filename):
+    if c:
-        with open(filename) as f:
+        for r in c.execute('SELECT * FROM tbl_hash_project_grants;'):
-            fieldnames = ['project', 'grant']
+            p = Project.objects.get(id=r[0])
-            reader = csv.DictReader(f, fieldnames=fieldnames)
+            g = Grant.objects.get(id=r[1])
-            for r in reader:
+            p.grants.add(g)
-                p = Project.objects.get(id=r['project'])
+            p.save()
-                g = Grant.objects.get(id=r['grant'])
-                p.grants.add(g)
-                p.save()
 
 
 def remove_project_grant(apps, schema_editor):
-    Grant = apps.get_model('projects', 'Grant')
+    pass
-    Grant.projects.clear()
 
 
 class Migration(migrations.Migration):

ccdb/projects/migrations/0008_grantreport_data.py

@@ -1,30 +1,24 @@
-# -*- coding: utf-8 -*-
+from django.db import migrations, models, transaction
-from __future__ import unicode_literals
 
-import csv
+from ccdb.utils.data_import import setup_sqlite
-import os
-from datetime import datetime
-
-from django.db import migrations, models
 
 
+@transaction.atomic
 def import_grantreport(apps, schema_editor):
     GrantReport = apps.get_model('projects', 'GrantReport')
     Grant = apps.get_model('projects', 'Grant')
-    filename = 'data/tbl_LU_Grant_Reports.csv'
+    c = setup_sqlite()
-    if os.path.exists(filename):
+    if c:
-        with open(filename) as f:
+        q = '''
-            fieldnames = ['id', 'grant_id', 'title', 'report_type', 'description',
+               SELECT *, report_due_date AS "due_date [dtdt]"
-                'due_date', 'submitted_date', 'attachment', 'sort_order']
+               FROM tbl_lu_grant_reports;
-            reader = csv.DictReader(f, fieldnames=fieldnames)
+            '''
-            for r in reader:
+        for r in c.execute(q):
-                r['sort_order'] = None
+            g = Grant.objects.get(id=r[0])
-                r['due_date'] = datetime.strptime(' '.join(r['due_date'].split(' AKDT ')), '%a %b %d %H:%M:%S %Y')
+            gr = GrantReport(grant=g, title=r[1], report_type=r[2],
-                r['submitted_date'] = None
+                description=r[3], due_date=r[8], submitted_date=r[5],
-                grant_id = r.pop('grant_id')
+                attachment=r[6], sort_order=r[7])
-                g = Grant.objects.get(id=grant_id)
+            gr.save()
-                gr = GrantReport(grant=g, **r)
-                gr.save()
 
 
 def remove_grantreport(apps, schema_editor):

ccdb/utils/data_import.py

@@ -0,0 +1,21 @@
+import sqlite3
+import os
+
+import dateutil.parser as dp
+
+
+def dtdt(s):
+    return dp.parse(s)
+
+
+sqlite3.register_converter("dtdt", dtdt)
+
+
+def setup_sqlite():
+    dbfile = 'data/CC_Database_101314.sqlite'
+    if os.path.exists(dbfile):
+        db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
+        db.row_factory = sqlite3.Row
+        return db.cursor()
+    else:
+        return None

requirements/base.txt

@@ -33,3 +33,6 @@ django-tables2==1.0.4
 
 # Admin
 django-grappelli==2.7.3
+
+# Date/time strings
+python-dateutil==2.4.2