From d96487c943ae738bb1b477fceb566101ac553bf7 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Thu, 21 Jan 2016 16:22:07 -0700 Subject: [PATCH] Import from sqlite --- ccdb/projects/migrations/0002_project_data.py | 25 +++++-------- ccdb/projects/migrations/0004_grant_data.py | 24 +++++-------- .../migrations/0006_project_grant_data.py | 29 ++++++--------- .../migrations/0008_grantreport_data.py | 36 ++++++++----------- ccdb/utils/__init__.py | 0 ccdb/utils/data_import.py | 21 +++++++++++ requirements/base.txt | 3 ++ 7 files changed, 68 insertions(+), 70 deletions(-) create mode 100644 ccdb/utils/__init__.py create mode 100644 ccdb/utils/data_import.py diff --git a/ccdb/projects/migrations/0002_project_data.py b/ccdb/projects/migrations/0002_project_data.py index a0ba64e..e8c26aa 100644 --- a/ccdb/projects/migrations/0002_project_data.py +++ b/ccdb/projects/migrations/0002_project_data.py @@ -1,24 +1,17 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +from django.db import migrations, models, transaction -import csv -import os - -from django.db import migrations, models +from ccdb.utils.data_import import setup_sqlite +@transaction.atomic def import_projects(apps, schema_editor): Project = apps.get_model('projects', 'Project') - filename = 'data/tbl_LU_Projects.csv' - if os.path.exists(filename): - with open(filename) as f: - fieldnames = ['id', 'name', 'code', 'iacuc_number', - 'description', 'sort_order'] - reader = csv.DictReader(f, fieldnames=fieldnames) - for r in reader: - r['sort_order'] = int(float(r['sort_order'])) - p = Project(**r) - p.save() + c = setup_sqlite() + if c: + for r in c.execute('SELECT * FROM tbl_lu_projects;'): + p = Project(id=r[0], name=r[1], code=r[2], iacuc_number=r[3], + description=r[4], sort_order=r[5]) + p.save() def remove_projects(apps, schema_editor): diff --git a/ccdb/projects/migrations/0004_grant_data.py b/ccdb/projects/migrations/0004_grant_data.py index 480a542..2b078fb 100644 --- a/ccdb/projects/migrations/0004_grant_data.py +++ b/ccdb/projects/migrations/0004_grant_data.py @@ -1,23 +1,17 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +from django.db import migrations, models, transaction -import csv -import os - -from django.db import migrations, models +from ccdb.utils.data_import import setup_sqlite +@transaction.atomic def import_grants(apps, schema_editor): Grant = apps.get_model('projects', 'Grant') - filename = 'data/tbl_LU_Grants.csv' - if os.path.exists(filename): - with open(filename) as f: - fieldnames = ['id', 'title', 'code', 'description', 'sort_order'] - reader = csv.DictReader(f, fieldnames=fieldnames) - for r in reader: - r['sort_order'] = None - g = Grant(**r) - g.save() + c = setup_sqlite() + if c: + for r in c.execute('SELECT * FROM tbl_lu_grants;'): + g = Grant(id=r[0], title=r[1], code=r[2], + description=r[3], sort_order=r[4]) + g.save() def remove_grants(apps, schema_editor): diff --git a/ccdb/projects/migrations/0006_project_grant_data.py b/ccdb/projects/migrations/0006_project_grant_data.py index e91b7e1..090dc8c 100644 --- a/ccdb/projects/migrations/0006_project_grant_data.py +++ b/ccdb/projects/migrations/0006_project_grant_data.py @@ -1,30 +1,23 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +from django.db import migrations, models, transaction -import csv -import os - -from django.db import migrations, models +from ccdb.utils.data_import import setup_sqlite +@transaction.atomic def import_project_grant(apps, schema_editor): Project = apps.get_model('projects', 'Project') Grant = apps.get_model('projects', 'Grant') - filename = 'data/tbl_HASH_Project_Grants.csv' - if os.path.exists(filename): - with open(filename) as f: - fieldnames = ['project', 'grant'] - reader = csv.DictReader(f, fieldnames=fieldnames) - for r in reader: - p = Project.objects.get(id=r['project']) - g = Grant.objects.get(id=r['grant']) - p.grants.add(g) - p.save() + c = setup_sqlite() + if c: + for r in c.execute('SELECT * FROM tbl_hash_project_grants;'): + p = Project.objects.get(id=r[0]) + g = Grant.objects.get(id=r[1]) + p.grants.add(g) + p.save() def remove_project_grant(apps, schema_editor): - Grant = apps.get_model('projects', 'Grant') - Grant.projects.clear() + pass class Migration(migrations.Migration): diff --git a/ccdb/projects/migrations/0008_grantreport_data.py b/ccdb/projects/migrations/0008_grantreport_data.py index 46803fd..efef7e0 100644 --- a/ccdb/projects/migrations/0008_grantreport_data.py +++ b/ccdb/projects/migrations/0008_grantreport_data.py @@ -1,30 +1,24 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +from django.db import migrations, models, transaction -import csv -import os -from datetime import datetime - -from django.db import migrations, models +from ccdb.utils.data_import import setup_sqlite +@transaction.atomic def import_grantreport(apps, schema_editor): GrantReport = apps.get_model('projects', 'GrantReport') Grant = apps.get_model('projects', 'Grant') - filename = 'data/tbl_LU_Grant_Reports.csv' - if os.path.exists(filename): - with open(filename) as f: - fieldnames = ['id', 'grant_id', 'title', 'report_type', 'description', - 'due_date', 'submitted_date', 'attachment', 'sort_order'] - reader = csv.DictReader(f, fieldnames=fieldnames) - for r in reader: - r['sort_order'] = None - r['due_date'] = datetime.strptime(' '.join(r['due_date'].split(' AKDT ')), '%a %b %d %H:%M:%S %Y') - r['submitted_date'] = None - grant_id = r.pop('grant_id') - g = Grant.objects.get(id=grant_id) - gr = GrantReport(grant=g, **r) - gr.save() + c = setup_sqlite() + if c: + q = ''' + SELECT *, report_due_date AS "due_date [dtdt]" + FROM tbl_lu_grant_reports; + ''' + for r in c.execute(q): + g = Grant.objects.get(id=r[0]) + gr = GrantReport(grant=g, title=r[1], report_type=r[2], + description=r[3], due_date=r[8], submitted_date=r[5], + attachment=r[6], sort_order=r[7]) + gr.save() def remove_grantreport(apps, schema_editor): diff --git a/ccdb/utils/__init__.py b/ccdb/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ccdb/utils/data_import.py b/ccdb/utils/data_import.py new file mode 100644 index 0000000..beaf92f --- /dev/null +++ b/ccdb/utils/data_import.py @@ -0,0 +1,21 @@ +import sqlite3 +import os + +import dateutil.parser as dp + + +def dtdt(s): + return dp.parse(s) + + +sqlite3.register_converter("dtdt", dtdt) + + +def setup_sqlite(): + dbfile = 'data/CC_Database_101314.sqlite' + if os.path.exists(dbfile): + db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) + db.row_factory = sqlite3.Row + return db.cursor() + else: + return None diff --git a/requirements/base.txt b/requirements/base.txt index 2f4bdf5..599693f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -33,3 +33,6 @@ django-tables2==1.0.4 # Admin django-grappelli==2.7.3 + +# Date/time strings +python-dateutil==2.4.2