From ffbfdce5b294d6ee82dec17da647feffce0e1edf Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Thu, 28 Jan 2016 11:01:06 -0700 Subject: [PATCH] SQLite Import moved to mgmt command for Heroku --- ...initial_grant.py => 0002_initial_grant.py} | 2 +- ccdb/projects/migrations/0002_project_data.py | 31 -------- ...ant_projects.py => 0003_grant_projects.py} | 2 +- ccdb/projects/migrations/0004_grant_data.py | 30 -------- ...treport.py => 0004_initial_grantreport.py} | 2 +- .../migrations/0006_project_grant_data.py | 31 -------- .../migrations/0008_grantreport_data.py | 37 ---------- ccdb/utils/management/__init__.py | 0 ccdb/utils/management/commands/__init__.py | 0 ccdb/utils/management/commands/import_data.py | 73 +++++++++++++++++++ config/settings/base.py | 1 + requirements/base.txt | 3 + 12 files changed, 80 insertions(+), 132 deletions(-) rename ccdb/projects/migrations/{0003_initial_grant.py => 0002_initial_grant.py} (95%) delete mode 100644 ccdb/projects/migrations/0002_project_data.py rename ccdb/projects/migrations/{0005_grant_projects.py => 0003_grant_projects.py} (89%) delete mode 100644 ccdb/projects/migrations/0004_grant_data.py rename ccdb/projects/migrations/{0007_initial_grantreport.py => 0004_initial_grantreport.py} (95%) delete mode 100644 ccdb/projects/migrations/0006_project_grant_data.py delete mode 100644 ccdb/projects/migrations/0008_grantreport_data.py create mode 100644 ccdb/utils/management/__init__.py create mode 100644 ccdb/utils/management/commands/__init__.py create mode 100644 ccdb/utils/management/commands/import_data.py diff --git a/ccdb/projects/migrations/0003_initial_grant.py b/ccdb/projects/migrations/0002_initial_grant.py similarity index 95% rename from ccdb/projects/migrations/0003_initial_grant.py rename to ccdb/projects/migrations/0002_initial_grant.py index eb7d15f..e2fd99e 100644 --- a/ccdb/projects/migrations/0003_initial_grant.py +++ b/ccdb/projects/migrations/0002_initial_grant.py @@ -7,7 +7,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('projects', '0002_project_data'), + ('projects', '0001_initial_project'), ] operations = [ diff --git a/ccdb/projects/migrations/0002_project_data.py b/ccdb/projects/migrations/0002_project_data.py deleted file mode 100644 index e8c26aa..0000000 --- a/ccdb/projects/migrations/0002_project_data.py +++ /dev/null @@ -1,31 +0,0 @@ -from django.db import migrations, models, transaction - -from ccdb.utils.data_import import setup_sqlite - - -@transaction.atomic -def import_projects(apps, schema_editor): - Project = apps.get_model('projects', 'Project') - c = setup_sqlite() - if c: - for r in c.execute('SELECT * FROM tbl_lu_projects;'): - p = Project(id=r[0], name=r[1], code=r[2], iacuc_number=r[3], - description=r[4], sort_order=r[5]) - p.save() - - -def remove_projects(apps, schema_editor): - print("removing projects...") - Project = apps.get_model("projects", "Project") - Project.objects.all().delete() - - -class Migration(migrations.Migration): - - dependencies = [ - ('projects', '0001_initial_project'), - ] - - operations = [ - migrations.RunPython(import_projects, remove_projects), - ] diff --git a/ccdb/projects/migrations/0005_grant_projects.py b/ccdb/projects/migrations/0003_grant_projects.py similarity index 89% rename from ccdb/projects/migrations/0005_grant_projects.py rename to ccdb/projects/migrations/0003_grant_projects.py index 5bdb7eb..f226025 100644 --- a/ccdb/projects/migrations/0005_grant_projects.py +++ b/ccdb/projects/migrations/0003_grant_projects.py @@ -7,7 +7,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('projects', '0004_grant_data'), + ('projects', '0002_initial_grant'), ] operations = [ diff --git a/ccdb/projects/migrations/0004_grant_data.py b/ccdb/projects/migrations/0004_grant_data.py deleted file mode 100644 index 2b078fb..0000000 --- a/ccdb/projects/migrations/0004_grant_data.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.db import migrations, models, transaction - -from ccdb.utils.data_import import setup_sqlite - - -@transaction.atomic -def import_grants(apps, schema_editor): - Grant = apps.get_model('projects', 'Grant') - c = setup_sqlite() - if c: - for r in c.execute('SELECT * FROM tbl_lu_grants;'): - g = Grant(id=r[0], title=r[1], code=r[2], - description=r[3], sort_order=r[4]) - g.save() - - -def remove_grants(apps, schema_editor): - Grant = apps.get_model('projects', 'Grant') - Grant.objects.all().delete() - - -class Migration(migrations.Migration): - - dependencies = [ - ('projects', '0003_initial_grant'), - ] - - operations = [ - migrations.RunPython(import_grants, remove_grants), - ] diff --git a/ccdb/projects/migrations/0007_initial_grantreport.py b/ccdb/projects/migrations/0004_initial_grantreport.py similarity index 95% rename from ccdb/projects/migrations/0007_initial_grantreport.py rename to ccdb/projects/migrations/0004_initial_grantreport.py index c3566d5..d7c7a39 100644 --- a/ccdb/projects/migrations/0007_initial_grantreport.py +++ b/ccdb/projects/migrations/0004_initial_grantreport.py @@ -7,7 +7,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('projects', '0006_project_grant_data'), + ('projects', '0003_grant_projects'), ] operations = [ diff --git a/ccdb/projects/migrations/0006_project_grant_data.py b/ccdb/projects/migrations/0006_project_grant_data.py deleted file mode 100644 index 090dc8c..0000000 --- a/ccdb/projects/migrations/0006_project_grant_data.py +++ /dev/null @@ -1,31 +0,0 @@ -from django.db import migrations, models, transaction - -from ccdb.utils.data_import import setup_sqlite - - -@transaction.atomic -def import_project_grant(apps, schema_editor): - Project = apps.get_model('projects', 'Project') - Grant = apps.get_model('projects', 'Grant') - c = setup_sqlite() - if c: - for r in c.execute('SELECT * FROM tbl_hash_project_grants;'): - p = Project.objects.get(id=r[0]) - g = Grant.objects.get(id=r[1]) - p.grants.add(g) - p.save() - - -def remove_project_grant(apps, schema_editor): - pass - - -class Migration(migrations.Migration): - - dependencies = [ - ('projects', '0005_grant_projects'), - ] - - operations = [ - migrations.RunPython(import_project_grant, remove_project_grant), - ] diff --git a/ccdb/projects/migrations/0008_grantreport_data.py b/ccdb/projects/migrations/0008_grantreport_data.py deleted file mode 100644 index efef7e0..0000000 --- a/ccdb/projects/migrations/0008_grantreport_data.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.db import migrations, models, transaction - -from ccdb.utils.data_import import setup_sqlite - - -@transaction.atomic -def import_grantreport(apps, schema_editor): - GrantReport = apps.get_model('projects', 'GrantReport') - Grant = apps.get_model('projects', 'Grant') - c = setup_sqlite() - if c: - q = ''' - SELECT *, report_due_date AS "due_date [dtdt]" - FROM tbl_lu_grant_reports; - ''' - for r in c.execute(q): - g = Grant.objects.get(id=r[0]) - gr = GrantReport(grant=g, title=r[1], report_type=r[2], - description=r[3], due_date=r[8], submitted_date=r[5], - attachment=r[6], sort_order=r[7]) - gr.save() - - -def remove_grantreport(apps, schema_editor): - GrantReport = apps.get_model('projects', 'GrantReport') - GrantReport.objects.all().delete() - - -class Migration(migrations.Migration): - - dependencies = [ - ('projects', '0007_initial_grantreport'), - ] - - operations = [ - migrations.RunPython(import_grantreport, remove_grantreport), - ] diff --git a/ccdb/utils/management/__init__.py b/ccdb/utils/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ccdb/utils/management/commands/__init__.py b/ccdb/utils/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ccdb/utils/management/commands/import_data.py b/ccdb/utils/management/commands/import_data.py new file mode 100644 index 0000000..047c883 --- /dev/null +++ b/ccdb/utils/management/commands/import_data.py @@ -0,0 +1,73 @@ +import os +import shutil + +from django.core.management.base import BaseCommand + +import requests + +from ccdb.utils.data_import import setup_sqlite +from ccdb.projects.models import Project, Grant, GrantReport + + +class Command(BaseCommand): + help = 'Imports prior data into the DB' + + def add_arguments(self, parser): + parser.add_argument('manifest_url', type=str) + + def handle(self, **options): + _fetch_data(options['manifest_url'], self.stdout.write) + self.stdout.write('Fetched data') + _import_data() + self.stdout.write('Imported data') + + +def _fetch_data(url, write): + data_dir = 'data/' + r = requests.get(url) + files = r.json() + if not os.path.exists(data_dir): + os.makedirs(data_dir) + for f in files['files']: + p = ''.join([data_dir, f.split('/')[-1]]) + if not os.path.exists(p): + write('Grabbing {}'.format(p)) + r = requests.get(f, stream=True) + with open(p, 'wb') as out_file: + for chunk in r: + out_file.write(chunk) + + +def _import_data(): + c = setup_sqlite() + if c: + # Projects + for r in c.execute('SELECT * FROM tbl_lu_projects;'): + p = Project(id=r[0], name=r[1], code=r[2], iacuc_number=r[3], + description=r[4], sort_order=r[5]) + p.save() + + # Grants + for r in c.execute('SELECT * FROM tbl_lu_grants;'): + g = Grant(id=r[0], title=r[1], code=r[2], + description=r[3], sort_order=r[4]) + g.save() + + # Project-Grants + for r in c.execute('SELECT * FROM tbl_hash_project_grants;'): + p = Project.objects.get(id=r[0]) + g = Grant.objects.get(id=r[1]) + p.grants.add(g) + p.save() + + # Grant Reports + q = ''' + SELECT *, report_due_date AS "due_date [dtdt]" + FROM tbl_lu_grant_reports; + ''' + for r in c.execute(q): + g = Grant.objects.get(id=r[0]) + gr = GrantReport(grant=g, title=r[1], report_type=r[2], + description=r[3], due_date=r[8], submitted_date=r[5], + attachment=r[6], sort_order=r[7]) + gr.save() diff --git a/config/settings/base.py b/config/settings/base.py index 2cb1444..4572943 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -37,6 +37,7 @@ THIRD_PARTY_APPS = ( # Apps specific for this project go here. LOCAL_APPS = ( + 'ccdb.utils', 'ccdb.users', # custom users app 'ccdb.projects', ) diff --git a/requirements/base.txt b/requirements/base.txt index 599693f..850ebd8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -36,3 +36,6 @@ django-grappelli==2.7.3 # Date/time strings python-dateutil==2.4.2 + +# HTTP +requests==2.9.1