import os
from django.core.management.base import BaseCommand
from django.db import IntegrityError
import requests
from ccdb.utils.data_import import setup_sqlite
from ccdb.projects.models import Project, Grant, GrantReport
from ccdb.misc.models import MeasurementUnit, MeasurementType, Container, \
Material, Color
from ccdb.locations.models import Region, Site, MunicipalLocation, \
StudyLocation, StorageLocation
from ccdb.species.models import Species, CollectionSpecies
from ccdb.processing.models import ProcessType, Reagent, \
Flaw as ProcessingFlaw,Processing
from ccdb.collections_ccdb.models import CollectionType, CollectionMethod, \
Flaw as CollectionFlaw, ADFGPermit, Collection
from ccdb.experiments.models import Flaw as ExperimentFlaw, Experiment, \
ProtocolAttachment, TreatmentType, Treatment, TreatmentReplicate, \
AliveDeadCount
class Command(BaseCommand):
help = 'Imports prior data into the DB'
def add_arguments(self, parser):
parser.add_argument('manifest_url', type=str)
def handle(self, **options):
_fetch_data(options['manifest_url'], self.stdout.write)
self.stdout.write('Fetched data')
_import_admin_data()
self.stdout.write('Imported data')
def _fetch_data(url, write):
data_dir = 'data/'
r = requests.get(url)
files = r.json()
if not os.path.exists(data_dir):
os.makedirs(data_dir)
for f in files['files']:
p = ''.join([data_dir, f.split('/')[-1]])
if not os.path.exists(p):
write('Grabbing {}'.format(p))
r = requests.get(f, stream=True)
with open(p, 'wb') as out_file:
for chunk in r:
out_file.write(chunk)
def _import_admin_data():
c = setup_sqlite()
if c:
# Projects
for r in c.execute('SELECT * FROM tbl_lu_projects;'):
p = Project(id=r[0], name=r[1], code=r[2], iacuc_number=r[3],
description=r[4], sort_order=r[5])
p.save()
# Grants
for r in c.execute('SELECT * FROM tbl_lu_grants;'):
g = Grant(id=r[0], title=r[1], code=r[2],
description=r[3], sort_order=r[4])
g.save()
# Project-Grants
for r in c.execute('SELECT * FROM tbl_hash_project_grants;'):
p = Project.objects.get(id=r[0])
g = Grant.objects.get(id=r[1])
p.grants.add(g)
p.save()
# Grant Reports
q = '''
SELECT *, report_due_date AS "due_date [dtdt]"
FROM tbl_lu_grant_reports;
'''
for r in c.execute(q):
# No PK field in Andre's file
gr = GrantReport(grant_id=r[0], title=r[1], report_type=r[2],
description=r[3], due_date=r[8], submitted_date=r[5],
attachment=r[6], sort_order=r[7])
try:
gr.save()
except IntegrityError:
pass
# Measurement Units
for r in c.execute('SELECT * FROM tbl_lu_measurement_units;'):
mu = MeasurementUnit(id=r[0], name=r[1], code=r[2],
unit_class=r[3], description=r[4], sort_order=r[5])
mu.save()
# Measurement Types
for r in c.execute('SELECT * FROM tbl_lu_measurement_types;'):
mt = MeasurementType(id=r[0], name=r[1], code=r[2],
measurement_type_class=r[3], description=r[4],
default_measurement_unit_id=r[5], sort_order=r[6])
mt.save()
# Materials
for r in c.execute('SELECT * FROM tbl_lu_materials;'):
m = Material(id=r[0], name=r[1], code=r[2], material_class=r[3],
description=r[4], sort_order=r[5])
m.save()
# Colors
for r in c.execute('SELECT * FROM tbl_lu_colors;'):
cl = Color(id=r[0], name=r[1], code=r[2],
color_number=r[3], sort_order=r[4])
cl.save()
# Containers
for r in c.execute('SELECT * FROM tbl_lu_containers;'):
cl = Container(id=r[0], name=r[1], code=r[2], application=r[3],
color_id=r[4], material_id=r[5], volume=r[6],
measurement_unit_id=r[7], sort_order=r[8])
cl.save()
# Regions
for r in c.execute('SELECT * FROM tbl_lu_regions;'):
re = Region(id=r[0], name=r[1], code=r[2], sort_order=r[3])
re.save()
# Site
for r in c.execute('SELECT * FROM tbl_lu_sites;'):
s = Site(region_id=r[0], id=r[1], name=r[2], code=r[3],
description=r[4], sort_order=r[5])
s.save()
# Municipal Locations
for r in c.execute('SELECT * FROM tbl_lu_municipal_locations;'):
ml = MunicipalLocation(id=r[1], name=r[2], code=r[3],
municipal_location_type=r[4], description=r[5], sort_order=r[6])
ml.save()
# Study Locations
for r in c.execute('SELECT * FROM tbl_lu_study_locations;'):
sl = StudyLocation(site_id=r[0], id=r[1], name=r[2], code=r[3],
study_location_type=r[4], treatment_type=r[5],
municipal_location_id=r[6], collecting_location=r[7],
description=r[13], sort_order=r[14])
sl.save()
# Storage Location
for r in c.execute('SELECT * FROM tbl_lu_storage_locations;'):
bldg = "".join(e[0].upper() for e in r[2].split())
temp_c = '20'
if r[5]:
temp_c = r[5]
freezer = 'No Freezer'
if r[4]:
freezer = r[4]
code = " ".join([bldg, str(temp_c)+'C', str(freezer)])
sl = StorageLocation(id=r[0], facility=r[1], building=r[2],
room=r[3], freezer=r[4], temp_c=r[5], code=code,
description=r[6], sort_order=r[7])
sl.save()
# Species
for r in c.execute('SELECT * FROM tbl_lu_species;'):
s = Species(id=r[0], common_name=r[1], genus=r[2], species=r[3],
parasite=r[4], sort_order=r[5])
s.save()
# Processing Type
for r in c.execute('SELECT * FROM tbl_lu_process_types;'):
pt = ProcessType(id=r[0], name=r[1], code=r[2], description=r[3],
sort_order=r[4])
pt.save()
# Reagent
for r in c.execute('SELECT * FROM tbl_lu_reagents;'):
rg = Reagent(id=r[0], name=r[1], code=r[2], reagent_class=r[3],
sort_order=r[4])
rg.save()
# Collection Type
for r in c.execute('SELECT * FROM tbl_lu_collection_types;'):
ct = CollectionType(id=r[0], name=r[1], code=r[2], sort_order=r[3])
ct.save()
# Collection Method
for r in c.execute('SELECT * FROM tbl_lu_collection_methods;'):
cm = CollectionMethod(id=r[0], name=r[1], code=r[2],
collection_method_class=r[3], sort_order=r[4])
cm.save()
# Collection
for r in c.execute('''
SELECT *,
collection_start_date AS "collection_start_date [dtdt]",
collection_start_time AS "collection_start_time [dtdt]",
collection_end_date AS "collection_end_date [dtdt]",
collection_end_time AS "collection_end_time [dtdt]"
FROM tbl_collections;
'''):
if r[14] is not '':
permit, _ = ADFGPermit.objects.get_or_create(name=r[14])
else:
permit = None
col = Collection(project_id=r[0], id=r[1], study_location_id=r[2],
collection_type_id=r[3], collection_method_id=r[4],
number_of_traps=r[5], collection_start_date=r[17],
collection_start_time=r[18], collection_end_date=r[19],
collection_end_time=r[20], storage_location_id=r[10],
specimen_state=r[11], process_type_id=r[12], reagent_id=r[13],
adfg_permit=permit)
col.save()
# Collection Species
for r in c.execute('SELECT * FROM tbl_hash_collection_species;'):
# No PK field in Andre's file
cs = CollectionSpecies(collection_id=r[0], species_id=r[1],
sex=r[2], count=r[3], count_estimated=r[4])
try:
cs.save()
except IntegrityError:
pass
# Experiment
for r in c.execute('SELECT * FROM tbl_lu_experiments;'):
e = Experiment(id=r[0], name=r[1], code=r[2],
description=r[3], sort_order=r[6])
e.save()
# Treatment Type
for r in c.execute('SELECT * FROM tbl_lu_treatment_types;'):
tt = TreatmentType(experiment_id=r[0], id=r[1], name=r[2], code=r[3],
treatment_type=r[4], placement=r[5], description=r[6])
tt.save()
# Treatment
for r in c.execute('SELECT * FROM tbl_treatments;'):
t = Treatment(id=r[0], treatment_type_id=r[1], container_id=r[2],
study_location_id=r[3], species_id=r[4], sex=r[5])
t.save()
# Treatment Replicate
for r in c.execute('''
SELECT *, setup_date AS "setup_date [dtdt]"
FROM tbl_treatment_replicates tr
LEFT OUTER JOIN tbl_lu_record_flaws f ON f.flawid=tr.flawid;
'''):
if r[7]:
flaw, _ = ExperimentFlaw.objects.get_or_create(name=r[10])
else:
flaw = None
tr = TreatmentReplicate(treatment_id=r[0], id=r[1], name=r[2],
setup_date=r[13], setup_sample_size=r[5], mass_g=r[6], flaw=flaw)
tr.save()
# Alive-Dead Count
for r in c.execute('''
SELECT *,
status_date AS "status_date [dtdt]",
status_time AS "status_time [dtdt]"
FROM tbl_alive_dead_counts adc
LEFT OUTER JOIN tbl_lu_record_flaws f ON f.flawid=adc.flawid;
'''):
if r[6]:
flaw, _ = ExperimentFlaw.objects.get_or_create(name=r[9])
else:
flaw = None
adc = AliveDeadCount(treatment_replicate_id=r[0], id=r[1],
status_date=r[12], status_time=r[13], count_alive=r[4],
count_dead=r[5], flaw=flaw)
adc.save()
# Experiment-Collection
for r in c.execute('SELECT * FROM tbl_hash_collection_experiments;'):
c = Collection.objects.get(id=r[0])
e = Experiment.objects.get(id=r[1])
e.collections.add(c)
e.save()