From 8b95a83b1ba54d6958a1cab95633c0ef9e765c3f Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Wed, 15 Jun 2016 10:14:48 -0700 Subject: [PATCH] experiments: data import and tests --- ccdb/experiments/factories.py | 4 +- .../migrations/0009_DATA_initial.py | 113 ++++++++++++++++++ ccdb/experiments/models.py | 2 +- ccdb/experiments/tests/test_models.py | 87 ++++++++++++++ 4 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 ccdb/experiments/migrations/0009_DATA_initial.py diff --git a/ccdb/experiments/factories.py b/ccdb/experiments/factories.py index e9ade70..cdb8cee 100644 --- a/ccdb/experiments/factories.py +++ b/ccdb/experiments/factories.py @@ -77,7 +77,7 @@ class TreatmentReplicateFactory(DjangoModelFactory): class Meta: model = TreatmentReplicate - treatment_replicate = SubFactory(TreatmentFactory) + treatment = SubFactory(TreatmentFactory) name = Sequence(lambda n: 'treatment_replicate{}'.format(n)) setup_date = FuzzyDate(date(2012, 1, 1)) setup_time = LazyFunction(time) @@ -90,7 +90,7 @@ class AliveDeadCountFactory(DjangoModelFactory): class Meta: model = AliveDeadCount - treatment_replicate = SubFactory(TreatmentFactory) + treatment_replicate = SubFactory(TreatmentReplicateFactory) status_date = FuzzyDate(date(2012, 1, 1)) status_time = LazyFunction(time) count_alive = FuzzyInteger(0) diff --git a/ccdb/experiments/migrations/0009_DATA_initial.py b/ccdb/experiments/migrations/0009_DATA_initial.py new file mode 100644 index 0000000..8fc3ff8 --- /dev/null +++ b/ccdb/experiments/migrations/0009_DATA_initial.py @@ -0,0 +1,113 @@ +from django.db import migrations +from django.forms import modelform_factory + +from ccdb.utils.data import get_data_sources + + +class Migration(migrations.Migration): + def migrate(apps, schema_editor): + sources = get_data_sources() + if not sources: + return + + c = sources['db0'] + + Flaw = apps.get_model('experiments', 'Flaw') + Experiment = apps.get_model('experiments', 'Experiment') + TreatmentType = apps.get_model('experiments', 'TreatmentType') + Treatment = apps.get_model('experiments', 'Treatment') + TreatmentReplicate = apps.get_model('experiments', 'TreatmentReplicate') + AliveDeadCount = apps.get_model('experiments', 'AliveDeadCount') + + for model in [AliveDeadCount, TreatmentReplicate, Treatment, TreatmentType, + Experiment, Flaw]: + model.objects.all().delete() + + ExperimentForm = modelform_factory(Experiment, exclude=('collections',)) + TreatmentTypeForm = modelform_factory(TreatmentType, fields='__all__') + TreatmentForm = modelform_factory(Treatment, fields='__all__') + TreatmentReplicateForm = modelform_factory(TreatmentReplicate, fields='__all__') + AliveDeadCountForm = modelform_factory(AliveDeadCount, fields='__all__') + + for r in c.execute('SELECT * FROM tbl_lu_experiments;'): + form = ExperimentForm(dict(name=r[1], code=r[2], description=r[3], + sort_order=int(r[4]) if r[4] else None)) + if form.is_valid(): + Experiment.objects.create(id=r[0], **form.cleaned_data) + else: + print('experiment', r[0:], form.errors.as_data()) + + for r in c.execute('SELECT * FROM tbl_lu_treatment_types;'): + form = TreatmentTypeForm(dict(experiment=r[0], name=r[2], code=r[3], + treatment_type=r[4], placement=r[5], + description=r[6])) + if form.is_valid(): + TreatmentType.objects.create(id=r[1], **form.cleaned_data) + else: + print('treatment type', r[0:], form.errors.as_data()) + + for r in c.execute('SELECT * FROM tbl_treatments;'): + form = TreatmentForm(dict(treatment_type=r[1], container=r[2], + study_location=r[3], species=r[4], sex=r[5])) + if form.is_valid(): + Treatment.objects.create(id=r[0], **form.cleaned_data) + else: + print('treatment', r[0:], form.errors.as_data()) + + for r in c.execute(''' + SELECT *, setup_date AS "setup_date [dtdt]" + FROM tbl_treatment_replicates tr + LEFT OUTER JOIN tbl_lu_record_flaws f ON f.flawid=tr.flawid; + '''): + flaw = None + if r[7]: + flaw = Flaw.objects.create(name=r[10]).pk + form = TreatmentReplicateForm(dict(treatment=r[0], name=r[2], + setup_date=r[13], + setup_sample_size=r[5], mass_g=r[6], + flaw=flaw)) + if form.is_valid(): + TreatmentReplicate.objects.create(id=r[1], **form.cleaned_data) + else: + print('treatment replicate', r[0:], form.errors.as_data()) + + for r in c.execute(''' + SELECT *, + status_date AS "status_date [dtdt]", + status_time AS "status_time [dtdt]" + FROM tbl_alive_dead_counts adc + LEFT OUTER JOIN tbl_lu_record_flaws f ON f.flawid=adc.flawid; + '''): + flaw = None + if r[6]: + flaw = Flaw.objects.create(name=r[9]).pk + form = AliveDeadCountForm(dict(treatment_replicate=r[0], + status_date=r[12], + status_time=r[13].time() if r[13] else None, + count_alive=r[4], count_dead=r[5], + flaw=flaw)) + if form.is_valid(): + AliveDeadCount.objects.create(id=r[1], **form.cleaned_data) + else: + print('alive-dead count', r[0:], form.errors.as_data()) + + def rollback(apps, schema_editor): + Flaw = apps.get_model('experiments', 'Flaw') + Experiment = apps.get_model('experiments', 'Experiment') + TreatmentType = apps.get_model('experiments', 'TreatmentType') + Treatment = apps.get_model('experiments', 'Treatment') + TreatmentReplicate = apps.get_model('experiments', 'TreatmentReplicate') + AliveDeadCount = apps.get_model('experiments', 'AliveDeadCount') + + for model in [AliveDeadCount, TreatmentReplicate, Treatment, TreatmentType, + Experiment, Flaw]: + model.objects.all().delete() + + dependencies = [ + ('experiments', '0008_treatment_display_name'), + ('collections_ccdb', '0005_DATA_initial'), + ] + + operations = [ + migrations.RunPython(migrate, rollback), + ] diff --git a/ccdb/experiments/models.py b/ccdb/experiments/models.py index 546903e..07f4d98 100644 --- a/ccdb/experiments/models.py +++ b/ccdb/experiments/models.py @@ -91,7 +91,7 @@ class TreatmentReplicate(models.Model): def save(self, *args, **kwargs): self.display_name = "{}_{}_{}_{}".format(self.treatment, - self.setup_date.date(), self.name, + self.setup_date, self.name, self.setup_sample_size) super(TreatmentReplicate, self).save(*args, **kwargs) diff --git a/ccdb/experiments/tests/test_models.py b/ccdb/experiments/tests/test_models.py index e69de29..64c1cb2 100644 --- a/ccdb/experiments/tests/test_models.py +++ b/ccdb/experiments/tests/test_models.py @@ -0,0 +1,87 @@ +from django.test import TestCase +from django.db import IntegrityError, transaction + +from ..models import Flaw, Experiment, ProtocolAttachment, TreatmentType, \ + Treatment, TreatmentReplicate, AliveDeadCount +from ..factories import FlawFactory, ExperimentFactory, ProtocolAttachmentFactory, \ + TreatmentTypeFactory, TreatmentFactory, TreatmentReplicateFactory, \ + AliveDeadCountFactory + + +class FlawTestCase(TestCase): + def test_creation(self): + f = FlawFactory() + self.assertTrue(isinstance(f, Flaw)) + self.assertEqual(f.__str__(), f.name) + + +class ExperimentTestCase(TestCase): + def test_creation(self): + e = ExperimentFactory() + self.assertTrue(isinstance(e, Experiment)) + self.assertEqual(e.__str__(), e.name) + + def test_uniqueness(self): + e1 = ExperimentFactory() + with transaction.atomic(), self.assertRaises(IntegrityError): + ExperimentFactory(name=e1.name, code=e1.code) + e3 = ExperimentFactory() + self.assertTrue(isinstance(e3, Experiment)) + + +class ProtocolAttachmentTestCase(TestCase): + def test_creation(self): + p = ProtocolAttachmentFactory() + self.assertTrue(isinstance(p, ProtocolAttachment)) + self.assertEqual(p.__str__(), p.protocol) + + +class TreatmentTypeTestCase(TestCase): + def test_creation(self): + t = TreatmentTypeFactory() + self.assertTrue(isinstance(t, TreatmentType)) + label = "{} {} {} {}".format(t.experiment, t.name, t.treatment_type, + t.placement) + self.assertEqual(t.__str__(), label) + + def test_uniqueness(self): + t1 = TreatmentTypeFactory() + with transaction.atomic(), self.assertRaises(IntegrityError): + TreatmentTypeFactory(name=t1.name, experiment=t1.experiment) + t3 = TreatmentTypeFactory() + self.assertTrue(isinstance(t3, TreatmentType)) + + +class TreatmentTestCase(TestCase): + def test_creation(self): + t = TreatmentFactory() + self.assertTrue(isinstance(t, Treatment)) + label = "{}_{}_{}_{}".format(t.treatment_type, t.study_location, + t.species, t.sex) + self.assertEqual(t.__str__(), label) + + +class TreatmentReplicateTestCase(TestCase): + def test_creation(self): + t = TreatmentReplicateFactory() + self.assertTrue(isinstance(t, TreatmentReplicate)) + label = "{}_{}_{}_{}".format(t.treatment, t.setup_date, + t.name, t.setup_sample_size) + self.assertEqual(t.__str__(), label) + + def test_uniqueness(self): + t1 = TreatmentReplicateFactory() + with transaction.atomic(), self.assertRaises(IntegrityError): + TreatmentReplicateFactory(treatment=t1.treatment, name=t1.name, + setup_date=t1.setup_date, + setup_time=t1.setup_time) + t3 = TreatmentReplicateFactory() + self.assertTrue(isinstance(t3, TreatmentReplicate)) + + +class AliveDeadCountTestCase(TestCase): + def test_creation(self): + a = AliveDeadCountFactory() + self.assertTrue(isinstance(a, AliveDeadCount)) + label = "{}".format(a.status_date) + self.assertEqual(a.__str__(), label)