diff --git a/.gitignore b/.gitignore index 099b67a..61c89c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ *.py[cod] -.htaccess -passenger_wsgi.py +venv diff --git a/LICENSE b/LICENSE index 353b631..916ef2c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,7 @@ 2015 Matthew R. Dillon -This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. +This work is licensed under the Creative Commons +Attribution-NonCommercial-ShareAlike 3.0 Unported License. To view a copy of +this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ or send a +letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, +California, 94041, USA. diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..b15858e --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +web: gunicorn -w 4 manage:app --log-file=- +init: python manage.py initdb diff --git a/README.md b/README.md index 95f4d65..eeaac57 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,25 @@ AKIndices --- Alaska climate data ================================= -Air temperature data from over 400 communities, reduced to relevant engineering parameters. +Air temperature data from over 400 communities, reduced to relevant engineering +parameters. What is it? ----------- -[AKIndices](http://akindices.akdillon.net) is a Flask-driven web-app that builds on -the AKExtract project to provide an easy-to-use interface for working with SNAP datasets. +[AKIndices](http://akindices.akdillon.net) is a Flask-driven web-app that +builds on the AKExtract project to provide an easy-to-use interface for working +with SNAP datasets. Prerequisites ------------- - [AKExtract](http://github.com/thermokarst/akextract) - Flask (0.10.1) -- SQLAlchemy (0.8.2) -- psycopg2 (2.5.1) -- flask-wtf (0.9.1) -- numpy (1.7.1) -- PostgreSQL - - -Installation ------------- - -1) Clone the repo: - - git clone https://github.com/thermokarst/akindices - -2) Get the data from http://snap.uaf.edu - -3) Copy `config.py.default` to `config.py`, edit the parameters to suit your needs. - -4) Launch a python interpreter and populate the database with data: - - $ python - >>> import akindices - >>> akindices.database.init_db() - -5) Launch the application with: - - $ ./run.py +- SQLAlchemy (1.0.5) +- psycopg2 (2.6.1) +- flask-wtf (0.12) +- PostgreSQL (9.4+) Contact diff --git a/akindices/__init__.py b/akindices/__init__.py deleted file mode 100644 index ac0846f..0000000 --- a/akindices/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- - -from flask import Flask -import logging -from logging.handlers import RotatingFileHandler - -application = Flask(__name__) -application.config.from_pyfile('../config.py') - -format = '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]' - -if not application.debug: - file_handler = RotatingFileHandler(application.config['LOG'], - maxBytes=application.config['MAXLOG'], - backupCount=application.config['BACKUPCOUNT']) - application.logger.setLevel(logging.INFO) - file_handler.setFormatter(logging.Formatter(format)) - application.logger.addHandler(file_handler) - -from akindices import views diff --git a/akindices/database.py b/akindices/database.py deleted file mode 100644 index 9f304cd..0000000 --- a/akindices/database.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- coding: utf-8 -*- - -from sqlalchemy import create_engine -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy.ext.declarative import declarative_base -from akindices import application -import os - -engine = create_engine(application.config['ENGINE'], - convert_unicode=True) -db_session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=engine)) -Base = declarative_base() -Base.query = db_session.query_property() - - -def init_db(): - tempsnap = application.config['SNAPDATA'] - - import akextract - import numpy - import models - import datetime - - datafiles = [ f for f in os.listdir(tempsnap) if os.path.isfile(os.path.join(tempsnap,f)) ] - for f in datafiles: - print f - - datafiles.remove('.DS_Store') - community_file =application.config['COMMUNITIES'] - dt = numpy.dtype({'names':['community', 'northing', 'easting'], - 'formats':['S100', 'f8', 'f8']}) - communities, eastings, northings = numpy.loadtxt(community_file, - skiprows=1, delimiter=',', - unpack=True, dtype=dt) - communities = communities.tolist() - - modelnames = { - '5modelAvg': '5 Model Avg.', - 'cccma_cgcm3_1': 'CCCMA CGCM 3.1', - 'CRU': 'CRU', - 'gfdl_cm2_1': 'GFDL CM 2.1', - 'miroc3_2_medres': 'MIROC 3.2, medres', - 'mpi_echam5': 'MPI ECHAM 5', - 'ukmo_hadcm3': 'UKMO HADCM 3.1' - } - - datasets = [] - - Base.metadata.drop_all(bind=engine) - Base.metadata.create_all(bind=engine) - - for filename in datafiles: - dataset = akextract.GeoRefData(os.path.join(tempsnap, filename)) - tokens = filename.split('_') - startyr = int(tokens[-2]) - endyr = int(tokens[-1].split('.')[0]) - if endyr == 2100: - endyr = 2099 - starttime = datetime.datetime.now() - print datetime.datetime.now().strftime('%m-%d-%Y %I:%M%p') - print filename, startyr, endyr, dataset.model, dataset.scenario, dataset.resolution - - extracted_temps = dataset.extract_points(northings, eastings, - startyr, endyr) - print "Loading data into database..." - - if dataset.model == 'CRU': - datasetType = 'HISTORICAL' - else: - datasetType = 'PROJECTION' - - dataset_sql = models.Dataset.query.filter(models.Dataset.model == dataset.model, - models.Dataset.scenario == dataset.scenario).first() - - if dataset_sql is None: - print "not in dataset table...", (dataset.model, dataset.scenario) - dataset_sql = models.Dataset(datasetType, - dataset.model, - modelnames[dataset.model], - dataset.scenario, - dataset.resolution) - db_session.add(dataset_sql) - datasets.append((dataset.model, dataset.scenario)) - db_session.commit() - - min_year = numpy.min(extracted_temps['year']) - max_year = numpy.max(extracted_temps['year']) - - time_years = max_year - min_year + 1 - i = 0 - for community in communities: - longitude, latitude, elev = akextract.ne_to_wgs(northings[i], eastings[i]) - location = models.Community.as_unique(db_session, name=community, - northing=northings[i], - easting=eastings[i], - latitude=latitude, - longitude=longitude) - db_session.add(location) - - # Load up the temperature data - db_data = numpy.zeros((time_years, 13)) - db_data[:, 0] = numpy.arange(min_year, max_year+1) - db_data[:, 1:] = extracted_temps[i, :]['temperature'].reshape(time_years, 12) - - for row in db_data: - data = [models.Temperature(year=row[0], - january=row[1], february=row[2], - march=row[3], april=row[4], - may=row[5], june=row[6], - july=row[7], august=row[8], - september=row[9], october=row[10], - november=row[11], december=row[12], - updated=starttime)] - - dataset_sql.temperatures.extend(data) - location.temperatures.extend(data) - - db_session.commit() - i += 1 - - db_session.close() diff --git a/akindices/forms.py b/akindices/forms.py deleted file mode 100644 index d63787a..0000000 --- a/akindices/forms.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -from flask_wtf import Form -from wtforms import validators, ValidationError, IntegerField -from wtforms.ext.sqlalchemy.fields import QuerySelectField -from akindices.models import Community, Dataset, Temperature -from sqlalchemy import func - -class AKIYearField(IntegerField): - def pre_validate(self, form): - if form.model.data is not None: - ymin,ymax = Temperature.query \ - .with_entities(func.min(Temperature.year), - func.max(Temperature.year)) \ - .filter(Temperature.dataset_id == form.model.data.id) \ - .all()[0] - self.validators = [validators.NumberRange(min=ymin, max=ymax), - validators.Required()] - - -def communities(): - return Community.query.order_by('name') - -def datasets(): - return Dataset.query.order_by('datatype', 'model', 'scenario') - -def dataset_names(ds): - return "{type} ({resolution}) - {modelname} {scenario}".format(modelname=ds.modelname, - scenario=ds.scenario, - type=ds.datatype.lower()\ - .capitalize(), - resolution=ds.resolution) - -class AKIForm(Form): - community = QuerySelectField(query_factory=communities, - get_label='name', - allow_blank=True, - blank_text=u'---Select a community---', - validators=[validators.Required(message='Please select a community')]) - - minyear = AKIYearField(u'minyear') - - maxyear = AKIYearField(u'maxyear') - - model = QuerySelectField(query_factory=datasets, - get_label=dataset_names, - allow_blank=True, - blank_text=u'---Select a dataset---', - validators=[validators.Required(message='Please select a dataset')]) diff --git a/akindices/models.py b/akindices/models.py deleted file mode 100644 index 69ea58a..0000000 --- a/akindices/models.py +++ /dev/null @@ -1,195 +0,0 @@ -# -*- coding: utf-8 -*- - -from sqlalchemy import Column, ForeignKey, Integer, Float, String, DateTime, create_engine -from sqlalchemy.orm import relationship, backref -from sqlalchemy.schema import Index -from akindices.database import Base - - -class UniqueMixin(object): - """ - Usage recipe from: - http://www.sqlalchemy.org/trac/wiki/UsageRecipes/UniqueObject - """ - @classmethod - def unique_hash(cls, *arg, **kw): - """ - Unique hash stub - """ - raise NotImplementedError() - - @classmethod - def unique_filter(cls, query, *args, **kw): - """ - Unique filter stub - """ - raise NotImplementedError() - - @classmethod - def as_unique(cls, session, *arg, **kw): - """ - as_unique - """ - return _unique(session, cls, cls.unique_hash, cls.unique_filter, - cls, arg, kw) - -class Community(UniqueMixin, Base): - """ - Defines the data model for a Community - - :returns Community data - """ - - __tablename__ = 'communities' - - id = Column(Integer, primary_key=True) - name = Column(String(50), nullable=False, unique=True) - northing = Column(Float, nullable=False) - easting = Column(Float, nullable=False) - latitude = Column(Float, nullable=False) - longitude = Column(Float, nullable=False) - - temperatures = relationship("Temperature", backref='communities') - - @classmethod - def unique_hash(cls, name, northing, easting, latitude, longitude): - return name - - @classmethod - def unique_filter(cls, query, name, northing, easting, latitude, longitude): - return query.filter(Community.name == name, - Community.northing == northing, - Community.easting == easting, - Community.latitude == latitude, - Community.longitude == longitude) - - def __init__(self, name, northing, easting, latitude, longitude): - self.name = name - self.northing = northing - self.easting = easting - self.latitude = latitude - self.longitude = longitude - - def __repr__(self): - return "Community{data}".format(data=(self.name, self.northing, - self.easting, self.latitude, - self.longitude)) - - -class Dataset(Base): - """ - Defines the data model for a Dataset - - :returns Dataset data - """ - - __tablename__ = 'datasets' - - id = Column(Integer, primary_key=True) - datatype = Column(String(15), nullable=False) - model = Column(String(15), nullable=False) - modelname = Column(String(50), nullable=True) - scenario = Column(String(15), nullable=False) - resolution = Column(String(15), nullable=False) - - temperatures = relationship("Temperature", backref='datasets') - - def __init__(self, datatype, model, modelname, scenario, resolution): - self.datatype = datatype - self.model = model - self.modelname = modelname - self.scenario = scenario - self.resolution = resolution - - def __repr__(self): - return "Dataset{data}".format(data=(self.datatype, self.model, - self.modelname, self.scenario, - self.resolution)) - - -class Temperature(Base): - """ - Defines the data model for a Temperature - - :returns Temperature data - """ - - __tablename__ = 'temperatures' - - id = Column(Integer, primary_key=True) - dataset_id = Column(Integer, ForeignKey('datasets.id')) - community_id = Column(Integer, ForeignKey('communities.id')) - year = Column(Integer, nullable=False) - january = Column(Float, nullable=False) - february = Column(Float, nullable=False) - march = Column(Float, nullable=False) - april = Column(Float, nullable=False) - may = Column(Float, nullable=False) - june = Column(Float, nullable=False) - july = Column(Float, nullable=False) - august = Column(Float, nullable=False) - september = Column(Float, nullable=False) - october = Column(Float, nullable=False) - november = Column(Float, nullable=False) - december = Column(Float, nullable=False) - updated = Column(DateTime, nullable=True) - - dataset = relationship("Dataset", primaryjoin=dataset_id == Dataset.id) - - def __init__(self, year, january, february, march, april, may, june, - july, august, september, october, november, december, updated): - self.year = year - self.january = january - self.february = february - self.march = march - self.april = april - self.may = may - self.june = june - self.july = july - self.august = august - self.september = september - self.october = october - self.november = november - self.december = december - self.updated = updated - - def __repr__(self): - return "Temperature{data}".format(data=(self.year, - self.january, - self.february, - self.march, - self.april, - self.may, - self.june, - self.july, - self.august, - self.september, - self.october, - self.november, - self.december, - self.updated)) - - __table_args__ = (Index('idx_temps', 'dataset_id', 'community_id', 'year', unique=True),) - - -def _unique(session, cls, hashfunc, queryfunc, constructor, arg, kw): - """ - Function to checks for an existing instances - """ - cache = getattr(session, '_unique_cache', None) - if cache is None: - session._unique_cache = cache = {} - - key = (cls, hashfunc(*arg, **kw)) - if key in cache: - return cache[key] - else: - with session.no_autoflush: - q = session.query(cls) - q = queryfunc(q, *arg, **kw) - obj = q.first() - if not obj: - obj = constructor(*arg, **kw) - session.add(obj) - cache[key] = obj - return obj diff --git a/akindices/processing.py b/akindices/processing.py deleted file mode 100644 index 5124635..0000000 --- a/akindices/processing.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -import numpy -import os - -def avg_air_temp(temps): - return numpy.average(temps) - -def ann_air_indices(temps): - ATI, AFI = 0.0, 0.0 - indices = numpy.zeros((temps.shape[0], 2), dtype='int') - months = [0.0 for m in range(12)] - days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - i = 0 - for year in temps: - j = 0 - for month in months: - months[j] = days[j] * year[j] - j += 1 - - for ind in months: - if ind >= 0.0: - ATI = ATI + ind - else: - AFI = AFI + ind - indices[i, 0], indices[i, 1] = int(ATI), int(AFI) - ATI, AFI = 0.0, 0.0 - i += 1 - return indices - -def avg_air_indices(indices): - temp = numpy.average(indices, axis=0) - return (int(temp[0]), int(temp[1])) - -def des_air_indices(indices): - if indices.shape[0] > 2: - ati = numpy.sort(indices[:,0]) - afi = numpy.sort(indices[:,1]) - dti = (ati[-1] + ati[-2] + ati[-3]) / 3.0 - dfi = (afi[0] + afi[1] + afi[2]) / 3.0 - return (int(dti), int(dfi)) - else: - return (None, None) - -def c_to_f(temp): - return (temp * 9. / 5.) + 32. - -if __name__ == '__main__': - print("nothing to see here") diff --git a/akindices/templates/404.html b/akindices/templates/404.html deleted file mode 100644 index 975a908..0000000 --- a/akindices/templates/404.html +++ /dev/null @@ -1,6 +0,0 @@ -{% extends "base.html" %} -{% block content %} -

404 Page Not Found

-

What you were looking for is just not there.

-

Main

-{% endblock %} \ No newline at end of file diff --git a/akindices/templates/500.html b/akindices/templates/500.html deleted file mode 100644 index c033336..0000000 --- a/akindices/templates/500.html +++ /dev/null @@ -1,6 +0,0 @@ -{% extends "base.html" %} -{% block content %} -

500 Internal Server Error

-

Something didn't work

-

Main

-{% endblock %} \ No newline at end of file diff --git a/akindices/templates/details.html b/akindices/templates/details.html deleted file mode 100644 index 1fb26b9..0000000 --- a/akindices/templates/details.html +++ /dev/null @@ -1,68 +0,0 @@ -{% extends "base.html" %} -{% block content %} - - - - -

{{ community_name }}

- -
-
- -

Monthly Temperatures

-
- - - - - - - - - - - - - - - - - - - - {% for temp in temps %} - - - - - - - - - - - - - - - - {% endfor %} - - -
Year
 
January
°C
February
°C
March
°C
April
°C
May
°C
June
°C
July
°C
August
°C
September
°C
October
°C
November
°C
December
°C
{{ temp[0]|int }}{{ temp[1]|round(2) }}{{ temp[2]|round(2) }}{{ temp[3]|round(2) }}{{ temp[4]|round(2) }}{{ temp[5]|round(2) }}{{ temp[6]|round(2) }}{{ temp[7]|round(2) }}{{ temp[8]|round(2) }}{{ temp[9]|round(2) }}{{ temp[10]|round(2) }}{{ temp[11]|round(2) }}{{ temp[12]|round(2) }}
- - - - -{% endblock %} diff --git a/akindices/views.py b/akindices/views.py deleted file mode 100644 index e141f1f..0000000 --- a/akindices/views.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- - -from flask import render_template, jsonify, request, flash, redirect, url_for, session, current_app -from akindices import application -from akindices.database import db_session -from akindices.models import Community, Temperature, Dataset -from forms import AKIForm -from numpy import zeros, arange, hstack -from processing import ann_air_indices, avg_air_indices, des_air_indices, avg_air_temp, c_to_f - -@application.route('/', methods = ['GET', 'POST']) -def index(): - form = AKIForm() - - # Deal with form posting here - if request.method == 'POST': - if form.validate(): - session['community'] = request.form['community'] - session['minyear'] = request.form['minyear'] - session['maxyear'] = request.form['maxyear'] - if session['minyear'] > session['maxyear']: - session['maxyear'] = session['minyear'] - - session['datasets'] = request.form['model'] - return redirect('/') - else: - return render_template("index.html", - title = application.config['TITLE'], - form = form, - ) - - # Deal with page gets here - if request.method == 'GET': - session['community_data'] = None - modelstash = None # Need this - session['avg_temp'] = None - session['avg_indices'] = None - session['des_indices'] = None - - if 'community' in session: - community_id = session['community'] - - if all (key in session for key in ('minyear', 'maxyear', 'datasets')): - session['community_data'] = dict() - session['community_data']['id'] = community_id - session['community_data']['name'] = db_session.query(Community).get(community_id).name - session['community_data']['latitude'] = round(db_session.query(Community).get(community_id).latitude, 5) - session['community_data']['longitude'] = round(db_session.query(Community).get(community_id).longitude, 5) - - session['ds_name'] = db_session.query(Dataset.modelname, Dataset.scenario). \ - filter(Dataset.id == session['datasets']).all() - temps_arr = getTemps(session['datasets'], community_id, session['minyear'], session['maxyear']) - - session['avg_temp'] = avg_air_temp(temps_arr) - indices = ann_air_indices(temps_arr) - session['avg_indices'] = avg_air_indices(indices) - session['des_indices'] = des_air_indices(indices) - - return render_template("index.html", - title = application.config['TITLE'], - form = form - ) - -@application.route('/reset') -@application.route('/clear') -def reset(): - session.clear() - return redirect('/') - -@application.route('/save') -def save(): - if 'save' in session: - i = len(session['save']) - save = session['save'] - else: - save = dict() - i = 0 - - save[i] = dict() - save[i]['datasets'] = session['datasets'] - save[i]['ds_name'] = session['ds_name'] - save[i]['community_data'] = session['community_data'] - save[i]['minyear'] = session['minyear'] - save[i]['maxyear'] = session['maxyear'] - save[i]['avg_temp'] = session['avg_temp'] - save[i]['avg_indices'] = session['avg_indices'] - save[i]['des_indices'] = session['des_indices'] - session.clear() - session['save'] = save - return redirect('/') - -@application.route('/delete') -def delete(): - record = request.args.get('record', '') - session['save'].pop(record) - return redirect('/') - -@application.route('/datatypes') -def datatypes(): - return render_template("datatypes.html", - title=application.config['TITLE']) - -@application.route('/details') -def details(): - datasets = request.args.get('datasets', '') - community_id = request.args.get('community_id', '') - minyear = request.args.get('minyear', '') - maxyear = request.args.get('maxyear', '') - temps = getTemps(datasets, community_id, minyear, maxyear) - years = arange(int(minyear), int(maxyear)+1).reshape(int(maxyear)-int(minyear) + 1, 1) - temps = hstack((years, temps)) - return render_template("details.html", - lat=request.args.get('lat', ''), - lon=request.args.get('lon', ''), - community_name=request.args.get('name', ''), - temps=temps, - title=application.config['TITLE']) - -@application.teardown_appcontext -def shutdown_session(exception=None): - db_session.remove() - -def getTemps(datasets, community_id, minyear, maxyear): - # Get the temps - temps = db_session.query(Temperature, Dataset). \ - filter(Dataset.id == Temperature.dataset_id, - Dataset.id == datasets, - Temperature.community_id == community_id, - Temperature.year >= minyear, - Temperature.year <= maxyear) - - length = int(maxyear) - int(minyear) - temps_arr = zeros((length+1, 12)) - - i = 0 - for temp in temps.all(): - t = temp[0] - temps_arr[i,:] = [t.january, t.february, t.march, - t.april, t.may, t.june, - t.july, t.august, t.september, - t.october, t.november, t.december] - i += 1 - return temps_arr - -@application.before_request -def log_request(): - current_app.logger.info('Request\nIP: {addr}\nSESSION: {session}' \ - .format(addr=request.remote_addr, - session=session)) - -@application.errorhandler(404) -def page_not_found(e): - return render_template('404.html', - title=application.config['TITLE']), 404 - -@application.errorhandler(500) -def internal_server_error(e): - return render_template('500.html', - title=application.config['TITLE']), 500 diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..ec2f2ee --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,24 @@ +from flask import Flask +from flask.ext.sqlalchemy import SQLAlchemy + +from config import config + + +db = SQLAlchemy() + + +def create_app(config_name): + app = Flask(__name__) + app.config.from_object(config[config_name]) + config[config_name].init_app(app) + + db.init_app(app) + + import logging + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + from .main import main as main_blueprint + app.register_blueprint(main_blueprint) + + return app diff --git a/app/main/__init__.py b/app/main/__init__.py new file mode 100644 index 0000000..50a1a99 --- /dev/null +++ b/app/main/__init__.py @@ -0,0 +1,7 @@ +from flask import Blueprint + + +main = Blueprint('main', __name__, template_folder='templates') + + +from . import views diff --git a/app/main/forms.py b/app/main/forms.py new file mode 100644 index 0000000..b5c3fde --- /dev/null +++ b/app/main/forms.py @@ -0,0 +1,19 @@ +from flask_wtf import Form +from wtforms import IntegerField, SelectField +from wtforms.validators import NumberRange, Required + + +class AKIYearField(IntegerField): + def pre_validate(self, form): + if form.data['dataset'] == 'CRU,TS31': + self.validators = [NumberRange(min=1901, max=2009), Required()] + else: + self.validators = [NumberRange(min=2001, max=2099), Required()] + + +class AKIForm(Form): + community = SelectField(coerce=int, + validators=[Required(message='Please select a community')]) + dataset = SelectField(validators=[Required(message='Please select a dataset')]) + minyear = AKIYearField('minyear') + maxyear = AKIYearField('maxyear') diff --git a/app/main/models.py b/app/main/models.py new file mode 100644 index 0000000..c95f7c6 --- /dev/null +++ b/app/main/models.py @@ -0,0 +1,67 @@ +from app import db +from sqlalchemy.sql import text +from flask import abort + + +class DB: + @classmethod + def getCommunity(cls, id): + cmd = """ + SELECT id, name, latitude, longitude, northing, easting + FROM new_communities + WHERE id=:id; + """ + result = db.engine.execute(text(cmd), id=id).fetchone() + return result or abort(500) + + @classmethod + def getCommunities(cls): + cmd = """ + SELECT id, name + FROM new_communities + ORDER BY name ASC; + """ + result = db.engine.execute(text(cmd), id=id).fetchall() + return result or abort(500) + + @classmethod + def getDatasets(cls): + cmd = """ + SELECT DISTINCT ON ( + doc->'datatype', + doc->'resolution', + doc->'modelname', + doc->'scenario' + ) + doc->'datatype' AS datatype, + doc->'resolution' AS resolution, + doc->'modelname' AS modelname, + doc->'scenario' AS scenario + FROM new_communities c, + jsonb_array_elements(c.data) + WITH ORDINALITY t1(doc, rn) + ORDER BY datatype ASC, modelname ASC, scenario ASC; + """ + result = db.engine.execute(text(cmd)).fetchall() + return result or abort(500) + + @classmethod + def getTemps(cls, start, end, community_id, modelname, scenario): + years = [str(x) for x in range(int(start), int(end)+1)] + cmd = """ + WITH x AS ( + SELECT name, jsonb_array_elements(data) AS data + FROM new_communities + WHERE id=:community_id) + SELECT d.key AS year, d.value AS temperatures + FROM x, jsonb_each(data) d + WHERE data->>'modelname'=:modelname + AND data->>'scenario'=:scenario + AND d.key IN :years; + """ + result = db.engine.execute(text(cmd), + community_id=community_id, + modelname=modelname, + scenario=scenario, + years=tuple(years)).fetchall() + return result or abort(500) diff --git a/akindices/templates/_formhelpers.html b/app/main/templates/main/_formhelpers.html similarity index 94% rename from akindices/templates/_formhelpers.html rename to app/main/templates/main/_formhelpers.html index e247f10..bdda279 100644 --- a/akindices/templates/_formhelpers.html +++ b/app/main/templates/main/_formhelpers.html @@ -9,4 +9,4 @@
{% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/akindices/templates/base.html b/app/main/templates/main/base.html similarity index 56% rename from akindices/templates/base.html rename to app/main/templates/main/base.html index d08f1c8..44b046b 100644 --- a/akindices/templates/base.html +++ b/app/main/templates/main/base.html @@ -1,7 +1,7 @@ - {{title}} + {{ config['TITLE'] }} @@ -14,16 +14,16 @@
{% block content %}{% endblock %}
diff --git a/akindices/templates/datatypes.html b/app/main/templates/main/datatypes.html similarity index 98% rename from akindices/templates/datatypes.html rename to app/main/templates/main/datatypes.html index f743346..84dd617 100644 --- a/akindices/templates/datatypes.html +++ b/app/main/templates/main/datatypes.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "main/base.html" %} {% block content %}

Model Details

diff --git a/app/main/templates/main/details.html b/app/main/templates/main/details.html new file mode 100644 index 0000000..b62a71f --- /dev/null +++ b/app/main/templates/main/details.html @@ -0,0 +1,64 @@ +{% extends "main/base.html" %} +{% block content %} + + + + +

{{ community_name }}

+ +
+
+ +

Monthly Temperatures

+
+ + + + + + + + + + + + + + + + + + + + {% for temp in temps %} + + + + + + + + + + + + + + + + {% endfor %} + +
Year
 
January
°C
February
°C
March
°C
April
°C
May
°C
June
°C
July
°C
August
°C
September
°C
October
°C
November
°C
December
°C
{{ temp[0] }}{{ temp[1][0] }}{{ temp[1][1] }}{{ temp[1][2] }}{{ temp[1][3] }}{{ temp[1][4] }}{{ temp[1][5] }}{{ temp[1][6] }}{{ temp[1][7] }}{{ temp[1][8] }}{{ temp[1][9] }}{{ temp[1][10] }}{{ temp[1][11] }}
+ + + +{% endblock %} diff --git a/akindices/templates/index.html b/app/main/templates/main/index.html similarity index 92% rename from akindices/templates/index.html rename to app/main/templates/main/index.html index c10cbd4..5dc3e8b 100644 --- a/akindices/templates/index.html +++ b/app/main/templates/main/index.html @@ -1,5 +1,5 @@ -{% extends "base.html" %} -{% from "_formhelpers.html" import render_field %} +{% extends "main/base.html" %} +{% from "main/_formhelpers.html" import render_field %} {% block content %}

@@ -19,9 +19,9 @@

- {{ render_field(form.model, class='form-control', id='modelinput') }} + {{ render_field(form.dataset, class='form-control', id='modelinput') }} Historical (1901-2009) or Projection (2001-2099)
- + Learn more about the models and scenarios
@@ -46,11 +46,11 @@
+ value="Get Temperatures" />
@@ -81,7 +81,7 @@ {% if session['avg_temp'] %} - {{ session['avg_indices'][1] }} {{ session['des_indices'][0] }} {{ session['des_indices'][1] }} - {{ session['ds_name'][0][0] }} ({{ session['ds_name'][0][1] }}) + {{ session['ds_name'][0] }} ({{ session['ds_name'][1] }}) @@ -112,11 +112,6 @@ {% endif %} - {# - -   - - #} Saved Searches @@ -125,7 +120,7 @@ {% if session.save %} {% for key, value in session.save|dictsort %} - {{ value['avg_indices'][1] }} {{ value['des_indices'][0] }} {{ value['des_indices'][1] }} - {{ value['ds_name'][0][0] }} ({{ value['ds_name'][0][1] }}) + {{ value['ds_name'][0] }} ({{ value['ds_name'][1] }}) @@ -227,4 +222,4 @@ or submit a bug report! -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/app/main/utils.py b/app/main/utils.py new file mode 100644 index 0000000..6dfd528 --- /dev/null +++ b/app/main/utils.py @@ -0,0 +1,73 @@ +from .models import DB + + +def getTemps(session): + modelname, scenario = session['datasets'].split(',') + data = DB.getTemps(session['minyear'], + session['maxyear'], + session['community_data']['id'], + modelname, + scenario) + return data + + +def avg_air_temp(temps): + year_counter, total = 0, 0 + for temp in temps: + total += sum(temp[1]) + year_counter += 1 + return total / (year_counter * 12) + + +def ann_air_indices(temps): + ATI, AFI = 0.0, 0.0 + indices = [[0 for x in range(2)] for y in range(len(temps))] + months = [0.0 for m in range(12)] + days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + i = 0 + for year in temps: + j = 0 + for month in months: + months[j] = days[j] * year[1][j] + j += 1 + + for ind in months: + if ind >= 0.0: + ATI = ATI + ind + else: + AFI = AFI + ind + indices[i][0], indices[i][1] = int(ATI), int(AFI) + ATI, AFI = 0.0, 0.0 + i += 1 + return indices + + +def avg_air_indices(indices): + year_counter, total_freezing, total_thawing = 0, 0, 0 + for index in indices: + total_thawing += index[0] + total_freezing += index[1] + year_counter += 1 + return (int(total_thawing / year_counter), int(total_freezing / year_counter)) + + +def des_air_indices(indices): + if len(indices) > 2: + ati = sorted(indices, key=lambda arr: arr[0]) + afi = sorted(indices, key=lambda arr: arr[1]) + dti = (ati[-1][0] + ati[-2][0] + ati[-3][0]) / 3.0 + dfi = (afi[0][1] + afi[1][1] + afi[2][1]) / 3.0 + return (int(dti), int(dfi)) + else: + return (None, None) + + +def communitiesSelect(): + return [(c.id, c.name) for c in DB.getCommunities()] + + +def datasetsSelect(): + return [("{0.modelname},{0.scenario}".format(d), + "{x} ({d.resolution}) - {d.modelname} {d.scenario}".format(d=d, + x=d.datatype.title())) + for d in DB.getDatasets()] diff --git a/app/main/views.py b/app/main/views.py new file mode 100644 index 0000000..7903932 --- /dev/null +++ b/app/main/views.py @@ -0,0 +1,109 @@ +from flask import session, render_template, request, redirect, url_for + +from . import main +from .forms import AKIForm +from .utils import getTemps, avg_air_temp, ann_air_indices, \ + avg_air_indices, des_air_indices, communitiesSelect, datasetsSelect +from .models import DB + + +@main.route('/', methods=['GET', 'POST']) +def index(): + form = AKIForm() + form.community.choices = communitiesSelect() + form.dataset.choices = datasetsSelect() + + if form.validate_on_submit(): + session['community'] = request.form['community'] + session['minyear'] = request.form['minyear'] + session['maxyear'] = request.form['maxyear'] + if session['minyear'] > session['maxyear']: + session['maxyear'] = session['minyear'] + + session['datasets'] = request.form['dataset'] + return redirect(url_for('main.index')) + + session['community_data'] = None + session['avg_temp'] = None + session['avg_indices'] = None + session['des_indices'] = None + + if 'community' in session: + community_id = session['community'] + if all(key in session for key in ('minyear', 'maxyear', 'datasets')): + community = DB.getCommunity(community_id) + + session['community_data'] = { + 'id': community_id, + 'name': community['name'], + 'latitude': round(community['latitude'], 5), + 'longitude': round(community['longitude'], 5), + } + + session['ds_name'] = session['datasets'].split(',') + + temps_arr = getTemps(session) + + session['avg_temp'] = avg_air_temp(temps_arr) + indices = ann_air_indices(temps_arr) + session['avg_indices'] = avg_air_indices(indices) + session['des_indices'] = des_air_indices(indices) + + return render_template('main/index.html', form=form) + + +@main.route('/datatypes') +def datatypes(): + return render_template('main/datatypes.html') + + +@main.route('/reset') +def reset(): + session.clear() + return redirect(url_for('main.index')) + + +@main.route('/details') +def details(): + temps = getTemps({'datasets': request.args.get('datasets', ''), + 'minyear': request.args.get('minyear', ''), + 'maxyear': request.args.get('maxyear', ''), + 'community_data': {'id': request.args.get('community_id', '')} + }) + return render_template('main/details.html', + lat=request.args.get('lat', ''), + lon=request.args.get('lon', ''), + community_name=request.args.get('name', ''), + temps=temps) + + +@main.route('/save') +def save(): + if 'save' in session: + i = str(len(session['save'])) + save = session['save'] + else: + save = dict() + i = '0' + + save[i] = { + 'datasets': session['datasets'], + 'ds_name': session['ds_name'], + 'community_data': session['community_data'], + 'minyear': session['minyear'], + 'maxyear': session['maxyear'], + 'avg_temp': session['avg_temp'], + 'avg_indices': session['avg_indices'], + 'des_indices': session['des_indices'], + } + + session.clear() + session['save'] = save + return redirect(url_for('main.index')) + + +@main.route('/delete') +def delete(): + record = request.args.get('record', '') + session['save'].pop(record) + return redirect(url_for('main.index')) diff --git a/app/misc/transform.sql b/app/misc/transform.sql new file mode 100644 index 0000000..c563e1e --- /dev/null +++ b/app/misc/transform.sql @@ -0,0 +1,79 @@ +-- Transforms AKIndices v1 schema to v2 schema +SELECT t.community_id, + d.datatype, + d.model, + d.modelname, + d.scenario, + d.resolution, + t.year::text, + json_build_array( + t.january::decimal(3,1), + t.february::decimal(3,1), + t.march::decimal(3,1), + t.april::decimal(3,1), + t.may::decimal(3,1), + t.june::decimal(3,1), + t.july::decimal(3,1), + t.august::decimal(3,1), + t.september::decimal(3,1), + t.october::decimal(3,1), + t.november::decimal(3,1), + t.december::decimal(3,1))::jsonb AS temps +INTO TEMP temp01 +FROM temperatures t +INNER JOIN datasets d ON d.id=t.dataset_id; + +-------------------------------------------------------------------------------- + +SELECT community_id, + datatype, + model, + modelname, + scenario, + resolution, + json_object_agg(year, temps)::jsonb AS data +INTO TEMP temp02 +FROM temp01 +GROUP BY community_id, datatype, model, modelname, scenario, resolution; + +-------------------------------------------------------------------------------- + +CREATE TEMP SEQUENCE a; +SELECT nextval('a') AS id, + community_id, + json_build_object( + 'datatype', datatype, + 'model', model, + 'modelname', modelname, + 'scenario', scenario, + 'resolution', resolution)::jsonb as dataset, + data +INTO TEMP temp03 +FROM temp02; + +-------------------------------------------------------------------------------- + +WITH all_json_key_value AS ( + SELECT id, community_id, t1.key, t1.value + FROM temp03, jsonb_each(dataset) AS t1 + UNION + SELECT id, community_id, t1.key, t1.value + FROM temp03, jsonb_each(data) AS t1 +) +SELECT community_id, json_object_agg(key, value) AS data +INTO TEMP temp04 +GROUP BY id, community_id; + +-------------------------------------------------------------------------------- + +SELECT community_id, json_agg(data)::jsonb AS data +INTO TEMP temp05 +FROM temp04 +GROUP BY community_id; + +-------------------------------------------------------------------------------- + +SELECT c.name, c.latitude, c.longitude, c.northing, c.easting, t.data +INTO new_communities +FROM temp05 t +INNER JOIN communities c ON c.id=t.community_id; diff --git a/akindices/static/css/BOOTSTRAP_LICENSE b/app/static/css/BOOTSTRAP_LICENSE old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/css/BOOTSTRAP_LICENSE rename to app/static/css/BOOTSTRAP_LICENSE diff --git a/akindices/static/css/akindices.css b/app/static/css/akindices.css old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/css/akindices.css rename to app/static/css/akindices.css diff --git a/akindices/static/css/bootstrap-readable.min.css b/app/static/css/bootstrap-readable.min.css old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/css/bootstrap-readable.min.css rename to app/static/css/bootstrap-readable.min.css diff --git a/akindices/static/fonts/glyphicons-halflings-regular.eot b/app/static/fonts/glyphicons-halflings-regular.eot old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/fonts/glyphicons-halflings-regular.eot rename to app/static/fonts/glyphicons-halflings-regular.eot diff --git a/akindices/static/fonts/glyphicons-halflings-regular.svg b/app/static/fonts/glyphicons-halflings-regular.svg old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/fonts/glyphicons-halflings-regular.svg rename to app/static/fonts/glyphicons-halflings-regular.svg diff --git a/akindices/static/fonts/glyphicons-halflings-regular.ttf b/app/static/fonts/glyphicons-halflings-regular.ttf old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/fonts/glyphicons-halflings-regular.ttf rename to app/static/fonts/glyphicons-halflings-regular.ttf diff --git a/akindices/static/fonts/glyphicons-halflings-regular.woff b/app/static/fonts/glyphicons-halflings-regular.woff old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/fonts/glyphicons-halflings-regular.woff rename to app/static/fonts/glyphicons-halflings-regular.woff diff --git a/akindices/static/js/bootstrap.min.js b/app/static/js/bootstrap.min.js old mode 100644 new mode 100755 similarity index 100% rename from akindices/static/js/bootstrap.min.js rename to app/static/js/bootstrap.min.js diff --git a/config.py b/config.py new file mode 100644 index 0000000..0e41ae1 --- /dev/null +++ b/config.py @@ -0,0 +1,30 @@ +import os + + +class Config: + SECRET_KEY = os.environ.get('SECRET_KEY') + APP_NAME = 'AKIndices' + APP_VERSION = '0.2.0' + TITLE = 'AKIndices' + COPYRIGHT_YEAR = 2015 + CSRF_ENABLED = True + DEBUG = False + + @staticmethod + def init_app(app): + pass + + +class DevelopmentConfig(Config): + DEBUG = True + TITLE = 'AKIndices (test)' + SECRET_KEY = os.environ.get('SECRET_KEY') or 'top secret' + SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or \ + 'postgres://matthew@localhost/akindices' + + +config = { + 'development': DevelopmentConfig, + + 'default': DevelopmentConfig +} diff --git a/config.py.default b/config.py.default deleted file mode 100644 index 2087a3f..0000000 --- a/config.py.default +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- - -import datetime - -PERMANENT_SESSION_LIFETIME = datetime.timedelta(minutes=30) -CSRF_ENABLED = True -SECRET_KEY = 'A super secret key' -DEBUG = False - -# Custom -ENGINE = 'postgres://user:pass@localhost/akindices' -SNAPDATA = '/path/to/raw/data' -COMMUNITIES = '/path/to/list/of/communities' -LOG = 'akindices.log' -MAXLOG = 1000000 -BACKUPCOUNT = 10 -TITLE = 'AKIndices' diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..1beff4d --- /dev/null +++ b/manage.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +import os +from app import create_app, db +from flask.ext.script import Manager + + +app = create_app(os.getenv('FLASK_CONFIG') or 'default') +manager = Manager(app) + + +@manager.command +def initdb(): + from sqlalchemy.sql import text + cmd = """ + CREATE TABLE new_communities ( + id serial NOT NULL, + name character varying(50) NOT NULL, + northing double precision NOT NULL, + easting double precision NOT NULL, + latitude double precision NOT NULL, + longitude double precision NOT NULL, + data jsonb NOT NULL, + CONSTRAINT new_communities_pkey PRIMARY KEY (id) + );""" + _ = db.engine.execute(text(cmd)) + + +if __name__ == '__main__': + manager.run() diff --git a/requirements.txt b/requirements.txt index 63a465f..9167a4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ Flask==0.10.1 -Flask-WTF==0.9.1 -Jinja2==2.7.1 -MarkupSafe==0.18 -SQLAlchemy==0.8.2 -WTForms==1.0.4 -Werkzeug==0.9.4 -itsdangerous==0.23 -numpy==1.7.1 -psycopg2==2.5.1 -wsgiref==0.1.2 +Flask-SQLAlchemy==2.0 +Flask-Script==2.0.5 +Jinja2==2.7.3 +MarkupSafe==0.23 +SQLAlchemy==1.0.5 +Werkzeug==0.10.4 +itsdangerous==0.24 +psycopg2==2.6.1 +Flask-WTF==0.12 +WTForms==2.0.2 +gunicorn==19.3.0 diff --git a/run.py b/run.py deleted file mode 100755 index 09191f7..0000000 --- a/run.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python - -from akindices import application - -application.run() diff --git a/tmp/restart.txt b/tmp/restart.txt deleted file mode 100644 index e69de29..0000000