forked from NicoRiedel/barzooka
-
Notifications
You must be signed in to change notification settings - Fork 0
/
webapp.py
385 lines (309 loc) · 12.3 KB
/
webapp.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
"""Web app and utilities for rainbow colormap monitor
"""
import os
import os.path
import re
import math
import itertools
import flask
from flask_rq2 import RQ
from flask_mail import Mail, Message
from flask_wtf.csrf import CSRFProtect, CSRFError
from waitress import serve
from sqlalchemy import desc
import tweepy
import click
import pytest
from models import db, Biorxiv, Test
from biorxiv_scraper import find_authors, find_date, count_pages
from detect_bargraph import detect_graph_types_from_iiif
import utils
from fastai.vision import *
#ml_path = os.environ['WEBAPP_PATH']
learn = load_learner(path='/data01/webapp/barzooka/', file='export.pkl')
# Reads env file into environment, if found
_ = utils.read_env()
app = flask.Flask(__name__)
app.config['BASE_URL'] = os.environ['BASE_URL']
# For data storage
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ['SQLALCHEMY_DATABASE_URI']
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
db.init_app(app)
# For job handling
app.config['RQ_REDIS_URL'] = os.environ['RQ_REDIS_URL']
rq = RQ(app)
# For monitoring papers (until Biorxiv provides a real API)
app.config['TWITTER_APP_KEY'] = os.environ['TWITTER_APP_KEY']
app.config['TWITTER_APP_SECRET'] = os.environ['TWITTER_APP_SECRET']
app.config['TWITTER_KEY'] = os.environ['TWITTER_KEY']
app.config['TWITTER_SECRET'] = os.environ['TWITTER_SECRET']
app.config['DEBUG'] = os.environ.get('DEBUG', 0)
app.config['WEB_PASSWORD'] = os.environ['WEB_PASSWORD']
app.config['SECRET_KEY'] = os.environ['SECRET_KEY']
csrf = CSRFProtect(app)
tweepy_auth = tweepy.OAuthHandler(
app.config['TWITTER_APP_KEY'], app.config['TWITTER_APP_SECRET'])
tweepy_auth.set_access_token(
app.config['TWITTER_KEY'], app.config['TWITTER_SECRET'])
tweepy_api = tweepy.API(tweepy_auth)
@app.route('/')
def home():
"""Renders the website with current results
"""
cats = flask.request.args.get('categories')
if cats:
cats = [int(x) for x in cats.split(',')]
else:
cats = [-2, -1, 0, 1, 2]
papers = (Biorxiv.query
.filter(Biorxiv.parse_status.in_(cats))
.order_by(desc(Biorxiv.created))
.limit(2000)
.all())
return flask.render_template('main.html', app=app, papers=papers)
@app.route('/pages/<string:paper_id>')
def pages(paper_id, prepost=1, maxshow=10):
"""Pages to show for preview
1-index I think...
"""
record = Biorxiv.query.filter_by(id=paper_id).first()
if not record:
return flask.jsonify({})
pages = record.pages
page_count = record.page_count
# if requested, show all pages with each page's status
try:
all_pages = flask.request.args.get('all') == "1"
except:
all_pages = False
show_pgs = {}
if all_pages:
for i in range(1, page_count + 1):
if i in pages:
show_pgs[i] = True
else:
show_pgs[i] = False
elif pages:
# add all detected pages up to maxshow count
show_pgs = {i:True for i in pages[:maxshow]}
# pad with undetected pages
for i in pages:
for j in range(i - prepost, min(i + prepost + 1, page_count)):
if len(show_pgs) < maxshow:
if j not in pages:
show_pgs[j] = False
else:
break
else:
show_pgs = {i:False for i in range(1, maxshow + 1)}
return flask.jsonify({'pdf_url': record.pdf_url, 'pages': show_pgs})
@app.route('/detail/<string:paper_id>')
def show_details(paper_id, prepost=1, maxshow=10):
"""
"""
record = Biorxiv.query.filter_by(id=paper_id).first()
if not record:
flask.flash('Sorry! Results with that ID have not been found')
return flask.redirect('/')
# display images
return flask.render_template('detail.html',
paper_id=record.id, title=record.title, url=record.url,
pages=", ".join([str(p) for p in record.pages]),
pages_pie=", ".join([str(p) for p in record.pages_pie]),
pages_bardot=", ".join([str(p) for p in record.pages_bardot]),
pages_box=", ".join([str(p) for p in record.pages_box]),
pages_hist=", ".join([str(p) for p in record.pages_hist]),
pages_dot=", ".join([str(p) for p in record.pages_dot]),
pages_violin=", ".join([str(p) for p in record.pages_violin]),
pages_positive=", ".join([str(p) for p in record.pages_positive]),
parse_status=record.parse_status, email_sent=record.email_sent
)
@app.route('/notify/<string:paper_id>', methods=['POST'])
@app.route('/notify/<string:paper_id>/<int:force>', methods=['POST'])
def notify_authors(paper_id, force=0):
if flask.session.get('logged_in'):
record = Biorxiv.query.filter_by(id=paper_id).first()
if not record:
return flask.jsonify(result=False, message="paper not found")
addr = record.author_contact.values()
addr = list(itertools.chain.from_iterable(addr))
# don't bother everyone if there are a ton of authors
if len(addr) > 6:
addr = [*addr[:2], *addr[-3:]]
if addr is [] or '@' not in "".join(addr):
return flask.jsonify(result=False,
message="mangled or missing email addresses")
if force != 1 and record.email_sent == 1:
return flask.jsonify(result=False,
message="email already sent. use the cli to send another")
msg = Message(
"[JetFighter] bioRxiv manuscript {}".format(record.id),
recipients=addr,
reply_to=app.config['MAIL_REPLY_TO'])
msg.body = flask.render_template("email_notification.txt",
paper_id=paper_id,
pages=record.pages_str,
title=record.title,
detail_url=flask.url_for('show_details', paper_id=paper_id))
mail.send(msg)
record.email_sent = 1
db.session.merge(record)
db.session.commit()
return flask.jsonify(result=True, message="successfully sent")
else:
return flask.jsonify(result=False, message="not logged in")
@app.route('/toggle/<string:paper_id>', methods=['POST'])
def toggle_status(paper_id):
if flask.session.get('logged_in'):
record = Biorxiv.query.filter_by(id=paper_id).first()
if not record:
return flask.jsonify(result=False, message="paper not found")
if record.parse_status > 0:
record.parse_status = -2
elif record.parse_status < 0:
record.parse_status = 2
else:
return flask.jsonify(result=False, message="Not yet parsed")
db.session.merge(record)
db.session.commit()
return flask.jsonify(result=True, message="successfully changed")
else:
return flask.jsonify(result=False, message="not logged in")
@app.route('/login', methods=['GET', 'POST'])
def admin_login():
if flask.request.method == 'GET':
if flask.session.get('logged_in'):
flask.flash('You are already logged in!')
return flask.redirect('/')
return flask.render_template('login.html')
if flask.request.form['password'] == app.config['WEB_PASSWORD']:
flask.session['logged_in'] = True
else:
flask.flash('wrong password!')
return flask.render_template('login.html')
return flask.redirect('/')
@app.route('/logout', methods=['GET'])
def logout():
logged_in = flask.session.get('logged_in')
flask.session.clear()
if logged_in:
flask.flash("You have been successfully logged out")
else:
flask.flash("Not logged in! (but the session has been cleared)")
return flask.redirect('/')
@app.errorhandler(CSRFError)
def handle_csrf_error(e):
flask.flash('CSRF Error. Try again?')
return flask.redirect(flask.url_for('admin_login'))
@app.cli.command()
@click.option('--count', default=500)
def retrieve_timeline(count):
"""Picks up current timeline (for testing)
"""
# as we currently have to wait for a long time until the parsing is done
# go through tweets twice: first only add basic info to SQL database,
# second, parse pdf for bargraphs
for t in tweepy.Cursor(tweepy_api.user_timeline,
screen_name='biorxivpreprint', trim_user='True',
include_entities=True, tweet_mode='extended').items(count):
parse_tweet(t)
def parse_tweet(t, db=db, objclass=Biorxiv, verbose=True):
"""Parses tweets for relevant data,
writes each paper to the database,
dispatches a processing job to the processing queue (rq)
"""
try:
text = t.extended_tweet["full_text"]
except AttributeError:
pass
try:
text = t.full_text
except AttributeError:
text = t.text
if verbose:
print(t.id_str, text[:25], end='\r')
if not db:
return
try:
url = t.entities['urls'][0]['expanded_url']
code = os.path.basename(url)
except:
print('Error parsing url/code from tweet_id', t.id_str)
return
try:
title = re.findall('(.*?)\shttp', text)[0]
except:
# keep ASCII only (happens with some Test tweets)
title = re.sub(r'[^\x00-\x7f]', r'', text)
obj = objclass(
id=code,
created=t.created_at,
title=title,
)
obj = db.session.merge(obj)
db.session.commit()
# Only add to queue if not yet processed and if we actually want to do all the processing
if obj.parse_status == 0:
process_paper.queue(obj)
@rq.job(timeout='30m')
def process_paper(obj):
#Processes paper starting from url/code
#
#1. get object, find page count and posted date
#2. detect graph classes
#3. if bargraph, get authors
#4. update database entry with graph detection and author info
with app.app_context():
obj = db.session.merge(obj)
if obj.page_count == 0:
obj.page_count = count_pages(obj.id)
if obj.posted_date == "":
obj.posted_date = find_date(obj.id)
class_pages = detect_graph_types_from_iiif(obj.id, obj.page_count, learn)
obj.pages = class_pages["bar"]
obj.pages_pie = class_pages["pie"]
obj.pages_hist = class_pages["hist"]
obj.pages_bardot = class_pages["bardot"]
obj.pages_box = class_pages["box"]
obj.pages_dot = class_pages["dot"]
obj.pages_violin = class_pages["violin"]
obj.pages_positive = class_pages["positive"]
if (len(obj.pages) + len(obj.pages_pie)) > 0:
obj.parse_status = 1
if obj.author_contact is None:
obj.author_contact = find_authors(obj.id)
else:
obj.parse_status = -1
db.session.merge(obj)
db.session.commit()
## NOTE: NEEDS WORK
@pytest.fixture()
def test_setup_cleanup():
# should only be one, but... just in case
for obj in Test.query.filter_by(id='172627v1').all():
db.session.delete(obj)
db.session.commit()
# Delete temporary row
for obj in Test.query.filter_by(id='172627v1').all():
db.session.delete(obj)
db.session.commit()
def test_integration(test_setup_cleanup):
"""Submit job for known jet colormap. Remove from database beforehand.
Write to database.
Check for written authors.
"""
#testq = rq.Queue('testq', is_async=False)
preobj = Test(id='172627v1')
testq.enqueue(process_paper, preobj)
postobj = Test.query.filter_by(id='172627v1').first()
# check that document was correctly identified as having a rainbow colormap
assert postobj.parse_status
# check that authors were correctly retrieved
authors = postobj.author_contact
assert authors['corr'] == ['t.ellis@imperial.ac.uk']
assert set(authors['all']) == set([
'o.borkowski@imperial.ac.uk', 'carlos.bricio@gmail.com',
'g.stan@imperial.ac.uk', 't.ellis@imperial.ac.uk'])
if __name__ == "__main__":
#app.run(debug=True, threaded=True, use_reloader=True)
serve(app, host='127.0.0.1', port=5000, url_scheme='https')