Merge pull request #50 from DePaul-CCR/main

v0.83 Add data persistence

.gitignore

@@ -7,4 +7,6 @@ __pycache__/
 flask_session/
 .DS_Store
 .vscode/
-fs_q_regx.html
+fs_q_regx.html
+# ignores the Google Sheets API credential file
+dsqscreen-*.json

README.md

@@ -32,8 +32,20 @@ MacOS:
 -> To start production server  
 `gunicorn main:app prod`
 
-###NB
+###N.B.
 -If you change the routes please update the route-order "page_flow" tuple in 'utils/back_function.py' so the back button works correctly 
 + the docs/dsqitems_and_routes_map.txt file, which shows the DSQ item var names mapped to routes, should be updated also. 
++ dsq_columns in utils/export_columns_util.py are order-specific so will also be misaligned if the order of questions changes. (TODO: make the data export order-agnostic)
+
+-If you change which items / questions are used, you also need to update the dsq_columns in utils/export_columns_util.py
++Also, you will need to update the columns in the export sheet: depaulccrdev@gmail.com's "DSQScreen Output"
++Manually, you can compare the data frame which holds the responses to the header row in the Google sheet via
+```
+#Debug breakpoint on ~line 22 in utils/export_columns_util.py
+import nump as np
+gs_arr = np.array(#paste gsheet column headers in here formatted as a list ['x', 'y', 'z'...]
+)
+np.array(df.columns.to_list()) == gs_arr
+```
 
 -To make changes to question pages there is one partial for all the F/S questions and then the single-question items are less templated so need to be modified directly (dsq/viral.html and short_form/reduction.html)

main.py

@@ -816,6 +816,8 @@ def intolerant():
 
 @app.route('/dsq_dx', methods=['get'])
 def graph3():
+    # store IP for response data export
+    session['ip'] = request.remote_addr
     return dsq_utils.dsq_diagnose()
 
 @app.route('/about', methods=['post', 'get'])

requirements.txt

@@ -9,4 +9,5 @@ matplotlib
 mysqlclient
 gunicorn
 imblearn
-seaborn
+seaborn
+pygsheets

utils/domainScores.py

@@ -37,7 +37,7 @@
 
 #Maybe change this to the imputed data:
 df = pd.read_csv('utils/MECFS and Controls F+S Reduction.csv')
-sdf = df
+sdf = pd.read_csv('utils/MECFS and Controls F+S Reduction.csv')
 
 df['fatigue_mean'] = np.mean(df[fatigue_domain], axis=1)
 df['pem_mean'] = np.mean(df[pem_domain], axis=1)
@@ -48,7 +48,7 @@
 df['neuro_mean'] = np.mean(df[neuro_domain], axis=1)
 df['immune_mean'] = np.mean(df[immune_domain], axis=1)
 
-sdf['fatigue_mean'] = np.mean(df[sf_fatigue_domain], axis=1)
+sdf['fatigue_mean'] = np.mean(sdf[sf_fatigue_domain], axis=1)
 sdf['pem_mean'] = np.mean(sdf[sf_pem_domain], axis=1)
 sdf['sleep_mean'] = np.mean(sdf[sf_sleep_domain], axis=1)
 sdf['pain_mean'] = np.mean(sdf[sf_pain_domain], axis=1)

utils/dsq_utils.py

@@ -5,6 +5,7 @@
 import plotly.utils
 from utils.general_utils import get_score
 import utils.domainScores as ds
+from utils.export_columns_util import build_dataframe_for_export, dump_collected_data_to_sheet
 # see dsqitems_and_routes_map.txt for info on each section of the screener
 
 def dsq_diagnose():
@@ -314,6 +315,11 @@ def dsq_diagnose():
 
     graphJSON = dsq_graph()
 
+    # dump to Google Sheets
+    df = build_dataframe_for_export(session)
+    dump_collected_data_to_sheet(df)
+
+
     return render_template("results/graph3.html", graphJSON=graphJSON, dsq_message=dsq_message, 
                            ccc_msg=ccc_msg, ccc_fatiguecheck=ccc_fatiguecheck,
                            ccc_pemcheck=ccc_pemcheck, ccc_paincheck=ccc_paincheck, ccc_sleepcheck=ccc_sleepcheck,

utils/export_columns_util.py

@@ -0,0 +1,158 @@
+import pandas as pd
+import pygsheets
+import datetime
+
+def build_dataframe_for_export (session, stage = "dsq"):
+  df = pd.DataFrame()
+  df['sid'] = [session.sid]
+  # stores UTC time timestamp
+  df['date'] = datetime.datetime.now(datetime.timezone.utc).strftime("%m/%d/%Y")
+  df['time'] = datetime.datetime.now(datetime.timezone.utc).strftime("%H:%M:%S")
+  df['ip'] = session['ip']
+  # add all items currently collected to the data frame
+  for item in get_columns_for_stage(stage):
+    if session[item]:
+      df[item] = [session[item]]
+    else:
+      df[item] = None
+  return df
+
+def dump_collected_data_to_sheet(df):
+  gc = pygsheets.authorize(service_file='etc/secrets/dsqscreen-401016-0e6db07e3d56.json')
+  sh = gc.open('DSQScreen Output')
+  wks=sh[0]
+  # finds if session_id already exists in the sheet and replaced values if so
+  # session_row = wks.find(df['sid'][0])
+  # if bool(session_row):
+  #   wks.update_values(session_row[0].label, values=df.values.tolist())
+  # else:
+  wks.update_values("A" + get_next_row(wks), df.values.tolist())
+
+def get_next_row(wks):
+  # returns a string of length of column 1 + 1
+  return str(len(wks.get_col(1,include_tailing_empty=False)) + 1)
+
+def get_columns_for_stage(stage):
+  if stage == "screener":
+    return screener_columns
+  elif stage == "short_form":
+    return screener_columns + sf_columns
+  else:
+    return screener_columns + sf_columns + dsq_columns
+
+screener_columns = [
+"fatiguescoref",
+"fatiguescores",
+"minexf",
+"minexs",
+"sleepf",
+"sleeps",
+"rememberf",
+"remembers"]
+
+sf_columns = [
+"soref",
+"sores",
+"attentionf",
+"attentions",
+"musclef",
+"muscles",
+"bloatf",
+"bloats",
+"bowelf",
+"bowels",
+"unsteadyf",
+"unsteadys",
+"limbsf",
+"limbss",
+"hotf",
+"hots",
+"fluf",
+"flus",
+"smellf",
+"smells",
+"reduction"]
+
+dsq_columns = [
+"viral",
+"heavyf",
+"heavys",
+"mentalf",
+"mentals",
+"drainedf",
+"draineds",
+"napf",
+"naps",
+"fallf",
+"falls",
+"stayf",
+"stays",
+"earlyf",
+"earlys",
+"alldayf",
+"alldays",
+"jointpainf",
+"jointpains",
+"eyepainf",
+"eyepains",
+"chestpainf",
+"chestpains",
+"stomachf",
+"stomachs",
+"headachesf",
+"headachess",
+"twitchesf",
+"twitchess",
+"weakf",
+"weaks",
+"noisef",
+"noises",
+"lightsf",
+"lightss",
+"wordf",
+"words",
+"understandf",
+"understands",
+"focusf",
+"focuss",
+"visionf",
+"visions",
+"depthf",
+"depths",
+"slowf",
+"slows",
+"absentf",
+"absents",
+"bladderf",
+"bladders",
+"nauseaf",
+"nauseas",
+"shortf",
+"shorts",
+"dizzyf",
+"dizzys",
+"heartf",
+"hearts",
+"weightf",
+"weights",
+"appetitef",
+"appetites",
+"sweatf",
+"sweats",
+"nightf",
+"nights",
+"chillsf",
+"chillss",
+"hitempf",
+"hitemps",
+"lotempf",
+"lotemps",
+"alcoholf",
+"alcohols",
+"throatf",
+"throats",
+"lymphnodesf",
+"lymphnodess",
+"feverf",
+"fevers",
+"intolerant"]

utils/screener_utils.py

@@ -2,6 +2,7 @@
 import plotly.graph_objects as go
 import json
 import plotly.utils
+from utils.export_columns_util import build_dataframe_for_export, dump_collected_data_to_sheet
 # see dsqitems_and_routes_map.txt for info on each section of the screener
 
 def screener_diagnose():
@@ -61,6 +62,11 @@ def screener_diagnose():
     fig.update_yaxes(range=[0, 100], dtick=25, titlefont=dict(size=15))
     fig.update_xaxes(tickfont=dict(size=13), titlefont=dict(size=15))
     graphJSON = json.dumps(fig, cls=plotly.utils.PlotlyJSONEncoder)
+
+    # dump to Google Sheets
+    df = build_dataframe_for_export(session, "screener")
+    dump_collected_data_to_sheet(df)
+
     return render_template("results/graph.html",
                            graphJSON = graphJSON, 
                            screen_message = screen_message,

utils/short_form_utils.py

@@ -5,6 +5,7 @@
 import plotly.utils
 import utils.domainScores as ds
 from utils.general_utils import get_score
+from utils.export_columns_util import build_dataframe_for_export, dump_collected_data_to_sheet
 # see dsqitems_and_routes_map.txt for info on each section of the screener
 
 def short_form_diagnose():
@@ -136,6 +137,10 @@ def short_form_diagnose():
 
     graphJSON = short_form_graph()
 
+    # dump to Google Sheets
+    df = build_dataframe_for_export(session, "short_form")
+    dump_collected_data_to_sheet(df)
+
     return render_template("results/graph2.html", graphJSON=graphJSON, ccc_msg=ccc_msg, ccc_fatiguecheck=ccc_fatiguecheck,
                            ccc_pemcheck=ccc_pemcheck, ccc_paincheck=ccc_paincheck, ccc_sleepcheck=ccc_sleepcheck,
                            ccc_cogcheck=ccc_cogcheck, ccc_autocheck=ccc_autocheck, ccc_immunecheck=ccc_immunecheck,

website/screener_views.py

@@ -92,6 +92,8 @@ def page4():
 
 @screener_views.route('/screener_dx')
 def graph():
+    # store IP for response data export
+    session['ip'] = request.remote_addr
     return screener_utils.screener_diagnose()
 
 # not currently in use since we disabled users - PC 7/21/23

website/short_form.py

@@ -184,4 +184,6 @@ def reduction():
 
 @short_form.route('/short_form_dx', methods=['post', 'get'])
 def graph2():
+    # store IP for response data export
+    session['ip'] = request.remote_addr
     return short_form_utils.short_form_diagnose()

website/static/js/disable_next.js

@@ -7,7 +7,6 @@ if(document.querySelector('.single-question-container')) {
     if (radios.some(checked)) {
       nextButton.classList.remove('disabled')
     } else {
-      debugger
       nextButton.classList.add('disabled')
     }
   })
@@ -24,4 +23,10 @@ if(document.querySelector('.single-question-container')) {
       nextButton.classList.add('disabled')
     }
   })
-}
+}
+
+nextButton.addEventListener('click', (e) => {
+  setTimeout(function () {
+    e.target.disabled = true;
+}, 0);
+});