From 818fdc4abfe2ce8b786c91126901ca954aa90478 Mon Sep 17 00:00:00 2001
From: Brian Healy <42810347+bfhealy@users.noreply.github.com>
Date: Mon, 8 Apr 2024 12:00:45 -0500
Subject: [PATCH] Update slurm script generators with NSCA Delta specifics
 (#574)

---
 tools/combine_preds_slurm.py     |  4 ++++
 tools/generate_features_slurm.py | 10 ++++++++++
 tools/run_inference_slurm.py     | 13 +++++++++++++
 tools/train_algorithm_slurm.py   | 13 +++++++++++++
 4 files changed, 40 insertions(+)

diff --git a/tools/combine_preds_slurm.py b/tools/combine_preds_slurm.py
index c155c9ca..0c74df3f 100755
--- a/tools/combine_preds_slurm.py
+++ b/tools/combine_preds_slurm.py
@@ -128,6 +128,10 @@ def main():
     if args.cluster_name in ['Expanse', 'expanse', 'EXPANSE']:
         fid.write('module purge\n')
         fid.write(f'source activate {args.python_env_name}\n')
+    elif args.cluster_name in ['Delta', 'delta', 'DELTA']:
+        fid.write('module purge\n')
+        fid.write('module add anaconda3_cpu\n')
+        fid.write(f'source activate {args.python_env_name}\n')
 
     fid.write(
         'combine-preds --path-to-preds %s --combined-preds-dirname %s --specific-field %s --dateobs %s --dnn-directory %s --xgb-directory %s --agg-method %s --p-threshold %s %s \n'
diff --git a/tools/generate_features_slurm.py b/tools/generate_features_slurm.py
index 8eeab5df..24f9c7df 100755
--- a/tools/generate_features_slurm.py
+++ b/tools/generate_features_slurm.py
@@ -425,6 +425,12 @@ def main():
         fid.write('module add gpu/0.15.4\n')
         fid.write('module add cuda\n')
         fid.write(f'source activate {args.python_env_name}\n')
+    elif args.cluster_name in ['Delta', 'delta', 'DELTA']:
+        fid.write('module purge\n')
+        fid.write('module add anaconda3_gpu\n')
+        fid.write('module add cuda\n')
+        fid.write('module add gcc-runtime\n')
+        fid.write(f'source activate {args.python_env_name}\n')
 
     if args.doQuadrantFile:
         qid = '$QID'
@@ -502,6 +508,10 @@ def main():
         fid.write('module purge\n')
         fid.write('module add slurm\n')
         fid.write(f'source activate {args.python_env_name}\n')
+    elif args.cluster_name in ['Delta', 'delta', 'DELTA']:
+        fid.write('module purge\n')
+        fid.write('module add anaconda3_cpu\n')
+        fid.write(f'source activate {args.python_env_name}\n')
 
     if not args.doSubmitLoop:
         if args.runParallel:
diff --git a/tools/run_inference_slurm.py b/tools/run_inference_slurm.py
index 2219e0ce..4507f5b5 100755
--- a/tools/run_inference_slurm.py
+++ b/tools/run_inference_slurm.py
@@ -180,6 +180,15 @@ def main():
             fid.write('module add gpu/0.15.4\n')
             fid.write('module add cuda\n')
         fid.write(f'source activate {args.python_env_name}\n')
+    elif args.cluster_name in ['Delta', 'delta', 'DELTA']:
+        fid.write('module purge\n')
+        if args.gpus > 0:
+            fid.write('module add anaconda3_gpu\n')
+            fid.write('module add cuda\n')
+            fid.write('module add gcc-runtime\n')
+        else:
+            fid.write('module add anaconda3_cpu\n')
+        fid.write(f'source activate {args.python_env_name}\n')
 
     fid.write(f"{script_path} $FID" + '\n')
     fid.close()
@@ -204,6 +213,10 @@ def main():
         fid.write('module purge\n')
         fid.write('module add slurm\n')
         fid.write(f'source activate {args.python_env_name}\n')
+    elif args.cluster_name in ['Delta', 'delta', 'DELTA']:
+        fid.write('module purge\n')
+        fid.write('module add anaconda3_cpu\n')
+        fid.write(f'source activate {args.python_env_name}\n')
 
     fid.write(
         'run-inference-job-submission --dirname %s --scriptname %s --user %s --algorithm %s\n'
diff --git a/tools/train_algorithm_slurm.py b/tools/train_algorithm_slurm.py
index 88cec058..4ed282ea 100755
--- a/tools/train_algorithm_slurm.py
+++ b/tools/train_algorithm_slurm.py
@@ -229,6 +229,15 @@ def main():
             fid.write('module add gpu/0.15.4\n')
             fid.write('module add cuda\n')
         fid.write(f'source activate {args.python_env_name}\n')
+    elif args.cluster_name in ['Delta', 'delta', 'DELTA']:
+        fid.write('module purge\n')
+        if args.gpus > 0:
+            fid.write('module add anaconda3_gpu\n')
+            fid.write('module add cuda\n')
+            fid.write('module add gcc-runtime\n')
+        else:
+            fid.write('module add anaconda3_cpu\n')
+        fid.write(f'source activate {args.python_env_name}\n')
 
     fid.write("scope-train " + "--tag $TID " + " ".join(line_info) + '\n')
     fid.close()
@@ -253,6 +262,10 @@ def main():
         fid.write('module purge\n')
         fid.write('module add slurm\n')
         fid.write(f'source activate {args.python_env_name}\n')
+    elif args.cluster_name in ['Delta', 'delta', 'DELTA']:
+        fid.write('module purge\n')
+        fid.write('module add anaconda3_cpu\n')
+        fid.write(f'source activate {args.python_env_name}\n')
 
     if args.sweep:
         fid.write(