increases precision to 64 in order to prevent real constraint (likely…

.vscode/launch.json

@@ -28,6 +28,21 @@
         "--event-type=FL"
       ]
     },
+    {
+      "name": "Python: SDO CLI Train",
+      "type": "python",
+      "request": "launch",
+      "stopOnEntry": false,
+      "justMyCode": false,
+      "program": "${workspaceRoot}/src/sdo/cli.py",
+      "console": "integratedTerminal",
+      "args": [
+        "sood",
+        "ce_vae",
+        "train",
+        "--config-file=./config/ce-vae/run-1.yaml"
+      ]
+    },
     {
       "name": "Python: SDO CLI Predict",
       "type": "python",

README.md

@@ -40,7 +40,7 @@ Commands:
 
 **Examples:**
 
-Download images from the Curate Image Parameter Dataset:
+Download images from the Curated Image Parameter Dataset:
 
 ```
 sdo-cli data download --path='./data/aia_171_2012' --start='2012-03-07T00:02:00' --end='2012-03-07T00:40:00' --freq='6min' --wavelength='171'

config/ce-vae/run-1.yaml

@@ -1,6 +1,6 @@
 model:
   load_path:
-    value: /Users/mariusgiger/Downloads/model-36tfoo4q.ckpt
+    value: /Users/mariusgiger/Downloads/model-1l9kiqhd.ckpt
 data:
   data_dir:
     value: fdl-sdoml-v2/sdomlv2_small.zarr/

src/sdo/sood/algorithms/ce_vae.py

@@ -374,8 +374,8 @@ def main(
     logger.info("found config")
     logger.info(json.dumps(config, indent=2))
 
-    work_dir = Path(
-        config.log_dir.value) / Path(f"{datetime.datetime.now().strftime(folder_time_format)}_cevae")
+    current_run_name = f"{datetime.datetime.now().strftime(folder_time_format)}_cevae"
+    work_dir = Path(config.log_dir.value) / Path(current_run_name)
     if not os.path.exists(work_dir):
         os.makedirs(work_dir)
 
@@ -486,6 +486,7 @@ def main(
                              # distributed training does not yet work because the data loader lambda cannot be pickled
                              gpus=config.devices.gpus.value,
                              profiler=profiler,
+                             precision=64,
                              accelerator="auto",
                              default_root_dir=work_dir,
                              callbacks=callbacks)
@@ -508,6 +509,8 @@ def main(
             logger.error(
                 "Please either provide a log/output dir or a prediction dir")
             sys.exit(0)
+        else:
+            pred_dir = Path(pred_dir) / Path(current_run_name)
 
         if not os.path.exists(pred_dir):
             os.makedirs(pred_dir, exist_ok=True)

src/sdo/sood/models/aes.py

@@ -3,6 +3,7 @@
 import torch
 import torch.distributions as dist
 import logging
+import sys
 
 from sdo.sood.models.nets import BasicEncoder, BasicGenerator
 
@@ -107,7 +108,7 @@ def forward(self, inpt, sample=True, no_dist=False, **kwargs):
             else:
                 return x_rec, z_dist
         except Exception as e:
-            np.set_printoptions(threshold=np.nan)
+            np.set_printoptions(threshold=sys.maxsize)
             logger.error(f"could not forward sample {mu}", e)
             raise e