Feature/sdo ml v2 loader pt2

.vscode/launch.json

@@ -5,7 +5,15 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python: SDO CLI",
+      "name": "Python: Current File",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": true
+    },
+    {
+      "name": "Python: SDO CLI get events",
       "type": "python",
       "request": "launch",
       "stopOnEntry": false,
@@ -15,9 +23,55 @@
       "args": [
         "events",
         "get",
-        "--start=2011-08-01T00:00:00",
-        "--end=2011-08-02T23:59:59",
-        "--event-type=AR"
+        "--start=2011-01-01T00:00:00",
+        "--end=2020-12-31T23:59:59",
+        "--event-type=FL"
+      ]
+    },
+    {
+      "name": "Python: SDO CLI Predict",
+      "type": "python",
+      "request": "launch",
+      "stopOnEntry": false,
+      "justMyCode": false,
+      "program": "${workspaceRoot}/src/sdo/cli.py",
+      "console": "integratedTerminal",
+      "args": [
+        "sood",
+        "ce_vae",
+        "predict",
+        "--config-file=./config/ce-vae/run-1.yaml"
+      ]
+    },
+    {
+      "name": "Python: SDO CLI GOES Download",
+      "type": "python",
+      "request": "launch",
+      "stopOnEntry": false,
+      "justMyCode": false,
+      "program": "${workspaceRoot}/src/sdo/cli.py",
+      "console": "integratedTerminal",
+      "args": [
+        "goes",
+        "download",
+        "--start=2010-01-01T00:00:00",
+        "--end=2020-12-31T23:59:59",
+        "--output=./tmp/new"
+      ]
+    },
+    {
+      "name": "Python: SDO CLI GOES Get",
+      "type": "python",
+      "request": "launch",
+      "stopOnEntry": false,
+      "justMyCode": false,
+      "program": "${workspaceRoot}/src/sdo/cli.py",
+      "console": "integratedTerminal",
+      "args": [
+        "goes",
+        "get",
+        "--timestamp=2015-06-01T02:20:00",
+        "--cache-dir=./tmp"
       ]
     }
   ]

README.md

@@ -61,6 +61,7 @@ sdo-cli data patch --path='./data/aia_171_2012_256' --targetpath='./data/aia_171
 Loading Events from HEK:
 
 ```
+pip install psycopg2-binary
 docker-compose up
 sdo-cli events get --start="2012-01-01T00:00:00" --end="2012-01-02T23:59:59" --event-type="AR"
 ```
@@ -86,6 +87,14 @@ make setup
 make install
 ```
 
+### Publishing
+
+Add your pypi credentials to `~/.pypirc`, increase the version number in `setup.py` and run:
+
+```
+make publish
+```
+
 ### Troubleshooting
 
 Tensorflow only works with Python versions < 3.9.
@@ -103,3 +112,6 @@ Also refer to this [link](https://www.chrisjmendez.com/2017/08/03/installing-mul
 
 - [1] Ahmadzadeh, Azim, Dustin J. Kempton, and Rafal A. Angryk. "A Curated Image Parameter Data Set from the Solar Dynamics Observatory Mission." The Astrophysical Journal Supplement Series 243.1 (2019): 18.
 - [2] Zimmerer, David, et al. "Context-encoding variational autoencoder for unsupervised anomaly detection." arXiv preprint arXiv:1812.05941 (2018).
+
+
+sdo-cli events get --start="2012-01-01T00:00:00" --end="2012-01-02T23:59:59" --event-type="FL"

config/ce-vae/defaults.yaml

@@ -0,0 +1,95 @@
+model:
+  target_size:
+    value: 256
+    desc: "Target size of the reconstructed output"
+  z_dim:
+    value: 128
+    desc: "Dimension of the latent space"
+  fmap_sizes:
+    value: [16, 64, 256, 1024]
+    desc: "Feature map sizes for the CNN"
+  ce_factor:
+    value: 0.5
+    desc: "Amount to which the context-encoder contributes to the model (between 0 only VAE and 1 only CE)"
+  load_path:
+    value: null
+    desc: "Path to a pretrained model"
+data:
+  batch_size:
+    value: 16
+    desc: "How many samples per batch to load"
+  channel:
+    value: "171A"
+    desc: "Channel name that should be used. If None all available channels will be used."
+  data_dir:
+    value: ./data
+    desc: "Path to the root directory of the dataset"
+  dataset:
+    value: SDOMLDatasetV2
+    desc: "Which dataset to use (CuratedImageParameterDataset, SDOMLDatasetV1 or SDOMLDatasetV2)"
+  num_data_loader_workers:
+    value: 0
+    desc: "How many subprocesses to use for data loading. 0 means that the data will be loaded in the main process."
+  prefetch_factor:
+    value: 8
+    desc: "Number of samples loaded in advance by each worker. 2 means there will be a total of 2 * num_workers samples prefetched across all workers."
+  pin_memory:
+    value: true
+    desc: "If true, the data loader will copy Tensors into CUDA pinned memory before returning them"
+  sdo_ml_v2:
+    storage_driver:
+      value: fs
+      desc: "Storage driver used to load the data. Either 'gcs' (Google Storage Bucket) or 'fs' (local file system)"
+    year:
+      value: 2010
+      desc: "Allows to prefilter the dataset by year. If None all available years will be used."
+    train_start_date:
+      value: "2010-08-30 00:00:00"
+      desc: "Allows to restrict the dataset temporally"
+    train_end_date:
+      value: "2010-08-30 23:59:59"
+      desc: "Allows to restrict the dataset temporally"
+    test_start_date:
+      value: "2010-08-29 00:00:00"
+      desc: "Allows to restrict the dataset temporally"
+    test_end_date:
+      value: "2010-08-29 23:59:59"
+      desc: "Allows to restrict the dataset temporally"
+    freq:
+      value: null
+      desc: "Allows to downsample the dataset temporally, should be bigger than the min interval for the observed channel. When using freq, start and end should also be specified for train and test"
+    train_val_split_ratio:
+      value: 0.7
+      desc: "Split-ratio for the train-validation split"
+    train_val_split_temporal_chunk_size:
+      value: 14d
+      desc: "Temporal chunk size for the train-validation splits."
+predict:
+  mode:
+    value: pixel
+    desc: "Mode for anomaly scoring (pixel or sample)"
+  pred_dir:
+    value: "./output/predictions"
+    desc: "Output directory for predictions."
+  score_mode:
+    value: combi
+    desc: "Score mode used for anomaly scoring ('rec', 'grad' or 'combi')"
+train:
+  n_epochs:
+    value: 10
+    desc: "Stop training once this number of epochs is reached."
+  lr:
+    value: 0.0001
+    desc: "Learning rate"
+  use_geco:
+    value: false
+    desc: "Whether to use Generalized ELBO with Constrained Optimization update step."
+  beta:
+    value: 0.01
+    desc: "Weighting factor for KL loss influence on loss."
+  print_every_iter:
+    value: 100
+    desc: ""
+log_dir:
+  value: "./output/logs"
+  desc: "Output directory for log."

config/ce-vae/run-1.yaml

@@ -0,0 +1,32 @@
+model:
+  load_path:
+    value: /Users/mariusgiger/Downloads/model-36tfoo4q.ckpt
+data:
+  data_dir:
+    value: fdl-sdoml-v2/sdomlv2_small.zarr/
+  dataset:
+    value: SDOMLDatasetV2
+  num_data_loader_workers:
+    value: 0
+  prefetch_factor:
+    value: 2
+  batch_size:
+    value: 1
+  sdo_ml_v2:
+    storage_driver:
+      value: gcs
+    train_start_date:
+      value: "2010-08-30 00:00:00"
+    train_end_date:
+      value: "2010-08-30 23:59:59"
+    test_start_date:
+      value: "2010-08-29 00:00:00"
+    test_end_date:
+      value: "2010-08-29 23:59:59"
+    train_val_split_temporal_chunk_size:
+      value: 1h
+predict:
+  mode:
+    value: sample
+  pred_dir:
+    value: "./output/predictions"

config/ce-vae/run-fhnw-1.yaml

@@ -0,0 +1,34 @@
+model:
+  load_path:
+    value: null
+data:
+  data_dir:
+    value: /mnt/nas05/astrodata01/astroml_data/sdomlv2_small/sdomlv2_small.zarr
+  dataset:
+    value: SDOMLDatasetV2
+  num_data_loader_workers:
+    value: 16
+  prefetch_factor:
+    value: 8
+  batch_size:
+    value: 32
+  sdo_ml_v2:
+    train_start_date:
+      value: "2010-08-01 00:00:00"
+    train_end_date:
+      value: "2010-08-20 23:59:59"
+    test_start_date:
+      value: "2010-08-21 00:00:00"
+    test_end_date:
+      value: "2010-08-31 23:59:59"
+    train_val_split_ratio:
+      value: 0.8
+    train_val_split_temporal_chunk_size:
+      value: 3d
+predict:
+  mode:
+    value: sample
+  pred_dir:
+    value: "./output/predictions"
+log_dir:
+  value: "./output/train-sdo-ml"