Merge pull request #53 from i4Ds/bugfix/improvements

several improvements and fixes
i4Ds · Jul 10, 2022 · 510c7e9 · 510c7e9
2 parents d054c55 + 94927f9
commit 510c7e9
Show file tree

Hide file tree

Showing 8 changed files with 862 additions and 14 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -10,7 +10,7 @@
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
-      "justMyCode": true
+      "justMyCode": false
     },
     {
       "name": "Python: SDO CLI get events",

diff --git a/config/ce-vae/debug.yaml b/config/ce-vae/debug.yaml
@@ -0,0 +1,64 @@
+model:
+  load_path:
+    value: null
+data:
+  data_dir:
+    #value: /mnt/nas05/astrodata01/astroml_data/sdomlv2_full/sdomlv2.zarr
+    value: /home/marius/data/sdomlv2_full/sdomlv2.zarr
+  dataset:
+    value: SDOMLDatasetV2
+  num_data_loader_workers:
+    value: 16
+  prefetch_factor:
+    value: 4
+  batch_size:
+    value: 32
+  sdo_ml_v2:
+    train_year:
+      value:
+        - "2012"
+    train_start_date:
+      value: null
+    train_end_date:
+      value: null
+    test_year:
+      value:
+        - "2011"
+    test_start_date:
+      value: null
+    test_end_date:
+      value: null
+    train_val_split_ratio:
+      value: 0.9
+    train_val_split_temporal_chunk_size:
+      value: 14d
+    sampling_strategy:
+      value: "chunk"
+    mask_limb:
+      value: false
+    reduce_memory:
+      value: true
+predict:
+  mode:
+    value: sample
+  pred_dir:
+    value: "./output/predictions"
+log_dir:
+  value: "./output/train-sdo-ml"
+train:
+  wandb_run_id:
+    value: null
+  n_epochs:
+    value: 20
+  early_stopping:
+    value: true
+  use_geco:
+    value: false
+  profile:
+    value: true
+  fast_dev_run:
+    value: 100
+devices:
+  gpus:
+    value:
+      - 3
diff --git a/config/ce-vae/run-fhnw-full-1-limb.yaml b/config/ce-vae/run-fhnw-full-1-limb.yaml
@@ -3,7 +3,8 @@ model:
     value: null
 data:
   data_dir:
-    value: /mnt/nas05/astrodata01/astroml_data/sdomlv2_full/sdomlv2.zarr
+    #value: /mnt/nas05/astrodata01/astroml_data/sdomlv2_full/sdomlv2.zarr
+    value: /home/marius/data/sdomlv2_full/sdomlv2.zarr
   dataset:
     value: SDOMLDatasetV2
   num_data_loader_workers:
@@ -44,6 +45,8 @@ data:
       value: "chunk"
     mask_limb:
       value: true
+    mask_limb_radius_scale_factor:
+      value: 0.9
 predict:
   mode:
     value: sample
@@ -52,6 +55,8 @@ predict:
 log_dir:
   value: "./output/train-sdo-ml"
 train:
+  wandb_run_id:
+    value: "35pi1yaj"
   n_epochs:
     value: 20
   early_stopping:
@@ -61,4 +66,4 @@ train:
 devices:
   gpus:
     value:
-      - 2
+      - 3
diff --git a/config/ce-vae/run-fhnw-full-1-predict.yaml b/config/ce-vae/run-fhnw-full-1-predict.yaml
@@ -0,0 +1,63 @@
+model:
+  load_path:
+    value: "./artifacts/model-vzsks4y3:v11/model.ckpt"
+data:
+  data_dir:
+    value: /mnt/nas05/astrodata01/astroml_data/sdomlv2_full/sdomlv2.zarr
+    #value: /home/marius/data/sdomlv2_full/sdomlv2.zarr
+  dataset:
+    value: SDOMLDatasetV2
+  num_data_loader_workers:
+    value: 16
+  prefetch_factor:
+    value: 4
+  batch_size:
+    value: 32
+  sdo_ml_v2:
+    train_year:
+      value:
+        - "2010"
+        - "2012"
+        - "2013"
+        - "2015"
+        - "2017"
+        - "2018"
+        - "2020"
+    train_start_date:
+      value: null
+    train_end_date:
+      value: null
+    test_year:
+      value:
+        - "2011"
+        - "2014"
+        - "2016"
+        - "2019"
+    test_start_date:
+      value: null
+    test_end_date:
+      value: null
+    train_val_split_ratio:
+      value: 0.9
+    train_val_split_temporal_chunk_size:
+      value: 14d
+    sampling_strategy:
+      value: "chunk"
+predict:
+  mode:
+    value: sample
+  pred_dir:
+    value: "./output/predictions"
+log_dir:
+  value: "./output/train-sdo-ml"
+train:
+  n_epochs:
+    value: 20
+  early_stopping:
+    value: true
+  use_geco:
+    value: true
+devices:
+  gpus:
+    value:
+      - 2
diff --git a/notebooks/SDOMLv2Dataset-FHNW-DataExploration.ipynb b/notebooks/SDOMLv2Dataset-FHNW-DataExploration.ipynb
@@ -2,6 +2,7 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "f3259e23",
    "metadata": {},
    "source": [
     "## Data Exploration of the SDO ML v2 Dataset\n"
@@ -10,6 +11,7 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "a8efc40a",
    "metadata": {},
    "outputs": [
     {
@@ -28,6 +30,7 @@
   {
    "cell_type": "code",
    "execution_count": 2,
+   "id": "fabf4e0f",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -46,6 +49,7 @@
   {
    "cell_type": "code",
    "execution_count": 3,
+   "id": "2ed728c2",
    "metadata": {},
    "outputs": [
     {
@@ -213,6 +217,7 @@
   {
    "cell_type": "code",
    "execution_count": 4,
+   "id": "ca1d6e1b",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -225,6 +230,7 @@
   {
    "cell_type": "code",
    "execution_count": 5,
+   "id": "812d0635",
    "metadata": {},
    "outputs": [
     {
@@ -245,6 +251,7 @@
   {
    "cell_type": "code",
    "execution_count": 6,
+   "id": "bae04857",
    "metadata": {},
    "outputs": [
     {
@@ -2715,6 +2722,7 @@
   {
    "cell_type": "code",
    "execution_count": 7,
+   "id": "8894c410",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -2729,6 +2737,7 @@
   {
    "cell_type": "code",
    "execution_count": 8,
+   "id": "35ee1a64",
    "metadata": {},
    "outputs": [
     {
@@ -2842,6 +2851,7 @@
   {
    "cell_type": "code",
    "execution_count": 9,
+   "id": "9ec7b869",
    "metadata": {},
    "outputs": [
     {
@@ -2965,6 +2975,7 @@
   {
    "cell_type": "code",
    "execution_count": 10,
+   "id": "d16e3d6d",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -2974,6 +2985,7 @@
   {
    "cell_type": "code",
    "execution_count": 13,
+   "id": "d7ac0325",
    "metadata": {},
    "outputs": [
     {
@@ -3010,6 +3022,7 @@
   {
    "cell_type": "code",
    "execution_count": 14,
+   "id": "41043093",
    "metadata": {},
    "outputs": [
     {
@@ -3046,6 +3059,7 @@
   {
    "cell_type": "code",
    "execution_count": 15,
+   "id": "463f7c9d",
    "metadata": {},
    "outputs": [
     {
@@ -3082,6 +3096,7 @@
   {
    "cell_type": "code",
    "execution_count": 16,
+   "id": "c44e136c",
    "metadata": {},
    "outputs": [
     {
@@ -3118,6 +3133,7 @@
   {
    "cell_type": "code",
    "execution_count": 17,
+   "id": "2fe2850a",
    "metadata": {},
    "outputs": [
     {
@@ -3154,6 +3170,7 @@
   {
    "cell_type": "code",
    "execution_count": 11,
+   "id": "a78a2ff1",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -3163,6 +3180,7 @@
   {
    "cell_type": "code",
    "execution_count": 18,
+   "id": "5da6eab7",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -3194,6 +3212,7 @@
   {
    "cell_type": "code",
    "execution_count": 19,
+   "id": "4c3a4e53",
    "metadata": {},
    "outputs": [
     {
@@ -3373,6 +3392,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "113ee3d3",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -3383,6 +3403,7 @@
   {
    "cell_type": "code",
    "execution_count": 20,
+   "id": "13054847",
    "metadata": {},
    "outputs": [
     {
@@ -3487,6 +3508,7 @@
   {
    "cell_type": "code",
    "execution_count": 21,
+   "id": "2a891f30",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -3496,6 +3518,7 @@
   {
    "cell_type": "code",
    "execution_count": 22,
+   "id": "20b8d4ad",
    "metadata": {},
    "outputs": [
     {
@@ -3532,6 +3555,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "2720a3da",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -3553,7 +3577,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.0"
+   "version": "3.9.13"
   }
  },
  "nbformat": 4,