feat(substrate): enable deployment with proxy and vault via helm

platforms/shared/charts/bevel-scripts/scripts/bevel-vault.sh

@@ -37,7 +37,7 @@ initHashicorpVaultToken() {
     # Extract error message (if any) from the response using jq
     ERROR=$(echo "$RESPONSE" | jq -r '.errors[0]')
     # Extract the Vault secret data from the response using jq
-    VAULT_TOKEN=$(echo "$RESPONSE" | jq -r '.auth.client_token')
+    export VAULT_TOKEN=$(echo "$RESPONSE" | jq -r '.auth.client_token')
 
     # Check if the Vault token is empty, null, or contains errors
     if [ -z "$VAULT_TOKEN" ] || [ "$VAULT_TOKEN" = "null" ] || echo "$VAULT_TOKEN" | grep -q "errors"; then

platforms/shared/charts/bevel-vault-mgmt/templates/configmap.yaml

@@ -15,7 +15,6 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
-    {{- include "labels.custom" . | nindent 2 }}
 data:
   policies-config.json: |-
     {{ if eq .Values.global.vault.network "besu" -}}

platforms/shared/charts/bevel-vault-mgmt/templates/job.yaml

@@ -17,7 +17,6 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
-    {{- include "labels.custom" . | nindent 2 }}
 spec:
   backoffLimit: 6
   template:

platforms/substrate/charts/README.md

@@ -43,35 +43,17 @@ global:
 
 ## `Without Proxy and Vault`
 
-### 1. Install Genesis Node
+### 1. Install Genesis 
 ```bash
 # Install the genesis node
 helm install genesis ./substrate-genesis --namespace supplychain-subs --create-namespace --values ./values/noproxy-and-novault/genesis.yaml
 ```
 
-### 2. Install Bootnode
+### 2. Install Nodes
 ```bash
 # Install bootnode
 helm install validator-1 ./substrate-node --namespace supplychain-subs --values ./values/noproxy-and-novault/node.yaml --set node.isBootnode.enabled=false
-```
-
-### 3. Install Additional Nodes
 
-To deploy additional nodes, update the following section in the `./values/noproxy-and-novault/node.yaml` file only once:
-```yaml
-...
-node:
-  ...
-  isBootnode:
-    enabled: true
-    bootnodeName: <bootnode-name> # Here it'll be "validator-1" as defined above
-    bootnodeAddr: <bootnode-name>-substrate-node-0-rc-p2p.<bootnode-namespace> # Supporting no-proxy as of now. TODO: enable proxy method.
-    bootnodePort: 30333
-  ...
-...
-```
-Then install the nodes using the following commands:
-```bash
 helm install validator-2 ./substrate-node --namespace supplychain-subs --values ./values/noproxy-and-novault/node.yaml
 
 helm install validator-3 ./substrate-node --namespace supplychain-subs --values ./values/noproxy-and-novault/node.yaml
@@ -82,28 +64,56 @@ helm install    member-1 ./substrate-node --namespace supplychain-subs --values
 ```
 ## 4. Install IPFS Nodes
 
-**4.1.** Update the following section in the `./values/noproxy-and-novault/ipfs.yaml` file only once:
+**4.1.** Retrieve the `NODE_ID` from the Kubernetes secret:
 
-```yaml
-config:
-  # Specify the name of any running member's node that can be considered as a bootnode for the current IPFS node.
-  nodeHost: <member-node>-substrate-node # Here, it can be modified either as member-1-substrate-node or member-2-substrate-node
+```bash
+NODE_ID=$(kubectl get secret "substrate-node-member-1-keys" --namespace supplychain-subs -o jsonpath="{.data['substrate-node-keys']}" | base64 -d | jq -r '.data.node_id')
 ```
 
-**4.2.** Retrieve the `NODE_ID` from the Kubernetes secret:
+**4.2.** Now, install the IPFS nodes:
 
 ```bash
-NODE_ID=$(kubectl get secret "substrate-node-<member-node>-keys" --namespace supplychain-subs -o jsonpath="{.data['substrate-node-keys']}" | base64 -d | jq -r '.data.node_id')
+helm install dscp-ipfs-node-1 ./dscp-ipfs-node --namespace supplychain-subs --values ./values/noproxy-and-novault/ipfs.yaml \
+--set config.ipfsBootNodeAddress="/dns4/dscp-ipfs-node-1-swarm.supplychain-subs/tcp/4001/p2p/$NODE_ID"
 ```
 
-**4.3.** Now, install the IPFS nodes:
+### _With Ambassador proxy and Vault_
+
+### 1. Install Genesis 
+
+Replace the `global.vault.address`, `global.cluster.kubernetesUrl` and `global.proxy.externalUrlSuffix` in all the files in `./values/proxy-and-vault/` folder.
 
 ```bash
-helm install dscp-ipfs-node-1 ./dscp-ipfs-node --namespace supplychain-subs --values ./values/noproxy-and-novault/ipfs.yaml \
---set config.ipfsBootNodeAddress="/dns4/dscp-ipfs-node-1-swarm.supplychain-subs/tcp/4001/p2p/$NODE_ID"
+# If the namespace does not exist already
+kubectl create namespace supplychain-subs 
+# Create the roottoken secret
+kubectl -n supplychain-subs create secret generic roottoken --from-literal=token=<VAULT_ROOT_TOKEN>
+
+helm install genesis ./substrate-genesis --namespace supplychain-subs --values ./values/proxy-and-vault/genesis.yaml
+```
+### 2. Install Nodes
+```bash
+helm install validator-1 ./substrate-node --namespace supplychain-subs --values ./values/proxy-and-vault/validator.yaml --set global.proxy.p2p=15051
+
+helm install validator-2 ./substrate-node --namespace supplychain-subs --values ./values/proxy-and-vault/validator.yaml --set global.proxy.p2p=15052
+
+helm install validator-3 ./substrate-node --namespace supplychain-subs --values ./values/proxy-and-vault/validator.yaml --set global.proxy.p2p=15053
+
+helm install validator-4 ./substrate-node --namespace supplychain-subs --values ./values/proxy-and-vault/validator.yaml --set global.proxy.p2p=15054
+
+helm install    member-1 ./substrate-node --namespace supplychain-subs --values ./values/proxy-and-vault/node.yaml --set node.role=full
+
+```
+
+# Spin up a IPFS nodes
 
-helm install dscp-ipfs-node-2 ./dscp-ipfs-node --namespace supplychain-subs --values ./values/noproxy-and-novault/ipfs.yaml \
---set config.ipfsBootNodeAddress="/dns4/dscp-ipfs-node-2-swarm.supplychain-subs/tcp/4001/p2p/$NODE_ID"
+```bash
+NODE_ID=$(kubectl get secret "substrate-node-member-1-keys" --namespace supplychain-subs -o jsonpath="{.data['substrate-node-keys']}" | base64 -d | jq -r '.data.node_id')
+```
+
+```bash
+helm install dscp-ipfs-node-1 ./dscp-ipfs-node --namespace supplychain-subs --values ./values/proxy-and-vault/ipfs.yaml \
+--set config.ipfsBootNodeAddress="/dns4/dscp-ipfs-node-1-swarm.supplychain-subs/tcp/4001/p2p/$NODE_ID"
 ```
 
 ## Clean-up
@@ -116,6 +126,5 @@ helm uninstall validator-3      --namespace supplychain-subs
 helm uninstall validator-4      --namespace supplychain-subs
 helm uninstall member-1         --namespace supplychain-subs
 helm uninstall dscp-ipfs-node-1 --namespace supplychain-subs
-helm uninstall dscp-ipfs-node-2 --namespace supplychain-subs
 helm uninstall genesis          --namespace supplychain-subs
 ```

platforms/substrate/charts/dscp-ipfs-node/templates/statefulset.yaml

@@ -98,11 +98,13 @@ spec:
             - name: VAULT_ADDR
               value: {{ $.Values.global.vault.address }}
             - name: KUBERNETES_AUTH_PATH
-              value: {{ $.Values.global.vault.authpath }}
+              value: {{ $.Values.global.vault.authPath }}
             - name: VAULT_APP_ROLE
               value: {{ $.Values.global.vault.role }}
-            - name: CERTS_SECRET_PREFIX
-              value: {{ .Values.global.vault.certsecretprefix }}
+            - name: VAULT_SECRET_ENGINE
+              value: {{ .Values.global.vault.secretEngine }}
+            - name: VAULT_SECRET_PREFIX
+              value: {{ .Values.global.vault.secretPrefix }}
 {{- end }}
           volumeMounts:
             - mountPath: {{ .Values.config.ipfsDataPath }}
@@ -167,7 +169,7 @@ spec:
                 jq -r 'if .errors then . else .auth.client_token end')
 
               validateVaultResponse 'vault login token' "${VAULT_CLIENT_TOKEN}"
-              vault_secret_key="${CERTS_SECRET_PREFIX}/ipfs"
+              vault_secret_key="${VAULT_SECRET_ENGINE}"/"${VAULT_SECRET_PREFIX}/ipfs"
               # Save the generated keys to VAULT
               LOOKUP_SECRET_RESPONSE=$(curl -sS -H "X-Vault-Token: ${VAULT_CLIENT_TOKEN}" \
                 -H "Content-Type: application/json" \

platforms/substrate/charts/dscp-ipfs-node/values.yaml

@@ -39,7 +39,7 @@ config:
   healthCheckPollPeriod: 30000
   healthCheckTimeout: 2000
   # External DSCP-Node hostname to query, this overrides dscpNode.enabled
-  nodeHost: ""
+  nodeHost: member-1-substrate-node
   # External DSCP-Node port to query
   nodePort: 9944
   # Public key for the IPFS subsystem

platforms/substrate/charts/substrate-genesis/templates/generate-keys.yaml

@@ -48,12 +48,16 @@ spec:
           env:
           - name: VAULT_ADDR
             value: {{ $.Values.global.vault.address }}
+          - name: VAULT_SECRET_ENGINE
+            value: {{ .Values.global.vault.secretEngine }}
           - name: KUBERNETES_AUTH_PATH
             value: {{ $.Values.global.vault.authPath }}
           - name: VAULT_APP_ROLE
             value: {{ $.Values.global.vault.role }}
-          - name: CERTS_SECRET_PREFIX
+          - name: VAULT_SECRET_PREFIX
             value: {{ .Values.global.vault.secretPrefix }}
+          - name: VAULT_TYPE
+            value: {{ $.Values.global.vault.type }}  
 {{- end }} 
           command: ["bash", "-c"]
           args:
@@ -83,6 +87,60 @@ spec:
                 chmod +x kubectl
                 mv kubectl /usr/local/bin/
                 kubectl version --client
+{{- if eq .Values.global.vault.type "hashicorp" }}
+                # Source the script containing vault-related functions
+                . /scripts/bevel-vault.sh
+
+                echo "Generate a customize token."
+                vaultBevelFunc "init"
+
+                # Function to safely write keys
+                safeWriteSecret() {
+                  local key="$1"
+                  local payload_json="$2"
+
+                  # Read secret from vault
+                  vaultBevelFunc "readJson" "${VAULT_SECRET_ENGINE}/${VAULT_SECRET_PREFIX}/${key}"
+                  # Check if secrets are available in the vault
+                  if [ "$SECRETS_AVAILABLE" == "yes" ]
+                    then
+                      # Extract secrets from JSON response
+                      local node_id=$(echo ${VAULT_SECRET} | jq -r '.["node_id"]')
+                      local node_key=$(echo ${VAULT_SECRET} | jq -r '.["node_key"]')
+                      local aura_seed=$(echo ${VAULT_SECRET} | jq -r '.["aura_seed"]')
+                      local aura_addr=$(echo ${VAULT_SECRET} | jq -r '.["aura_addr"]')
+                      local grandpa_seed=$(echo ${VAULT_SECRET} | jq -r '.["grandpa_seed"]')
+                      local grandpa_addr=$(echo ${VAULT_SECRET} | jq -r '.["grandpa_addr"]')
+                      local aura_file_b64=$(echo ${VAULT_SECRET} | jq -r '.["aura_file_b64"]' | base64 -d)
+                      local grandpa_file_b64=$(echo ${VAULT_SECRET} | jq -r '.["grandpa_file_b64"]' | base64 -d)
+
+                      # Check if Kubernetes secret exists, if not, create one
+                      if ! kubectl get secret substrate-node-validator-${i}-keys --namespace {{ .Release.Namespace }} &> /dev/null; then                      
+                        kubectl create secret generic substrate-node-validator-${i}-keys --namespace {{ .Release.Namespace }} \
+                        --from-literal=node_id=${node_id} \
+                        --from-literal=node_key=${node_key} \
+                        --from-literal=aura_seed=${aura_seed} \
+                        --from-literal=aura_addr=${aura_addr} \
+                        --from-literal=grandpa_seed=${grandpa_seed} \
+                        --from-literal=grandpa_addr=${grandpa_addr} \
+                        --from-literal=aura_file_b64=${aura_file_b64}
+                        --from-literal=grandpa_file_b64=${grandpa_file_b64}
+                      fi
+                  else
+                    # Push data to vault
+                    vaultBevelFunc 'write' "${VAULT_SECRET_ENGINE}/${VAULT_SECRET_PREFIX}/${key}" "${payload_json}" #${key}" 
+                    #rm nodePayload.json
+                  fi
+                }
+{{- else }}
+                safeWriteSecret() {
+                  # Placeholder:
+                  # - Implement code to fetch the keys if using any cloud-native service or platform different from HashiCorp to store the keys
+                  # - After fetching the keys, create Kubernetes secrets from them
+                  # - For guidance, refer to the code written for HashiCorp Vault for the same purpose
+                  return 0
+                }
+{{- end }}
 
                 echo "Step 2: Define functions to generate keys."
                 generate_key() {
@@ -155,13 +213,21 @@ spec:
                   command={{ $.Values.node.command }}
                   # Generate keys for member nodes
                   for ((i=1; i<= {{ $.Values.node.member.count }}; i++)); do
-                    echo "Generating nodekey for member-$i"
-                    generate_nodekey "member" "$i"
+                      if [[ $.Values.global.vault.type != "hashicorp" ]]; then
+                          echo "Generating nodekey for member-$i"
+                          generate_nodekey "member" "$i"
+                      fi
+                      echo "Writing secrets for member-$i"
+                      safeWriteSecret "substrate-node-member-$i-keys" "finalJSON.json"
                   done
                   # Generate keys for validator nodes
                   for ((i=1; i<= {{ $.Values.node.validator.count }}; i++)); do
-                    echo "Generating nodekey for validator-$i"
-                    generate_nodekey "validator" "$i"
+                      if [[ $.Values.global.vault.type != "hashicorp" ]]; then
+                          echo "Generating nodekey for validator-$i"
+                          generate_nodekey "validator" "$i"
+                      fi
+                      echo "Writing secrets for validator-$i"
+                      safeWriteSecret "substrate-node-validator-$i-keys" "finalJSON.json"
                   done
                 }
                 main

platforms/substrate/charts/substrate-genesis/templates/genesis.yaml

@@ -59,16 +59,18 @@ spec:
         - name: VAULT_SECRET_PREFIX
           value: {{ .Values.global.vault.secretPrefix }}
         - name: VAULT_SECRET_ENGINE
-          value: "{{ .Values.global.vault.secretEngine }}"
+          value: {{ .Values.global.vault.secretEngine }}
+        - name: VAULT_TYPE
+          value: {{ .Values.global.vault.type }}
+        - name: CERTS_SECRET_PREFIX
+          value: {{ .Values.global.vault.certPrefix }}
 {{- end }}
         - name: MOUNT_PATH
           value: "certcheck"
         command: ["bash", "-c"]
         args:
         - |-
 
-          #!/usr/bin/env bash
-
           echo "Step 1: Check if the node image is available and install necessary packages if needed."
           {{- if ne $.Values.node.image "docker.io/paritytech/substrate-playground-template-node-template" }}
             # Install necessary packages using custom package manager script
@@ -123,7 +125,7 @@ spec:
 
           echo "Step 5: Insert AURA & GRANDPA keys into genesis for validators."
           echo "Inserting keys into genesis for validators"
-          for ((i=1; i<= {{ $.Values.node.validator.count }}; i++)); do
+          for ((i=1; i<={{ $.Values.node.validator.count }}; i++)); do
             secret_data=$(kubectl get secret "substrate-node-validator-${i}-keys" --namespace {{ .Release.Namespace }} -o json | jq -r '.data["substrate-node-keys"]' | base64 -d)
 
             # Check if secret_data is empty or not
@@ -153,7 +155,7 @@ spec:
           done <<< "$secret_names"
 
           echo "Step 6: Adding member accounts and their balances to genesis."
-          for ((i=1; i<= {{ $.Values.node.member.count }}; i++)); do
+          for ((i=1; i<={{ $.Values.node.member.count }}; i++)); do
             account_addr=$(kubectl get secret "substrate-node-member-$i-keys" --namespace {{ .Release.Namespace }} -o json | jq -r '.data["substrate-node-keys"]' | base64 -d | jq -r '.data.account_addr')
             # Add account address and balance into genesis block
             GENESIS=$(echo "$GENESIS" | jq --arg account_id "$account_addr" --arg balance {{ $.Values.node.member.balance }} '.genesis.runtime.balances.balances += [[$account_id, ($balance | tonumber)]]')
@@ -195,6 +197,29 @@ spec:
             kubectl create configmap "substrate-genesis" --namespace {{ .Release.Namespace }} --from-file=genesis="${MOUNT_PATH}/genesis_base64" --from-file=sudoKeygenOutput="${MOUNT_PATH}/sudoKeygenOutput.json"
           fi
           echo "COMPLETED!"
+{{- if eq .Values.global.vault.type "hashicorp" }}
+          # Initialize the token
+          . /scripts/bevel-vault.sh
+          vaultBevelFunc "init"
+          vault_secret_key="${VAULT_SECRET_ENGINE}/${VAULT_SECRET_PREFIX}/genesis"
+          vaultBevelFunc "readJson" "$vault_secret_key"
+
+          # The vault CLI is required for this job as the genesis file is too large to be passed in via a vault API call
+          echo "Installing Vault CLI"
+          curl -O -L https://releases.hashicorp.com/vault/1.7.1/vault_1.7.1_linux_amd64.zip
+          unzip vault_1.7.1_linux_amd64.zip
+          {{- if eq $.Values.node.image "docker.io/paritytech/substrate-playground-template-node-template" }}
+          export PATH=$PATH:~/workspace
+          {{- else }}
+          mv vault /bin
+          {{- end }}
+          vault --version
+          # Save the generated keys to VAULT
+          if [ "$SECRETS_AVAILABLE" == "no" ]; then
+            vault_secret_key="${VAULT_SECRET_ENGINE}/${CERTS_SECRET_PREFIX}/genesis"
+            vault kv put "$vault_secret_key" genesis=@"${MOUNT_PATH}/genesis_base64" sudo_details=@"${MOUNT_PATH}/sudoKeygenOutput.json"
+          fi
+{{- end }}
       volumes:
         - name: certcheck
           emptyDir:

platforms/substrate/charts/substrate-node/templates/service.yaml

@@ -107,7 +107,7 @@ apiVersion: getambassador.io/v3alpha1
 kind: Listener
 metadata:
   name: "{{ $fullname }}-listener"
-  namespace: {{ $.Values.namespace }}
+  namespace: {{ $.Release.Namespace }}
 spec:
   port: {{ $.Values.proxy.p2p }}
   protocol: TCP
@@ -121,9 +121,9 @@ apiVersion: getambassador.io/v3alpha1
 kind: TCPMapping
 metadata:
   name: "{{ $fullname }}-tcpmapping"
-  namespace: {{ $.Values.namespace }}
+  namespace: {{ $.Release.Namespace }}
 spec:
   port: {{ $.Values.proxy.p2p }}
-  service: "{{ $fullname }}-{{ $i }}-rc-p2p.{{ $.Values.namespace }}:{{ $.Values.node.ports.p2p }}"
+  service: "{{ $fullname }}-{{ $i }}-rc-p2p.{{ $.Release.Namespace }}:{{ $.Values.node.ports.p2p }}"
 {{- end }}
 {{- end }}

platforms/substrate/charts/substrate-node/templates/statefulset.yaml

@@ -263,8 +263,9 @@ spec:
                 EXTERNAL_ADDRESS={{ $fullname }}-${POD_INDEX}-rc-p2p.{{ .Release.Namespace }}.svc.cluster.local
                 EXTERNAL_P2P_PORT="{{ .Values.node.ports.p2p }}"
               {{- else if eq .Values.proxy.provider "ambassador" }}
-                EXTERNAL_ADDRESS="{{ .Values.proxy.external_url }}"
+                EXTERNAL_ADDRESS="{{ $fullname }}-${POD_INDEX}.{{ .Values.proxy.external_url }}"
                 EXTERNAL_P2P_PORT="{{ .Values.proxy.p2p }}"
+                echo "test-one"
               {{- end }}
 
               echo "PARA_CHAIN_P2P_PORT:$PARA_CHAIN_P2P_PORT"
@@ -477,4 +478,4 @@ spec:
         storageClassName: storage-{{ .Release.Name }}
         resources:
           requests:
-            storage: {{ .Values.node.dataVolumeSize }}
+            storage: {{ .Values.storage.size }}