Rename variables and deps update

.github/workflows/smoke_test.yml

@@ -25,7 +25,7 @@ jobs:
           make kube
           make install
           kubectl get pod -n rococo
-          kubectl wait --for=condition=Ready pods --all -n rococo --timeout=900s
+          kubectl wait --for=condition=Ready pods --all -n rococo --timeout=1200s
           kubectl get pod -n rococo
 
       - name: Tests

.github/workflows/test.yml

@@ -17,11 +17,13 @@ jobs:
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.3.1
+          version: 1.5.1
           virtualenvs-create: true
           virtualenvs-in-project: true
       - name: Run tests
         run : |
+          # PyYaml is required until docker-compose is migrated to v2
+          pip install pyyaml==5.3.1
           pip install docker-compose
           poetry install --no-interaction --no-ansi
           docker-compose version

README.md

@@ -36,13 +36,12 @@ Set environment variables, eg:
 
     NAMESPACE=rococo
     HEALTHY_MIN_PEER_COUNT="1"
-    WS_ENDPOINT: "wss://rococo-rpc.polkadot.io"
-    NODE_HTTP_PATTERN: "http://NODE_NAME.rococo:9933"
-    NODE_WS_PATTERN: "ws://NODE_NAME.rococo:9944"
+    RELAY_CHAIN_RPC_URL: "wss://rococo-rpc.polkadot.io"
+    RPC_NODE_URL_PATTERN: "NODE_NAME.rococo:9944"
     HEALTHY_MIN_PEER_COUNT: "1"
     SUDO_SEED=***
-    VALIDATORS_ROOT_SEED=***
-    TESTNET_MANAGER_CONSENSUS=poa
+    DERIVATION_ROOT_SEED=***
+    RELAY_CHAIN_CONSENSUS=poa
 
 Start the app:
 

app/config/network_configuration.py

@@ -28,35 +28,50 @@ def get_node_logs_link():
         return node_logs_link
 
 
-def network_ws_endpoint():
-    return get_var_from_env('WS_ENDPOINT')
+def get_relay_chain_rpc_url():
+    if environ.get('WS_ENDPOINT'):
+        log.warning('WS_ENDPOINT will be deprecated. Please update to RELAY_CHAIN_RPC_URL.')
+        return get_var_from_env('WS_ENDPOINT')
+    else:
+        return get_var_from_env('RELAY_CHAIN_RPC_URL')
 
 
+# Retain HTTP endpoint until HTTP RPC is completely removed
 def node_http_endpoint(node_name):
-    node_http_pattern = get_var_from_env('NODE_HTTP_PATTERN')
-    return node_http_pattern.replace("NODE_NAME", node_name)
-
+    if environ.get('NODE_HTTP_PATTERN'):
+        log.warning('NODE_HTTP_PATTERN will be deprecated. Please update to RPC_NODE_URL_PATTERN.')
+        node_http_pattern = get_var_from_env('NODE_HTTP_PATTERN')
+        return node_http_pattern.replace("NODE_NAME", node_name)
+    else:
+        node_http_pattern = get_var_from_env('RPC_NODE_URL_PATTERN')
+        return f'ws://{node_http_pattern.replace("NODE_NAME", node_name)}'
 
+# Retain WS endpoint until HTTP RPC is completely removed
 def node_ws_endpoint(node_name):
-    node_ws_pattern = get_var_from_env('NODE_WS_PATTERN')
-    return node_ws_pattern.replace("NODE_NAME", node_name)
+    if environ.get('NODE_WS_PATTERN'):
+        log.warning('NODE_WS_PATTERN will be deprecated. Please update to RPC_NODE_URL_PATTERN.')
+        node_ws_pattern = get_var_from_env('NODE_WS_PATTERN')
+        return node_ws_pattern.replace("NODE_NAME", node_name)
+    else:
+        node_ws_pattern = get_var_from_env('RPC_NODE_URL_PATTERN')
+        return f'ws://{node_ws_pattern.replace("NODE_NAME", node_name)}'
 
 
-def network_root_seed():
-    return get_var_from_env('VALIDATORS_ROOT_SEED')
+def derivation_root_seed():
+    return get_var_from_env('DERIVATION_ROOT_SEED')
 
 
 def network_sudo_seed():
     return get_var_from_env('SUDO_SEED')
 
 
-def get_network_ss58_format():
-    network_ss58_format = get_var_from_env('NETWORK_SS58_FORMAT')
-    if network_ss58_format is None:
+def get_relay_chain_ss58_format():
+    relay_chain_ss58_format = get_var_from_env('RELAY_CHAIN_SS58_FORMAT')
+    if relay_chain_ss58_format is None:
         # If unset set, set to Rococo value
         return '42'
     else:
-        return network_ss58_format
+        return relay_chain_ss58_format
 
 
 def network_healthy_min_peer_count():
@@ -67,9 +82,9 @@ def network_tasks_cron_schedule():
     return get_var_from_env('TASKS_CRON_SCHEDULE')
 
 
-def network_consensus():
+def relay_chain_consensus():
     # poa or pos (default id poa )
-    value = get_var_from_env('TESTNET_MANAGER_CONSENSUS')
+    value = get_var_from_env('RELAY_CHAIN_CONSENSUS')
     if value is None:
         value = "poa"
     return value.lower()

app/lib/collator_account.py

@@ -4,13 +4,13 @@
 from substrateinterface import Keypair, KeypairType
 from hashlib import blake2b
 
-from app.config.network_configuration import network_root_seed
+from app.config.network_configuration import derivation_root_seed
 from app.lib.substrate import get_node_client
 
 log = logging.getLogger(__name__)
 
 def get_derived_collator_seed(node_name):
-    root_seed = network_root_seed()
+    root_seed = derivation_root_seed()
     return root_seed + "//collator//" + node_name
 
 
@@ -32,7 +32,9 @@ def get_derived_collator_session_keys(node_name):
         }
     except Exception as e:
         log.error("Unable to get_derived_collator_session_keys. Error: {}, stacktrace:\n".format(e, traceback.print_exc()))
-        return None
+        return {
+            'aura': None
+        }
 
 
 def get_moon_node_collator_uri(root_seed, node_name):
@@ -52,6 +54,6 @@ def get_moon_keypair_from_uri(uri):
 
 
 def get_derived_moon_collator_account(node_name):
-    key_seed = network_root_seed()
+    key_seed = derivation_root_seed()
     keypair = get_moon_keypair_from_uri(get_moon_node_collator_uri(key_seed, node_name))
     return keypair.ss58_address

app/lib/collator_manager.py

@@ -4,7 +4,7 @@
 
 from app.lib.collator_account import get_moon_keypair_from_uri
 from app.lib.collator_moonbeam import get_moon_node_collator_uri, register_moon_collator
-from app.config.network_configuration import network_root_seed
+from app.config.network_configuration import derivation_root_seed
 from app.lib.collator_tick import register_tick_collator
 from app.lib.collator_mint import register_mint_collator, deregister_mint_collator
 from app.lib.kubernetes_client import list_collator_pods
@@ -14,13 +14,13 @@
 
 
 def get_derived_collator_account(node_name, ss58_format):
-    key_seed = network_root_seed()
+    key_seed = derivation_root_seed()
     keypair = Keypair.create_from_uri(key_seed + "//collator//" + node_name, ss58_format=int(ss58_format))
     return keypair.ss58_address
 
 
 def get_derived_moon_collator_account(node_name):
-    key_seed = network_root_seed()
+    key_seed = derivation_root_seed()
     keypair = get_moon_keypair_from_uri(get_moon_node_collator_uri(key_seed, node_name))
     return keypair.ss58_address
 

app/lib/collator_mint.py

@@ -4,7 +4,7 @@
 
 from substrateinterface import Keypair
 
-from app.config.network_configuration import network_root_seed, get_network_ss58_format, \
+from app.config.network_configuration import derivation_root_seed, get_relay_chain_ss58_format, \
     network_sudo_seed
 from app.lib.balance_utils import transfer_funds, teleport_funds
 from app.lib.collator_account import get_derived_collator_keypair, get_derived_collator_seed, get_derived_collator_session_keys
@@ -22,7 +22,7 @@ def register_mint_collator(node_name, ss58_format, rotate_key=False):
     try:
         # 1. Generating stash account keypair
         node_client = get_node_client(node_name)
-        keypair_rich = Keypair.create_from_uri(network_root_seed(), ss58_format=int(ss58_format))
+        keypair_rich = Keypair.create_from_uri(derivation_root_seed(), ss58_format=int(ss58_format))
         collator_seed = get_derived_collator_seed(node_name)
         keypair = get_derived_collator_keypair(node_name, ss58_format)
         candidates = node_client.query('CollatorSelection', 'Candidates').value
@@ -118,7 +118,7 @@ def collator_set_keys(node_name, para_id, ss58_format):
             sudo_keypair = Keypair.create_from_seed(network_sudo_seed())
             log.info(f"Funding {collator_account_address}[ss58format={ss58_format}](funds={collator_account_funds}) via Teleport from relay-chain")
             # Get corresponding collator account address on the relay-chain (with the relay-chain ss58 format)
-            relay_chain_collator_account = get_derived_collator_keypair(node_name, get_network_ss58_format()).ss58_address
+            relay_chain_collator_account = get_derived_collator_keypair(node_name, get_relay_chain_ss58_format()).ss58_address
             teleport_result = teleport_funds(relay_chain_client, sudo_keypair, para_id, [relay_chain_collator_account], 1)
             if not teleport_result:
                 log.error("Unable fund account: {}, node: {}".format(

app/lib/collator_moonbeam.py

@@ -6,7 +6,7 @@
 from app.lib.substrate import substrate_call
 from app.lib.balance_utils import transfer_funds
 from app.lib.substrate import get_node_client
-from app.config.network_configuration import network_root_seed
+from app.config.network_configuration import derivation_root_seed
 
 log = logging.getLogger('collator_moonbeam')
 
@@ -15,7 +15,7 @@ def register_moon_collator(node_name, rotate_key=False):
     try:
         # init
         node_client = get_node_client(node_name)
-        collator_root_seed = network_root_seed()
+        collator_root_seed = derivation_root_seed()
         rich_key_uri = get_moon_root_uri(collator_root_seed)
         collator_key_uri = get_moon_node_collator_uri(collator_root_seed, node_name)
         keypair = Keypair.create_from_uri(collator_key_uri, crypto_type=KeypairType.ECDSA)

app/lib/network_utils.py

@@ -2,8 +2,8 @@
 import logging
 from datetime import datetime, timedelta
 
-from app.config.network_configuration import network_ws_endpoint, network_sudo_seed, network_root_seed, \
-    node_http_endpoint, get_network, network_consensus, node_ws_endpoint
+from app.config.network_configuration import get_relay_chain_rpc_url, network_sudo_seed, derivation_root_seed, \
+    node_http_endpoint, get_network, relay_chain_consensus, node_ws_endpoint
 from app.lib.balance_utils import fund_accounts
 from app.lib.collator_account import get_derived_moon_collator_account, get_derived_collator_account, \
     get_derived_collator_session_keys
@@ -40,7 +40,7 @@ def get_validator_account_from_pod(pod):
     if validator_account:
         return validator_account
     else:
-        return get_derived_node_stash_account_address(network_root_seed(), node_name)
+        return get_derived_node_stash_account_address(derivation_root_seed(), node_name)
 
 
 def get_collator_account_from_pod(pod):
@@ -70,7 +70,7 @@ def get_validator_status(node_stash_account_address, validator_set, validators_t
 
 
 def list_validators(stateful_set_name=''):
-    ws_endpoint = network_ws_endpoint()
+    ws_endpoint = get_relay_chain_rpc_url()
     external_validators = get_external_validators_from_configmap()
     validator_pods = list_validator_pods(stateful_set_name)
     validator_set = get_validator_set(ws_endpoint)
@@ -203,7 +203,7 @@ def get_substrate_node(node_name):
     node_info = get_node_info_from_pod(pod)
     node_info.update(get_node_info_from_rpc(node_info.get("name")))
     if node_info.get("role") == "authority":
-        ws_endpoint = network_ws_endpoint()
+        ws_endpoint = get_relay_chain_rpc_url()
         validator_set = get_validator_set(ws_endpoint)
         validators_to_add = get_validators_pending_addition(ws_endpoint)
         validators_to_retire = get_validators_pending_deletion(ws_endpoint)
@@ -251,8 +251,8 @@ def get_substrate_node(node_name):
 # Setup validator node (rotate + submit session key): returns stash address/empty string depending on success
 async def setup_validators_session_keys(node_name):
     log.info("Setting up validator session key for {}".format(node_name))
-    ws_endpoint = network_ws_endpoint()
-    validators_root_seed = network_root_seed()
+    ws_endpoint = get_relay_chain_rpc_url()
+    validators_root_seed = derivation_root_seed()
 
     # Rotate session keys
     node_endpoint = node_http_endpoint(node_name)
@@ -289,24 +289,24 @@ def is_validator_address_already_registered(address, validator_set, validators_t
 
 def register_validator_addresses(validator_addresses_to_register):
     log.info(f'registering the following validators addresses: {validator_addresses_to_register}')
-    ws_endpoint = network_ws_endpoint()
+    ws_endpoint = get_relay_chain_rpc_url()
     sudo_seed = network_sudo_seed()
     register_validators(ws_endpoint, sudo_seed, validator_addresses_to_register)
 
 
 async def register_validator_pods(pods):
     log.info(f'registering validators pods')
 
-    ws_endpoint = network_ws_endpoint()
+    ws_endpoint = get_relay_chain_rpc_url()
     substrate_client = get_relay_chain_client()
     sudo_seed = network_sudo_seed()
     validator_set = get_validator_set(ws_endpoint)
     validators_to_add = get_validators_pending_addition(ws_endpoint)
     validators_to_retire = get_validators_pending_deletion(ws_endpoint)
     node_stash_accounts = []
     nodes_to_register = []
-    consensus = network_consensus()
-    validators_root_seed = network_root_seed()
+    consensus = relay_chain_consensus()
+    validators_root_seed = derivation_root_seed()
 
     for pod in pods:
         node = pod.metadata.name
@@ -358,7 +358,7 @@ async def register_validator_nodes(nodes):
 
 
 async def deregister_validator_addresses(validator_addresses_to_deregister):
-    ws_endpoint = network_ws_endpoint()
+    ws_endpoint = get_relay_chain_rpc_url()
     sudo_seed = network_sudo_seed()
     log.info(
         f'removing {len(validator_addresses_to_deregister)} addresses from the validator set: {validator_addresses_to_deregister}')
@@ -367,11 +367,11 @@ async def deregister_validator_addresses(validator_addresses_to_deregister):
 
 async def deregister_validator_pods(pods):
     log.info(f'deregistering validators pods on {relay_chain_network_name}')
-    ws_endpoint = network_ws_endpoint()
+    ws_endpoint = get_relay_chain_rpc_url()
     validator_set = get_validator_set(ws_endpoint)
     validators_to_add = get_validators_pending_addition(ws_endpoint)
     validators_to_retire = get_validators_pending_deletion(ws_endpoint)
-    consensus = network_consensus()
+    consensus = relay_chain_consensus()
 
     accounts_to_deregister = []
     pods_to_deregister = []
@@ -396,7 +396,7 @@ async def deregister_validator_pods(pods):
     if pods_to_deregister and consensus == "pos":
         log.info(f'The following validators will be deregister: {pods_to_deregister}')
         for pod_name in pods_to_deregister:
-            validator_stash_mnemonic = get_node_stash_account_mnemonic(network_root_seed(), pod_name)
+            validator_stash_mnemonic = get_node_stash_account_mnemonic(derivation_root_seed(), pod_name)
             staking_chill(ws_endpoint, validator_stash_mnemonic)
 
 

app/lib/staking_utils.py

@@ -2,7 +2,7 @@
 
 from substrateinterface import Keypair
 
-from app.config.network_configuration import network_root_seed
+from app.config.network_configuration import derivation_root_seed
 from app.lib.balance_utils import transfer_funds
 from app.lib.kubernetes_client import get_pod
 from app.lib.network_utils import get_validator_account_from_pod
@@ -29,7 +29,7 @@ async def create_nominators_for_validator_node(substrate_client, funding_account
 
 
 def get_validator_nominator_mnemonic(validator_name, nominator_index):
-    return f'{network_root_seed()}//{validator_name}//{nominator_index}'
+    return f'{derivation_root_seed()}//{validator_name}//{nominator_index}'
 
 
 def check_is_nominator(substrate_client, nominator_keypair):

app/lib/substrate.py

@@ -3,7 +3,7 @@
 from substrateinterface import SubstrateInterface, Keypair
 from substrateinterface.utils.hasher import blake2_256
 
-from app.config.network_configuration import network_ws_endpoint, node_ws_endpoint, network_sudo_seed
+from app.config.network_configuration import get_relay_chain_rpc_url, node_ws_endpoint, network_sudo_seed
 from app.config import ws_pool
 
 log = logging.getLogger(__name__)
@@ -38,7 +38,7 @@ def get_substrate_client(url):
 
 
 def get_relay_chain_client():
-    url = network_ws_endpoint()
+    url = get_relay_chain_rpc_url()
     return get_substrate_client(url)