Add support geth websockets rcp. (#28)

Add support for connecting to geth via websockets rpc. The default port is set to `8546` and will always be set, but in order to use websockets they need to be enabled in the `geth.Geth_Startup_Param` inside the config yaml, e.g. --ws --wsorigins=\"*\" --wsaddr 0.0.0.0 see examples/config/qubernetes-ingress-ws.yaml

To override the default set it in the config under `geth.Node_WSPort`.

note:
If trying to connect externally via an ingress be aware that depending on the backend k8s service and how the load balancing is implemented the health checks may fail, e.g. for GKE currently has a health check that uses http, and setting a configurable health check in k8s is not supported yet,
but will be available soon: kubernetes/ingress-gce#1067.

* Change RPCPort 7nodes from 8546->8545.

change the 7nodes examples configs RPCPort to the default geth rpc port
so that it doesn't clash with the default websocket port (8546).

7nodes/istanbul-7nodes-constellation/qubernetes-istanbul-constellation-7nodes-pvc.yaml

@@ -38,7 +38,7 @@ quorum:
     Capacity: 200Mi
 # generic geth related options
 geth:
-  Node_RPCPort: 8546
+  Node_RPCPort: 8545
   NodeP2P_ListenAddr: 21000
   network:
     # network id (1: mainnet, 3: ropsten, 4: rinkeby ... )

7nodes/istanbul-7nodes-tessera/qubernetes-istanbul-tessera-7nodes-pvc.yaml

@@ -38,7 +38,7 @@ quorum:
     Capacity: 200Mi
 # generic geth related options
 geth:
-  Node_RPCPort: 8546
+  Node_RPCPort: 8545
   NodeP2P_ListenAddr: 21000
   network:
     # network id (1: mainnet, 3: ropsten, 4: rinkeby ... )

7nodes/raft-7nodes-constellation/qubernetes-raft-constellation-7nodes-pvc.yaml

@@ -38,7 +38,7 @@ quorum:
     Capacity: 200Mi
 # generic geth related options
 geth:
-  Node_RPCPort: 8546
+  Node_RPCPort: 8545
   NodeP2P_ListenAddr: 21000
   network:
     # network id (1: mainnet, 3: ropsten, 4: rinkeby ... )

7nodes/raft-7nodes-tessera/qubernetes-raft-tessera-7nodes-pvc.yaml

@@ -38,7 +38,7 @@ quorum:
     Capacity: 200Mi
 # generic geth related options
 geth:
-  Node_RPCPort: 8546
+  Node_RPCPort: 8545
   NodeP2P_ListenAddr: 21000
   network:
     # network id (1: mainnet, 3: ropsten, 4: rinkeby ... )

examples/config/qubernetes-ingress-ws.yaml

@@ -0,0 +1,41 @@
+#namespace:
+# name: quorum-test
+# number of nodes to deploy
+sep_deployment_files: true
+nodes:
+  number: 4
+service:
+  # NodePort | ClusterIP | LoadBalancer
+  type: NodePort
+  Ingress:
+    # OneToMany | OneToOne
+    Strategy: OneToMany
+    Host: "quorum.testnet.com"
+quorum:
+  # supported: (raft | istanbul)
+  consensus: istanbul
+  # related to quorum containers
+  quorum:
+    # container images at https://hub.docker.com/u/quorumengineering/
+    Quorum_Version: 2.2.5
+  # related to transaction manager containers
+  tm:
+    # container images at https://hub.docker.com/u/quorumengineering/
+    # (tessera|constellation)
+    Name: tessera
+    Tm_Version: 0.11
+  # persistent storage is handled by Persistent Volume Claims (PVC) https://kubernetes.io/docs/concepts/storage/persistent-volumes/
+  # test locally and on GCP
+  # The data dir is persisted here
+  storage:
+    # PVC (Persistent_Volume_Claim - tested with GCP).
+    Type: PVC
+    ## when redeploying cannot be less than previous values
+    Capacity: 200Mi
+# generic geth related options
+geth:
+  Node_WSPort: 8546
+  # general verbosity of geth [1..5]
+  verbosity: 9
+  # additional startup params to pass into geth/quorum
+  Geth_Startup_Params: --rpccorsdomain=\"*\" --rpcvhosts=\"*\" --ws --wsorigins=\"*\" --wsaddr 0.0.0.0

qubernetes

@@ -66,6 +66,11 @@ if @config["geth"] and @config["geth"]["Node_RPCPort"]
   @Node_RPCPort = @config["geth"]["Node_RPCPort"]
 end
 
+@Node_WSPort = 8546
+if @config["geth"] and @config["geth"]["Node_WSPort"]
+  @Node_WSPort = @config["geth"]["Node_WSPort"]
+end
+
 @NodeP2P_ListenAddr = 30303
 if @config["geth"] and @config["geth"]["NodeP2P_ListenAddr"]
   @NodeP2P_ListenAdd = @config["geth"]["NodeP2P_ListenAddr"]

templates/k8s/network-policy.yaml.erb

@@ -19,6 +19,7 @@ spec:
         -  port: <%= @TM_Port %>
         -  port: 9080
         -  port: <%= @Raft_Port %>
+        -  port: <%= @Node_WSPort %>
         -  port: <%= @Node_RPCPort %>
         -  port: <%= @NodeP2P_ListenAddr %>
   egress:
@@ -31,6 +32,7 @@ spec:
         - port: 9080
         - port: <%= @Raft_Port %>
         - port: <%= @Node_RPCPort %>
+        - port: <%= @Node_WSPort %>
         - port: <%= @NodeP2P_ListenAddr %>
 
 ---
@@ -55,3 +57,4 @@ spec:
         - port: <%= @TM_Port %>
         - port: 9080
         - port:  <%= @Node_RPCPort %>
+        - port: <%= @Node_WSPort %>

templates/k8s/quorum-deployment.yaml.erb

@@ -231,12 +231,13 @@ spec:
 
            touch $QUORUM_DATA_DIR/password.txt;
            NETWORK_ID=<%= @Geth_Network_Id %>
+           RPC_APIS=admin,db,eth,debug,miner,net,shh,txpool,personal,web3,quorum
          <%- if @config["quorum"]["consensus"] == "raft" -%>
-           args=\" --gcmode archive --raft  --raftport <%= @Raft_Port %> --rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,quorum,raft \";
+           args=\" --gcmode archive --raft  --raftport <%= @Raft_Port %> \";
+           RPC_APIS=\"$RPC_APIS,raft\";
          <%- elsif @config["quorum"]["consensus"] == "istanbul" -%>
-           args=\" --gcmode archive --istanbul.blockperiod 3 --syncmode full --mine --minerthreads 1 --rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,quorum,istanbul \";
-         <%- else -%>
-           args=\"--rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,quorum \";
+           args=\" --gcmode archive --istanbul.blockperiod 3 --syncmode full --mine --minerthreads 1 \";
+           RPC_APIS=\"$RPC_APIS,istanbul\";
          <%- end -%>
            /usr/local/bin/geth \
            --datadir $QUORUM_DATA_DIR \
@@ -251,12 +252,16 @@ spec:
            --rpc \
            --rpcaddr 0.0.0.0 \
            --rpcport <%= @Node_RPCPort %> \
+           --rpcapi $RPC_APIS \
+           --wsapi $RPC_APIS \
+           --wsport <%= @Node_WSPort %> \
            --port <%= @NodeP2P_ListenAddr %> \
            <%= @Geth_Startup_Params %> \
            --password $QUORUM_DATA_DIR/password.txt 2>&1 | tee -a <%= @Node_DataDir%>/logs/quorum.log;"
         ports:
           - containerPort: <%= @Raft_Port %>
           - containerPort: <%= @Node_RPCPort %>
+          - containerPort: <%= @Node_WSPort %>
           - containerPort: <%= @NodeP2P_ListenAddr %>
         env:
         - name: PRIVATE_CONFIG

templates/k8s/quorum-ingress.yaml.erb

@@ -36,6 +36,10 @@ spec:
             backend:
               serviceName: <%= @Node_UserIdent %>
               servicePort: <%= @Node_RPCPort %>
+          - path: /quorum-ws
+            backend:
+              serviceName: <%= @Node_UserIdent %>
+              servicePort: <%= @Node_WSPort %>
   tls:
     - hosts:
         - "<%= @Node_UserIdent %>.<%= @Ingress_Host %>"
@@ -46,6 +50,10 @@ spec:
             backend:
               serviceName: <%= @Node_UserIdent %>
               servicePort: <%= @Node_RPCPort %>
+          - path: /quorum-ws
+            backend:
+              serviceName: <%= @Node_UserIdent %>
+              servicePort: <%= @Node_WSPort %>
           <%- end -%> <%# End Host conditional %>
      <% end -%> <%# end this node iteration %>
 <%- end -%> <%# End Ingress OneForEach (this node's ingress) %>
@@ -76,6 +84,10 @@ spec:
             backend:
               serviceName: <%= @Node_UserIdent %>
               servicePort: <%= @Node_RPCPort %>
+          - path: /<%= @Node_UserIdent %>/quorum-ws
+            backend:
+              serviceName: <%= @Node_UserIdent %>
+              servicePort: <%= @Node_WSPort %>
           <%- end -%> <%# end setting path for each node %>
 <%- if @Ingress_Host -%>
   tls:

templates/k8s/quorum-services.yaml.erb

@@ -47,6 +47,10 @@ spec:
       protocol: TCP
       port: <%= @Node_RPCPort %>
       targetPort: <%= @Node_RPCPort %>
+    - name: wsport
+      protocol: TCP
+      port: <%= @Node_WSPort %>
+      targetPort: <%= @Node_WSPort %>
     - name: quorum-listener
       protocol: TCP
       port: <%= @NodeP2P_ListenAddr %>