From ddb926872e97070cf49314720509d6f46c2b260c Mon Sep 17 00:00:00 2001 From: David Main <51991544+StriderDM@users.noreply.github.com> Date: Thu, 2 Dec 2021 09:52:06 +0200 Subject: [PATCH] fix!: multiple monerod addresses in tari merge mining proxy (#3628) Description --- This PR allows multiple monero daemons to be specified in the configuration of tari_merge_mining_proxy, this will help ensure that the proxy is still able to service requests in the event a monero daemon in the list were to become unreachable whether it be temporarily or permanently. Motivation and Context --- Reliability How Has This Been Tested? --- cargo test --all Manually nvm use 12.22.6 && node_modules/.bin/cucumber-js --profile "ci" --tags "not @long-running and not @broken and @merge-mining" --- README.md | 49 ++++++-- .../launchpad/docker_rig/docker-compose.yml | 5 +- .../tari_merge_mining_proxy/src/error.rs | 2 + .../tari_merge_mining_proxy/src/proxy.rs | 109 ++++++++++++------ common/config/presets/merge_mining_proxy.toml | 17 ++- common/src/configuration/global.rs | 24 +++- integration_tests/helpers/config.js | 7 +- .../helpers/mergeMiningProxyProcess.js | 76 +----------- 8 files changed, 158 insertions(+), 131 deletions(-) diff --git a/README.md b/README.md index b8d3a63517..613b76e6c6 100644 --- a/README.md +++ b/README.md @@ -614,7 +614,7 @@ they are not enabled already: ``` ``` [base_node.weatherwax] - transport = "tor" + transpo*_r_*t = "tor" allow_test_addresses = false grpc_enabled = true grpc_base_node_address = "127.0.0.1:18142" @@ -627,7 +627,14 @@ And then depending on if you are using solo mining or self-select mining you wil - For the Tari Merge Mining Proxy, under section **`merge_mining_proxy.weatherwax`** ``` [merge_mining_proxy.weatherwax] - monerod_url = "http://monero-stagenet.exan.tech:38081" + monerod_url = [ # stagenet + "http://stagenet.xmr-tw.org:38081", + "http://stagenet.community.xmr.to:38081", + "http://monero-stagenet.exan.tech:38081", + "http://xmr-lux.boldsuck.org:38081", + "http://singapore.node.xmr.pm:38081", + ] + proxy_host_address = "127.0.0.1:7878" proxy_submit_to_origin = true monerod_use_auth = false @@ -640,7 +647,14 @@ And then depending on if you are using solo mining or self-select mining you wil - For the Tari Merge Mining Proxy, under section **`merge_mining_proxy.weatherwax`** ``` [merge_mining_proxy.weatherwax] - monerod_url = "http://18.132.124.81:18081" + monerod_url = [ # stagenet + "http://stagenet.xmr-tw.org:38081", + "http://stagenet.community.xmr.to:38081", + "http://monero-stagenet.exan.tech:38081", + "http://xmr-lux.boldsuck.org:38081", + "http://singapore.node.xmr.pm:38081", + ] + proxy_host_address = "127.0.0.1:7878" proxy_submit_to_origin = false monerod_use_auth = false @@ -651,8 +665,8 @@ And then depending on if you are using solo mining or self-select mining you wil **Note:** The ports `7878`, `18142` and `18143` shown in the example above should not be in use by other processes. If they are, choose different ports. You will need to update the ports in the steps below as well. -The `monerod_url` must be set to a valid address (`host:port`) for `monerod` that is running Monero mainnet (e.g. -`http://18.132.124.81:18081`) or stagenet (e.g. `http://monero-stagenet.exan.tech:38081`), which can be a +The `monerod_url` set must contain valid addresses (`host:port`) for `monerod` that is running Monero mainnet (e.g. +`["http://18.132.124.81:18081"]`) or stagenet (e.g. `["http://monero-stagenet.exan.tech:38081"]`), which can be a [public node hosted by XMR.to](https://community.xmr.to/nodes.html), or to a local instance. To test if the `monerod_url` address is working properly, try to paste `host:port/get_height` in an internet browser, for example: @@ -688,7 +702,7 @@ in via the command line upon runtime. being a subaddress. It is possible to do with the self-select configuration since the template is requested by the miner with the wallet address of the pool. -###### Solo mining +###### Solo-mining The [XMRig configuration wizard](https://xmrig.com/wizard) can be used to create a solo mining configuration file in JSON format: @@ -832,8 +846,19 @@ Monero wallet address: ``` # URL to monerod -#monerod_url = "http://18.132.124.81:18081" # mainnet -monerod_url = "http://monero-stagenet.exan.tech:38081" # stagenet + monerod_url = [ # mainnet + "http://18.132.124.81:18081", + "http://xmr.support:18081", + "http://node1.xmr-tw.org:18081", + "http://xmr.nthrow.nyc:18081", + ] + monerod_url = [ # stagenet + "http://stagenet.xmr-tw.org:38081", + "http://stagenet.community.xmr.to:38081", + "http://monero-stagenet.exan.tech:38081", + "http://xmr-lux.boldsuck.org:38081", + "http://singapore.node.xmr.pm:38081", + ] ``` ###### Runtime @@ -891,8 +916,12 @@ The `monerod_url` field in the `config.toml` should be enabled for the mainnet v ``` # URL to monerod -monerod_url = "http://18.132.124.81:18081" # mainnet -#monerod_url = "http://monero-stagenet.exan.tech:38081" # stagenet + monerod_url = [ # mainnet + "http://18.132.124.81:18081", + "http://xmr.support:18081", + "http://node1.xmr-tw.org:18081", + "http://xmr.nthrow.nyc:18081", + ] ``` ###### Runtime diff --git a/applications/launchpad/docker_rig/docker-compose.yml b/applications/launchpad/docker_rig/docker-compose.yml index 7dde2f30fa..f7c0924d27 100644 --- a/applications/launchpad/docker_rig/docker-compose.yml +++ b/applications/launchpad/docker_rig/docker-compose.yml @@ -159,7 +159,7 @@ services: TARI_NETWORK: ${TARI_NETWORK} TARI_BASE_NODE__WEATHERWAX__GRPC_BASE_NODE_ADDRESS: "/dns4/base_node/tcp/18142" TARI_WALLET__GRPC_ADDRESS: "/dns4/wallet/tcp/18143" - TARI_MERGE_MINING_PROXY__WEATHERWAX__MONEROD_URL: ${TARI_MONEROD_URL:-http://monero-stagenet.exan.tech:38081} + TARI_MERGE_MINING_PROXY__WEATHERWAX__MONEROD_URL: ${TARI_MONEROD_URL:-["http://stagenet.community.xmr.to:38081","http://monero-stagenet.exan.tech:38081","http://stagenet.xmr-tw.org:38081","http://xmr-lux.boldsuck.org:38081","http://singapore.node.xmr.pm:38081"]} TARI_MERGE_MINING_PROXY__WEATHERWAX__MONEROD_USERNAME: ${TARI_MONEROD_USERNAME} TARI_MERGE_MINING_PROXY__WEATHERWAX__MONEROD_PASSWORD: ${TARI_MONEROD_PASSWORD} TARI_MERGE_MINING_PROXY__WEATHERWAX__MONEROD_USE_AUTH: ${TARI_MONEROD_USE_AUTH:-0} @@ -179,6 +179,3 @@ volumes: # `docker run --rm -v $(pwd):/backup -v blockchain:/blockchain ubuntu tar czvf /backup/backup.tar.gz /blockchain` blockchain: monero-blockchain: - - - diff --git a/applications/tari_merge_mining_proxy/src/error.rs b/applications/tari_merge_mining_proxy/src/error.rs index 7affa3a6e3..53a6ad1527 100644 --- a/applications/tari_merge_mining_proxy/src/error.rs +++ b/applications/tari_merge_mining_proxy/src/error.rs @@ -79,6 +79,8 @@ pub enum MmProxyError { InvalidHeaderValue(#[from] InvalidHeaderValue), #[error("Block was lost due to a failed precondition, and should be retried")] FailedPreconditionBlockLostRetry, + #[error("No reachable servers in configuration")] + ServersUnavailable, } impl From for MmProxyError { diff --git a/applications/tari_merge_mining_proxy/src/proxy.rs b/applications/tari_merge_mining_proxy/src/proxy.rs index f3a9ad5a96..0100602469 100644 --- a/applications/tari_merge_mining_proxy/src/proxy.rs +++ b/applications/tari_merge_mining_proxy/src/proxy.rs @@ -41,6 +41,7 @@ use std::{ sync::{ atomic::{AtomicBool, Ordering}, Arc, + RwLock, }, task::{Context, Poll}, time::Instant, @@ -61,7 +62,7 @@ const TARI_CHAIN_ID: &str = "xtr"; #[derive(Debug, Clone)] pub struct MergeMiningProxyConfig { pub network: Network, - pub monerod_url: String, + pub monerod_url: Vec, pub monerod_username: String, pub monerod_password: String, pub monerod_use_auth: bool, @@ -114,6 +115,7 @@ impl MergeMiningProxyService { base_node_client, wallet_client, initial_sync_achieved: Arc::new(AtomicBool::new(false)), + last_available_server: Arc::new(RwLock::new(None)), }, } } @@ -135,7 +137,7 @@ impl Service> for MergeMiningProxyService { let bytes = match proxy::read_body_until_end(request.body_mut()).await { Ok(b) => b, Err(err) => { - eprintln!("Method: Unknown, Failed to read request: {}", err); + eprintln!("Method: Unknown, Failed to read request: {:?}", err); let resp = proxy::json_response( StatusCode::BAD_REQUEST, &json_rpc::standard_error_response( @@ -153,8 +155,8 @@ impl Service> for MergeMiningProxyService { match inner.handle(&method_name, request).await { Ok(resp) => Ok(resp), Err(err) => { - error!(target: LOG_TARGET, "Error handling request: {}", err); - eprintln!("Method: {}, Failed to handle request: {}", method_name, err); + error!(target: LOG_TARGET, "Error handling request: {:?}", err); + eprintln!("Method: {}, Failed to handle request: {:?}", method_name, err); Ok(proxy::json_response( StatusCode::INTERNAL_SERVER_ERROR, &json_rpc::standard_error_response( @@ -180,6 +182,7 @@ struct InnerService { base_node_client: grpc::base_node_client::BaseNodeClient, wallet_client: grpc::wallet_client::WalletClient, initial_sync_achieved: Arc, + last_available_server: Arc>>, } impl InnerService { @@ -582,9 +585,36 @@ impl InnerService { Ok(proxy::into_response(parts, &resp)) } - fn get_fully_qualified_monerod_url(&self, uri: &Uri) -> Result { - let uri = format!("{}{}", self.config.monerod_url, uri.path()).parse::()?; - Ok(uri) + async fn get_fully_qualified_monerod_url(&self, uri: &Uri) -> Result { + { + let lock = self + .last_available_server + .read() + .expect("Read lock should not fail") + .clone(); + if let Some(server) = lock { + let uri = format!("{}{}", server, uri.path()).parse::()?; + return Ok(uri); + } + } + + for monerod_url in self.config.monerod_url.iter() { + let uri = format!("{}{}", monerod_url, uri.path()).parse::()?; + match reqwest::get(uri.clone()).await { + Ok(_) => { + let mut lock = self.last_available_server.write().expect("Write lock should not fail"); + *lock = Some(monerod_url.to_string()); + info!(target: LOG_TARGET, "Monerod server available: {:?}", uri.clone()); + return Ok(uri); + }, + Err(_) => { + warn!(target: LOG_TARGET, "Monerod server unavailable: {:?}", uri); + continue; + }, + } + } + + Err(MmProxyError::ServersUnavailable) } /// Proxy a request received by this server to Monerod @@ -592,7 +622,7 @@ impl InnerService { &self, request: Request, ) -> Result<(Request, Response), MmProxyError> { - let monerod_uri = self.get_fully_qualified_monerod_url(request.uri())?; + let monerod_uri = self.get_fully_qualified_monerod_url(request.uri()).await?; let mut headers = request.headers().clone(); // Some public monerod setups (e.g. those that are reverse proxied by nginx) require the Host header. @@ -744,34 +774,43 @@ impl InnerService { .join(","), ); - let (request, monerod_resp) = self.proxy_request_to_monerod(request).await?; - // Any failed (!= 200 OK) responses from Monero are immediately returned to the requester - let monerod_status = monerod_resp.status(); - if !monerod_status.is_success() { - // we dont break on xmrig returned error. - warn!( - target: LOG_TARGET, - "Monerod returned an error: {}", - monerod_resp.status() - ); - println!( - "Method: {}, MoneroD Status: {}, Proxy Status: N/A, Response Time: {}ms", - method_name, - monerod_status, - start.elapsed().as_millis() - ); - return Ok(monerod_resp.map(|json| json.to_string().into())); - } + match self.proxy_request_to_monerod(request).await { + Ok((request, monerod_resp)) => { + // Any failed (!= 200 OK) responses from Monero are immediately returned to the requester + let monerod_status = monerod_resp.status(); + if !monerod_status.is_success() { + // we dont break on monerod returning an error code. + warn!( + target: LOG_TARGET, + "Monerod returned an error: {}", + monerod_resp.status() + ); + println!( + "Method: {}, MoneroD Status: {}, Proxy Status: N/A, Response Time: {}ms", + method_name, + monerod_status, + start.elapsed().as_millis() + ); + return Ok(monerod_resp.map(|json| json.to_string().into())); + } - let response = self.get_proxy_response(request, monerod_resp).await?; - println!( - "Method: {}, MoneroD Status: {}, Proxy Status: {}, Response Time: {}ms", - method_name, - monerod_status, - response.status(), - start.elapsed().as_millis() - ); - Ok(response) + let response = self.get_proxy_response(request, monerod_resp).await?; + println!( + "Method: {}, MoneroD Status: {}, Proxy Status: {}, Response Time: {}ms", + method_name, + monerod_status, + response.status(), + start.elapsed().as_millis() + ); + Ok(response) + }, + Err(e) => { + // Monero Server encountered a problem processing the request, reset the last_available_server + let mut lock = self.last_available_server.write().expect("Write lock should not fail"); + *lock = None; + Err(e) + }, + } } } diff --git a/common/config/presets/merge_mining_proxy.toml b/common/config/presets/merge_mining_proxy.toml index ffc10048b8..6c3ef13ed7 100644 --- a/common/config/presets/merge_mining_proxy.toml +++ b/common/config/presets/merge_mining_proxy.toml @@ -7,10 +7,19 @@ [merge_mining_proxy.weatherwax] # URL to monerod -monerod_url = "http://monero-stagenet.exan.tech:38081" # stagenet -#monerod_url = "http://18.133.59.45:28081" # testnet -#monerod_url = "http://18.132.124.81:18081" # mainnet -#monerod_url = "http://monero.exan.tech:18081" # mainnet alternative +monerod_url = [ # stagenet + "http://stagenet.xmr-tw.org:38081", + "http://stagenet.community.xmr.to:38081", + "http://monero-stagenet.exan.tech:38081", + "http://xmr-lux.boldsuck.org:38081", + "http://singapore.node.xmr.pm:38081", +] +#monerod_url = [ # mainnet +# "http://18.132.124.81:18081", +# "http://xmr.support:18081", +# "http://node1.xmr-tw.org:18081", +# "http://xmr.nthrow.nyc:18081", +#] # Address of the tari_merge_mining_proxy application proxy_host_address = "127.0.0.1:7878" diff --git a/common/src/configuration/global.rs b/common/src/configuration/global.rs index c497dd05a8..90bab9ca46 100644 --- a/common/src/configuration/global.rs +++ b/common/src/configuration/global.rs @@ -121,7 +121,7 @@ pub struct GlobalConfig { pub wallet_base_node_service_request_max_age: u64, pub wallet_balance_enquiry_cooldown_period: u64, pub prevent_fee_gt_amount: bool, - pub monerod_url: String, + pub monerod_url: Vec, pub monerod_username: String, pub monerod_password: String, pub monerod_use_auth: bool, @@ -623,9 +623,25 @@ fn convert_node_config( ); let key = config_string("merge_mining_proxy", net_str, "monerod_url"); - let monerod_url = cfg - .get_str(&key) - .map_err(|e| ConfigurationError::new(&key, &e.to_string()))?; + let mut monerod_url: Vec = cfg + .get_array(&key) + .unwrap_or_default() + .into_iter() + .map(|v| { + v.into_str() + .map_err(|err| ConfigurationError::new(&key, &err.to_string())) + }) + .collect::>()?; + + // default to stagenet on empty + if monerod_url.is_empty() { + monerod_url = vec![ + "http://stagenet.xmr-tw.org:38081".to_string(), + "http://singapore.node.xmr.pm:38081".to_string(), + "http://xmr-lux.boldsuck.org:38081".to_string(), + "http://monero-stagenet.exan.tech:38081".to_string(), + ]; + } let key = config_string("merge_mining_proxy", net_str, "monerod_use_auth"); let monerod_use_auth = cfg diff --git a/integration_tests/helpers/config.js b/integration_tests/helpers/config.js index 49bf822dbd..70698c4540 100644 --- a/integration_tests/helpers/config.js +++ b/integration_tests/helpers/config.js @@ -94,8 +94,13 @@ function baseEnvs(peerSeeds = [], forceSyncPeers = []) { TARI_BASE_NODE__LOCALNET__MAX_RANDOMX_VMS: "1", TARI_BASE_NODE__LOCALNET__AUTO_PING_INTERVAL: "15", TARI_BASE_NODE__LOCALNET__FLOOD_BAN_MAX_MSG_COUNT: "100000", - TARI_MERGE_MINING_PROXY__LOCALNET__MONEROD_URL: + TARI_MERGE_MINING_PROXY__LOCALNET__MONEROD_URL: [ + "http://stagenet.xmr-tw.org:38081", + "http://stagenet.community.xmr.to:38081", "http://monero-stagenet.exan.tech:38081", + "http://xmr-lux.boldsuck.org:38081", + "http://singapore.node.xmr.pm:38081", + ], TARI_MERGE_MINING_PROXY__LOCALNET__MONEROD_USE_AUTH: false, TARI_MERGE_MINING_PROXY__LOCALNET__MONEROD_USERNAME: '""', TARI_MERGE_MINING_PROXY__LOCALNET__MONEROD_PASSWORD: '""', diff --git a/integration_tests/helpers/mergeMiningProxyProcess.js b/integration_tests/helpers/mergeMiningProxyProcess.js index fe0b87159a..f31d85dde5 100644 --- a/integration_tests/helpers/mergeMiningProxyProcess.js +++ b/integration_tests/helpers/mergeMiningProxyProcess.js @@ -40,7 +40,7 @@ class MergeMiningProxyProcess { // console.log("MergeMiningProxyProcess init - assign server GRPC:", this.grpcPort); } - run(cmd, args, monerodUrl) { + async run(cmd, args) { return new Promise((resolve, reject) => { if (!fs.existsSync(this.baseDir)) { fs.mkdirSync(this.baseDir, { recursive: true }); @@ -67,9 +67,9 @@ class MergeMiningProxyProcess { const extraEnvs = { TARI_MERGE_MINING_PROXY__LOCALNET__PROXY_SUBMIT_TO_ORIGIN: this.submitOrigin, - TARI_MERGE_MINING_PROXY__LOCALNET__monerod_url: monerodUrl, }; const completeEnvs = { ...envs, ...extraEnvs }; + console.log(completeEnvs); const ps = spawn(cmd, args, { cwd: this.baseDir, // shell: true, @@ -105,75 +105,6 @@ class MergeMiningProxyProcess { }); } - async testWebsite(protocol, address, port, path) { - const url = protocol + "://" + address + ":" + port; - const webRequest = require(protocol); - - let request; - let thePromise; - const displayData = false; - try { - thePromise = await new Promise((resolve, reject) => { - request = webRequest - .get(url + path, (resp) => { - let data = ""; - // Read all data chunks until the end - resp.on("data", (chunk) => { - data += chunk; - }); - // Finish when complete response has been received - resp.on("end", () => { - if (displayData) { - console.log(data); // `data` is 'used' here to keep eslint happy - } - return resolve(true); - }); - }) - .on("error", () => { - return reject(false); - }); - }); - console.log( - " >> Info: `monerod` at", - url, - "is responsive and available" - ); - } catch { - console.log(" >> Warn: `monerod` at", url, "is not available!"); - } - request.end(); - - return thePromise; - } - - async getMoneroStagenetUrl() { - // See: https://monero.fail/?nettype=stagenet - const monerodUrl = [ - ["http", "singapore.node.xmr.pm", "38081"], - ["http", "stagenet.xmr-tw.org", "38081"], - ["http", "xmr-lux.boldsuck.org", "38081"], - ["http", "monero-stagenet.exan.tech", "38081"], - ["http", "3.104.4.129", "18081"], // flaky - ["http", "stagenet.community.xmr.to", "38081"], // flaky - ["http", "super.fast.node.xmr.pm", "38089"], // flaky - ]; - let url; - for (let i = 0; i < monerodUrl.length; i++) { - let availble = await this.testWebsite( - monerodUrl[i][0], - monerodUrl[i][1], - monerodUrl[i][2], - "/get_height" - ); - if (availble) { - url = - monerodUrl[i][0] + "://" + monerodUrl[i][1] + ":" + monerodUrl[i][2]; - break; - } - } - return url; - } - async startNew() { await this.init(); const args = ["--base-path", ".", "--init"]; @@ -181,8 +112,7 @@ class MergeMiningProxyProcess { args.push("--log-config", this.logFilePath); } - let url = await this.getMoneroStagenetUrl(); - return await this.run(await this.compile(), args, url); + return await this.run(await this.compile(), args); } async compile() {