Rebuild account reference counters (#249)

* Rebuild account reference counters part.1

* part.2

* part.3

* TODO

* Fix

* Fixes

Signed-off-by: Xavier Lau <xavier@inv.cafe>

* Doc

---------

Signed-off-by: Xavier Lau <xavier@inv.cafe>

tool/state-processor/src/identity/mod.rs

@@ -38,7 +38,7 @@ where
 					})
 			});
 
-			self.shell_state.reserve(a, r + rj);
+			self.shell_state.reserve(&a, r + rj);
 			self.shell_state.insert_value(b"AccountMigration", b"Identities", &k, v);
 		});
 

tool/state-processor/src/main.rs

@@ -16,7 +16,6 @@ use type_registry::*;
 mod balances;
 mod evm;
 mod identity;
-mod session;
 mod staking;
 mod system;
 mod vesting;
@@ -32,10 +31,10 @@ fn main() -> Result<()> {
 	pretty_env_logger::init();
 
 	// <Processor<Pangolin>>::new()?.process().save()?;
-	<Processor<Pangolin>>::new()?.test().process().save()?;
+	// <Processor<Pangolin>>::new()?.test().process().save()?;
 	// <Processor<Pangoro>>::new()?.process().save()?;
 	// <Processor<Pangoro>>::new()?.test().process().save()?;
-	// <Processor<Crab>>::new()?.process().save()?;
+	<Processor<Crab>>::new()?.process().save()?;
 	// <Processor<Crab>>::new()?.test().process().save()?;
 	// <Processor<Darwinia>>::new()?.process().save()?;
 	// <Processor<Darwinia>>::new()?.test().process().save()?;

tool/state-processor/src/processor.rs

@@ -11,15 +11,14 @@ use std::{
 use crate::*;
 // crates.io
 use anyhow::Result;
-use fxhash::{FxHashMap, FxHashSet};
+use fxhash::FxHashMap;
 use once_cell::sync::Lazy;
 use parity_scale_codec::{Decode, Encode};
 use serde::de::DeserializeOwned;
 use serde_json::Value;
 // hack-ink
 use subspector::ChainSpec;
 
-pub type Set = FxHashSet<String>;
 pub type Map<V> = FxHashMap<String, V>;
 
 pub static NOW: Lazy<RwLock<u32>> = Lazy::new(|| RwLock::new(0));
@@ -193,23 +192,6 @@ impl<R> State<R> {
 		self.map.keys().into_iter().any(|k| k.starts_with(&item_key(pallet, item)))
 	}
 
-	pub fn reserve<A>(&mut self, account_id_32: A, amount: u128)
-	where
-		A: AsRef<[u8]>,
-	{
-		let account_id_32 = account_id_32.as_ref();
-		let (p, i, h) = if is_evm_address(account_id_32) {
-			(&b"System"[..], &b"Account"[..], &account_id_32[11..31])
-		} else {
-			(&b"AccountMigration"[..], &b"Accounts"[..], account_id_32)
-		};
-
-		self.mutate_value(p, i, &blake2_128_concat_to_string(h), |a: &mut AccountInfo| {
-			a.data.free -= amount;
-			a.data.reserved += amount;
-		});
-	}
-
 	pub fn take_raw_map<F>(
 		&mut self,
 		prefix: &str,
@@ -232,23 +214,6 @@ impl<R> State<R> {
 		self
 	}
 
-	pub fn take_keys<F>(&mut self, prefix: &str, keys: &mut Set, process_key: F) -> &mut Self
-	where
-		F: Fn(&str, &str) -> String,
-	{
-		self.map.retain(|k, _| {
-			if k.starts_with(prefix) {
-				keys.insert(process_key(k, prefix));
-
-				false
-			} else {
-				true
-			}
-		});
-
-		self
-	}
-
 	pub fn take_value<D>(
 		&mut self,
 		pallet: &[u8],
@@ -336,6 +301,40 @@ impl<R> State<R> {
 
 		self
 	}
+
+	fn mutate_account<F>(&mut self, account_id_32: &str, f: F)
+	where
+		F: FnOnce(&mut AccountInfo),
+	{
+		let account_id_32 = array_bytes::hex2array_unchecked::<_, 32>(account_id_32);
+		let (p, i, h) = if is_evm_address(&account_id_32) {
+			(&b"System"[..], &b"Account"[..], &account_id_32[11..31])
+		} else {
+			(&b"AccountMigration"[..], &b"Accounts"[..], &account_id_32[..])
+		};
+
+		self.mutate_value(p, i, &blake2_128_concat_to_string(h), f);
+	}
+
+	pub fn inc_consumers_by(&mut self, account_id_32: &str, x: RefCount) {
+		self.mutate_account(account_id_32, |a| a.consumers += x);
+	}
+
+	pub fn reserve(&mut self, account_id_32: &str, amount: u128) {
+		self.mutate_account(account_id_32, |a| {
+			if a.data.free < amount {
+				log::warn!(
+					"`Account({account_id_32})` can't afford the latest runtime reservation amount"
+				);
+
+				a.data.reserved += a.data.free;
+				a.data.free = 0;
+			} else {
+				a.data.free -= amount;
+				a.data.reserved += amount;
+			}
+		});
+	}
 }
 
 pub fn from_file<D>(path: &str) -> Result<D>

tool/state-processor/src/staking/mod.rs

@@ -30,9 +30,12 @@ impl<S> Processor<S> {
 				let hash_k = blake2_128_concat_to_string(v.stash);
 				let deposit_k = format!("{deposit_ik}{hash_k}");
 				let staking_k = format!("{staking_ik}{hash_k}");
+				let mut consumers = 1;
 				let mut staked_deposits = Vec::default();
 
 				if !v.deposit_items.is_empty() {
+					consumers += 1;
+
 					let mut deposit_ring = Balance::default();
 
 					self.shell_state.insert_raw_key_value(
@@ -61,6 +64,10 @@ impl<S> Processor<S> {
 				ring_pool += v.active;
 				kton_pool += v.active_kton;
 
+				// Some accounts were killed.
+				// But their staking data didn't get deleted.
+				// TODO: https://github.com/darwinia-network/darwinia-2.0/issues/6
+				self.shell_state.inc_consumers_by(&array_bytes::bytes2hex("", v.stash), consumers);
 				self.shell_state.insert_raw_key_value(
 					staking_k,
 					Ledger {

tool/state-processor/src/system/README.md

@@ -11,11 +11,11 @@
   - take para `Balances::TotalIssuance` and `Balances::Locks`
     - there should not be any locks on parachain
   - check if there are any other locks
-- decrease pallet-session account references
-- use all previous data to build the new accounts and calculate total issuances
+- burn parachain backing ring
+- build the new accounts and calculate total issuances
   - merge solo and para account infos
-    - how to deal with the account references? - TODO
-  - burn parachain backing ring
+  - reset account reference counters
+  - increase `EVM::AccountCodes`'s `sufficients`
 - set `Balances::TotalIssuance`
   - compare the calculated one with the storage one
   - remove para backing account

tool/state-processor/src/system/mod.rs

@@ -6,9 +6,6 @@ use crate::*;
 #[derive(Debug)]
 pub struct AccountAll {
 	pub nonce: u32,
-	pub consumers: RefCount,
-	pub providers: RefCount,
-	pub sufficients: RefCount,
 	pub ring: Balance,
 	pub ring_locks: Vec<BalanceLock>,
 	pub kton: Balance,
@@ -22,31 +19,13 @@ where
 	pub fn process_system(&mut self) -> &mut Self {
 		// System storage items.
 		// https://github.dev/darwinia-network/substrate/blob/darwinia-v0.12.5/frame/system/src/lib.rs#L545
-		let mut solo_account_infos = self.process_solo_account_infos();
-		let mut para_account_infos = self.process_para_account_infos();
+		let solo_account_infos = self.process_solo_account_infos();
+		let para_account_infos = self.process_para_account_infos();
 		let (ring_total_issuance_storage, kton_total_issuance_storage) = self.process_balances();
-		let (solo_validators, para_collators) = self.process_session();
 		let mut accounts = Map::default();
 		let mut ring_total_issuance = Balance::default();
 		let mut kton_total_issuance = Balance::default();
 
-		// Skip for testnets, due to https://github.com/paritytech/substrate/issues/13172.
-		// log::info!("decrease solo pallet-session account references");
-		// solo_validators.into_iter().for_each(|k| {
-		// 	if let Some(a) = solo_account_infos.get_mut(&k) {
-		// 		a.consumers -= 1;
-		// 	}
-		// });
-
-		// Skip, due to https://github.com/paritytech/substrate/issues/13172.
-		// log::info!("decrease para pallet-session account references");
-		// para_collators.into_iter().for_each(|k| {
-		// 	if let Some(a) = para_account_infos.get_mut(&k) {
-		// 		dbg!(get_last_64(&k));
-		// 		a.consumers -= 1;
-		// 	}
-		// });
-
 		log::info!("build accounts");
 		log::info!("calculate total issuance");
 		solo_account_infos.into_iter().for_each(|(k, v)| {
@@ -57,12 +36,6 @@ where
 				k,
 				AccountAll {
 					nonce: v.nonce,
-					// ---
-					// TODO: check if we could ignore para's.
-					consumers: v.consumers,
-					providers: v.providers,
-					sufficients: v.sufficients,
-					// ---
 					ring,
 					ring_locks: Default::default(),
 					kton,
@@ -84,9 +57,6 @@ where
 				})
 				.or_insert(AccountAll {
 					nonce: v.nonce,
-					consumers: v.consumers,
-					providers: v.providers,
-					sufficients: v.sufficients,
 					ring,
 					ring_locks: Default::default(),
 					kton: Default::default(),
@@ -128,14 +98,15 @@ where
 			is_frozen: false,
 		};
 
-		log::info!("fix `EVM::AccountCodes`'s `sufficients` and set `Assets::Account`, `System::Account`, `AccountMigration::KtonAccounts` and `AccountMigration::Accounts`");
+		log::info!("increase `EVM::AccountCodes`'s `sufficients` and set `Assets::Account`, `System::Account`, `AccountMigration::KtonAccounts` and `AccountMigration::Accounts`");
 		accounts.into_iter().for_each(|(k, v)| {
 			let key = get_last_64(&k);
 			let mut a = AccountInfo {
 				nonce: v.nonce,
-				consumers: v.consumers,
-				providers: v.providers,
-				sufficients: v.sufficients,
+				consumers: Default::default(),
+				// https://github.com/paritytech/substrate/blob/3bc3742d5c0c5269353d7809d9f8f91104a93273/frame/system/src/lib.rs#L1708
+				providers: 1,
+				sufficients: Default::default(),
 				data: AccountData {
 					free: v.ring,
 					reserved: Default::default(),
@@ -145,14 +116,12 @@ where
 			};
 
 			if let Some(k) = try_get_evm_address(&key) {
-				// If the evm account is a contract contract with sufficients, then we should
-				// increase the sufficients by one.
+				// https://github.dev/paritytech/frontier/blob/ab0f4a47e42ad17e4d8551fb9b3c3a6b4c5df2db/frame/evm/src/lib.rs#L705
 				if self.solo_state.contains_key(&full_key(
 					b"EVM",
 					b"AccountCodes",
 					&blake2_128_concat_to_string(k),
-				)) && a.sufficients == 0
-				{
+				)) {
 					a.sufficients += 1;
 				}
 
@@ -276,6 +245,7 @@ fn new_kton_account(
 	asset_details: &mut AssetDetails,
 	balance: Balance,
 ) -> AssetAccount {
+	// https://github.com/paritytech/substrate/blob/3bc3742d5c0c5269353d7809d9f8f91104a93273/frame/assets/src/functions.rs#L75
 	account_info.sufficients += 1;
 	asset_details.accounts += 1;
 	asset_details.sufficients += 1;

tool/state-processor/src/tests.rs

@@ -116,6 +116,7 @@ where
 
 // --- System & Balances & Assets ---
 
+// TODO: more testcases about the account references and reservations
 #[test]
 fn solo_chain_substrate_account_adjust() {
 	run_test(|tester| {
@@ -132,7 +133,7 @@ fn solo_chain_substrate_account_adjust() {
 		// after migrate
 
 		let migrated_account = tester.migration_accounts.get(test_addr).unwrap();
-		assert_eq!(solo_account.consumers, migrated_account.consumers);
+		assert_eq!(migrated_account.consumers, 2);
 		assert_eq!(solo_account.providers, migrated_account.providers);
 		assert_eq!(solo_account.sufficients + 1, migrated_account.sufficients);
 		// nonce reset

tool/state-processor/src/util.rs

@@ -73,10 +73,6 @@ pub fn is_evm_address(address: &[u8]) -> bool {
 		&& address[1..31].iter().fold(address[0], |checksum, &b| checksum ^ b) == address[31]
 }
 
-pub fn key_to_account32(key: &str) -> [u8; 32] {
-	array_bytes::hex2array_unchecked(get_last_64(key))
-}
-
 pub fn build_spec(chain: &str) -> Result<()> {
 	log::info!("build {chain} spec");