Breakdown fees protocol 3/3: DB field and returning on /get_trades (#…

…2910)

# Description
Related to #2862

Exposes executed protocol fees from database on /get_trades endpoint.

Database fee field is an array of <token, amount> so that we have the
flexibility now (and also in the future) to save taken fee in whatever
token we want. This was inspired by discussion where even for the same
order we considered defining taken protocol fee in sell token for
"surplus" variant, while, for "volume" variant it's more logical to
define it in non-surplus token (sell token for sell order and buy token
for buy order) so that the taken fee doesn't depend on the amount of
surplus provided in a solution.

# Changes
<!-- List of detailed changes (how the change is accomplished) -->

- [ ] Add new columns `protocol_fee_tokens` and `protocol_fee_amounts`
to `order_execution` database table.
- [ ] Expand `executedProtocolFees` field on `Trade` struct to return
both executed protocol fees and fee policies.

## How to test
Updated db test.
Existing tests.
e2e test updated to prove correctness.

crates/autopilot/src/domain/settlement/mod.rs

@@ -5,7 +5,6 @@
 
 use {
     crate::{domain, domain::eth, infra},
-    num::Saturating,
     std::collections::HashMap,
 };
 
@@ -85,22 +84,10 @@ impl Settlement {
     }
 
     /// Per order fees breakdown. Contains all orders from the settlement
-    pub fn order_fees(&self) -> HashMap<domain::OrderUid, Option<trade::ExecutedFee>> {
+    pub fn fee_breakdown(&self) -> HashMap<domain::OrderUid, Option<trade::FeeBreakdown>> {
         self.trades
             .iter()
-            .map(|trade| {
-                let total = trade.fee_in_sell_token();
-                let protocol = trade.protocol_fees_in_sell_token(&self.auction);
-                let fee = match (total, protocol) {
-                    (Ok(total), Ok(protocol)) => {
-                        let network =
-                            total.saturating_sub(protocol.iter().map(|(fee, _)| *fee).sum());
-                        Some(trade::ExecutedFee { protocol, network })
-                    }
-                    _ => None,
-                };
-                (*trade.uid(), fee)
-            })
+            .map(|trade| (*trade.uid(), trade.fee_breakdown(&self.auction).ok()))
             .collect()
     }
 

crates/autopilot/src/domain/settlement/trade/math.rs

@@ -1,5 +1,6 @@
 pub use error::Error;
 use {
+    super::ExecutedProtocolFee,
     crate::{
         domain::{
             self,
@@ -154,24 +155,13 @@ impl Trade {
         })
     }
 
-    /// Protocol fees are defined by fee policies attached to the order.
-    pub fn protocol_fees_in_sell_token(
-        &self,
-        auction: &settlement::Auction,
-    ) -> Result<Vec<(eth::SellTokenAmount, fee::Policy)>, Error> {
-        self.protocol_fees(auction)?
-            .into_iter()
-            .map(|(fee, policy)| Ok((self.fee_into_sell_token(fee.amount)?, policy)))
-            .collect()
-    }
-
     /// Protocol fees are defined by fee policies attached to the order.
     ///
     /// Denominated in SURPLUS token
-    fn protocol_fees(
+    pub fn protocol_fees(
         &self,
         auction: &settlement::Auction,
-    ) -> Result<Vec<(eth::Asset, fee::Policy)>, Error> {
+    ) -> Result<Vec<ExecutedProtocolFee>, Error> {
         let policies = auction
             .orders
             .get(&self.uid)
@@ -180,11 +170,14 @@ impl Trade {
         let mut current_trade = self.clone();
         let mut total = eth::TokenAmount::default();
         let mut fees = vec![];
-        for (i, protocol_fee) in policies.iter().enumerate().rev() {
-            let fee = current_trade.protocol_fee(protocol_fee)?;
+        for (i, policy) in policies.iter().enumerate().rev() {
+            let fee = current_trade.protocol_fee(policy)?;
             // Do not need to calculate the last custom prices because in the last iteration
             // the prices are not used anymore to calculate the protocol fee
-            fees.push((fee, *protocol_fee));
+            fees.push(ExecutedProtocolFee {
+                policy: *policy,
+                fee,
+            });
             total += fee.amount;
             if !i.is_zero() {
                 current_trade.prices.custom = self.calculate_custom_prices(total)?;
@@ -426,7 +419,7 @@ impl Trade {
     fn protocol_fee_in_ether(&self, auction: &settlement::Auction) -> Result<eth::Ether, Error> {
         self.protocol_fees(auction)?
             .into_iter()
-            .map(|(fee, _)| {
+            .map(|ExecutedProtocolFee { fee, policy: _ }| {
                 let price = auction
                     .prices
                     .get(&fee.token)

crates/autopilot/src/domain/settlement/trade/mod.rs

@@ -67,18 +67,12 @@ impl Trade {
         math::Trade::from(self).fee_in_ether(prices)
     }
 
-    /// Total fee (protocol fee + network fee). Equal to a surplus difference
-    /// before and after applying the fees.
-    pub fn fee_in_sell_token(&self) -> Result<eth::SellTokenAmount, math::Error> {
-        math::Trade::from(self).fee_in_sell_token()
-    }
-
-    /// Protocol fees are defined by fee policies attached to the order.
-    pub fn protocol_fees_in_sell_token(
-        &self,
-        auction: &super::Auction,
-    ) -> Result<Vec<(eth::SellTokenAmount, fee::Policy)>, math::Error> {
-        math::Trade::from(self).protocol_fees_in_sell_token(auction)
+    /// All fees broke down into protocol fees per policy and total fee.
+    pub fn fee_breakdown(&self, auction: &super::Auction) -> Result<FeeBreakdown, math::Error> {
+        let trade = math::Trade::from(self);
+        let total = trade.fee_in_sell_token()?;
+        let protocol = trade.protocol_fees(auction)?;
+        Ok(FeeBreakdown { total, protocol })
     }
 
     pub fn new(trade: transaction::EncodedTrade, auction: &super::Auction, created: u32) -> Self {
@@ -154,17 +148,18 @@ pub struct Jit {
 /// Fee per trade in a solution. These fees are taken for the execution of the
 /// trade.
 #[derive(Debug, Clone)]
-pub struct ExecutedFee {
-    /// Gas fee spent to bring the order onchain
-    pub network: eth::SellTokenAmount,
-    /// Breakdown of protocol fees. Executed protocol fees are in the same order
-    /// as policies are defined for an order.
-    pub protocol: Vec<(eth::SellTokenAmount, fee::Policy)>,
+pub struct FeeBreakdown {
+    /// Total fee the trade was charged (network fee + protocol fee)
+    // TODO: express in surplus token
+    pub total: eth::SellTokenAmount,
+    /// Breakdown of protocol fees.
+    pub protocol: Vec<ExecutedProtocolFee>,
 }
 
-impl ExecutedFee {
-    /// Total fee paid for the trade.
-    pub fn total(&self) -> eth::SellTokenAmount {
-        self.network + self.protocol.iter().map(|(fee, _)| *fee).sum()
-    }
+#[derive(Debug, Clone)]
+pub struct ExecutedProtocolFee {
+    /// Policy that was used to calculate the fee.
+    pub policy: fee::Policy,
+    /// Fee that was taken for the trade, in surplus token.
+    pub fee: eth::Asset,
 }

crates/autopilot/src/infra/persistence/mod.rs

@@ -11,6 +11,7 @@ use {
     chrono::{DateTime, Utc},
     database::{
         order_events::OrderEventLabel,
+        order_execution::Asset,
         orders::{
             BuyTokenDestination as DbBuyTokenDestination,
             SellTokenSource as DbSellTokenSource,
@@ -589,7 +590,7 @@ impl Persistence {
             let gas_price = settlement.gas_price();
             let surplus = settlement.surplus_in_ether();
             let fee = settlement.fee_in_ether();
-            let order_fees = settlement.order_fees();
+            let fee_breakdown = settlement.fee_breakdown();
             let jit_orders = settlement.jit_orders();
 
             tracing::debug!(
@@ -599,7 +600,7 @@ impl Persistence {
                 ?gas_price,
                 ?surplus,
                 ?fee,
-                ?order_fees,
+                ?fee_breakdown,
                 ?jit_orders,
                 "settlement update",
             );
@@ -619,21 +620,35 @@ impl Persistence {
 
             store_order_events(
                 &mut ex,
-                order_fees.keys().cloned().collect(),
+                fee_breakdown.keys().cloned().collect(),
                 OrderEventLabel::Traded,
                 Utc::now(),
             )
             .await;
 
-            for (order, executed_fee) in order_fees {
+            for (order, order_fee) in fee_breakdown {
+                let total_fee = order_fee
+                    .as_ref()
+                    .map(|fee| u256_to_big_decimal(&fee.total.0))
+                    .unwrap_or_default();
+                let executed_protocol_fees = order_fee
+                    .map(|fee| {
+                        fee.protocol
+                            .into_iter()
+                            .map(|executed| Asset {
+                                token: ByteArray(executed.fee.token.0 .0),
+                                amount: u256_to_big_decimal(&executed.fee.amount.0),
+                            })
+                            .collect::<Vec<_>>()
+                    })
+                    .unwrap_or_default();
                 database::order_execution::save(
                     &mut ex,
                     &ByteArray(order.0),
                     auction_id,
                     block_number,
-                    &u256_to_big_decimal(
-                        &executed_fee.map(|fee| fee.total()).unwrap_or_default().0,
-                    ),
+                    &total_fee,
+                    &executed_protocol_fees,
                 )
                 .await?;
             }

crates/database/src/fee_policies.rs

@@ -4,6 +4,8 @@ use {
     std::collections::HashMap,
 };
 
+type Execution = (AuctionId, OrderUid);
+
 #[derive(Debug, Clone, PartialEq, sqlx::FromRow)]
 pub struct FeePolicy {
     pub auction_id: AuctionId,
@@ -55,8 +57,8 @@ pub async fn insert_batch(
 
 pub async fn fetch_all(
     ex: &mut PgConnection,
-    keys_filter: &[(AuctionId, OrderUid)],
-) -> Result<HashMap<(AuctionId, OrderUid), Vec<FeePolicy>>, sqlx::Error> {
+    keys_filter: &[Execution],
+) -> Result<HashMap<Execution, Vec<FeePolicy>>, sqlx::Error> {
     if keys_filter.is_empty() {
         return Ok(HashMap::new());
     }

crates/database/src/order_execution.rs

@@ -1,33 +1,101 @@
 use {
-    crate::{auction::AuctionId, OrderUid},
+    crate::{auction::AuctionId, Address, OrderUid},
     bigdecimal::BigDecimal,
-    sqlx::PgConnection,
+    sqlx::{PgConnection, QueryBuilder},
+    std::collections::HashMap,
 };
 
+type Execution = (AuctionId, OrderUid);
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Asset {
+    pub amount: BigDecimal,
+    pub token: Address,
+}
+
 pub async fn save(
     ex: &mut PgConnection,
     order: &OrderUid,
     auction: AuctionId,
     block_number: i64,
     executed_fee: &BigDecimal,
+    executed_protocol_fees: &[Asset],
 ) -> Result<(), sqlx::Error> {
+    let (protocol_fee_tokens, protocol_fee_amounts): (Vec<_>, Vec<_>) = executed_protocol_fees
+        .iter()
+        .map(|entry| (entry.token, entry.amount.clone()))
+        .unzip();
+
     const QUERY: &str = r#"
-INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, block_number)
-VALUES ($1, $2, $3, $4, $5)
+INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, block_number, protocol_fee_tokens, protocol_fee_amounts)
+VALUES ($1, $2, $3, $4, $5, $6, $7)
 ON CONFLICT (order_uid, auction_id)
-DO UPDATE SET reward = $3, surplus_fee = $4, block_number = $5
+DO UPDATE SET reward = $3, surplus_fee = $4, block_number = $5, protocol_fee_tokens = $6, protocol_fee_amounts = $7
 ;"#;
     sqlx::query(QUERY)
         .bind(order)
         .bind(auction)
         .bind(0.) // reward is deprecated but saved for historical analysis
         .bind(Some(executed_fee))
         .bind(block_number)
+        .bind(protocol_fee_tokens)
+        .bind(protocol_fee_amounts)
         .execute(ex)
         .await?;
     Ok(())
 }
 
+/// Fetch protocol fees for all keys in the filter
+pub async fn executed_protocol_fees(
+    ex: &mut PgConnection,
+    keys_filter: &[Execution],
+) -> Result<HashMap<Execution, Vec<Asset>>, sqlx::Error> {
+    if keys_filter.is_empty() {
+        return Ok(HashMap::new());
+    }
+
+    let mut query_builder = QueryBuilder::new(
+        "SELECT order_uid, auction_id, protocol_fee_tokens, protocol_fee_amounts FROM \
+         order_execution WHERE ",
+    );
+
+    for (i, (auction_id, order_uid)) in keys_filter.iter().enumerate() {
+        if i > 0 {
+            query_builder.push(" OR ");
+        }
+        query_builder
+            .push("(order_uid = ")
+            .push_bind(order_uid)
+            .push(" AND auction_id = ")
+            .push_bind(auction_id)
+            .push(")");
+    }
+
+    #[derive(Clone, Debug, Eq, PartialEq, sqlx::Type, sqlx::FromRow)]
+    struct ProtocolFees {
+        pub order_uid: OrderUid,
+        pub auction_id: AuctionId,
+        pub protocol_fee_tokens: Vec<Address>,
+        pub protocol_fee_amounts: Vec<BigDecimal>,
+    }
+    let query = query_builder.build_query_as::<ProtocolFees>();
+    let rows: Vec<ProtocolFees> = query.fetch_all(ex).await?;
+
+    let mut fees = HashMap::new();
+    for row in rows {
+        fees.insert(
+            (row.auction_id, row.order_uid),
+            row.protocol_fee_tokens
+                .into_iter()
+                .zip(row.protocol_fee_amounts)
+                .map(|(token, amount)| Asset { token, amount })
+                .collect(),
+        );
+    }
+
+    Ok(fees)
+}
+
 #[cfg(test)]
 mod tests {
     use {super::*, sqlx::Connection};
@@ -39,8 +107,42 @@ mod tests {
         let mut db = db.begin().await.unwrap();
         crate::clear_DANGER_(&mut db).await.unwrap();
 
-        save(&mut db, &Default::default(), 1, 0, &Default::default())
+        // save entry with protocol fees
+        let protocol_fees = vec![
+            Asset {
+                amount: BigDecimal::from(1),
+                token: Default::default(),
+            },
+            Asset {
+                amount: BigDecimal::from(2),
+                token: Default::default(),
+            },
+        ];
+        save(
+            &mut db,
+            &Default::default(),
+            1,
+            0,
+            &Default::default(),
+            protocol_fees.as_slice(),
+        )
+        .await
+        .unwrap();
+
+        // save entry without protocol fees (simulate case when we are still not
+        // calculating them)
+        save(&mut db, &Default::default(), 2, 0, &Default::default(), &[])
             .await
             .unwrap();
+
+        let keys: Vec<(AuctionId, OrderUid)> = vec![
+            (1, Default::default()),
+            (2, Default::default()),
+            (3, Default::default()),
+        ];
+
+        let read_protocol_fees = executed_protocol_fees(&mut db, &keys).await.unwrap();
+        assert_eq!(read_protocol_fees.len(), 2);
+        assert_eq!(read_protocol_fees[&(1, Default::default())], protocol_fees);
     }
 }