-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: new torrent repository using crossbeam_skiplist::SkipMap
SkipMap is an ordered map based on a lock-free skip list. It's al alternative to BTreeMap which supports concurrent access across multiple threads. One of the performance problems with the current solution is we can only add one torrent at the time because threads need to lock the whole BTreeMap. The SkipMap should avoid that problem. More info about SkiMap: https://docs.rs/crossbeam-skiplist/latest/crossbeam_skiplist/struct.SkipMap.html#method.remove The aquatic UDP load test was executed with the current implementation and the new one: Current Implementation: Requests out: 397287.37/second Responses in: 357549.15/second - Connect responses: 177073.94 - Announce responses: 176905.36 - Scrape responses: 3569.85 - Error responses: 0.00 Peers per announce response: 0.00 Announce responses per info hash: - p10: 1 - p25: 1 - p50: 1 - p75: 1 - p90: 2 - p95: 3 - p99: 104 - p99.9: 287 - p100: 371 New Implementation: Requests out: 396788.68/second Responses in: 357105.27/second - Connect responses: 176662.91 - Announce responses: 176863.44 - Scrape responses: 3578.91 - Error responses: 0.00 Peers per announce response: 0.00 Announce responses per info hash: - p10: 1 - p25: 1 - p50: 1 - p75: 1 - p90: 2 - p95: 3 - p99: 105 - p99.9: 287 - p100: 351 The result is pretty similar but the benchmarking for the repository using criterios shows that this implementations is a litle bit better than the current one.
- Loading branch information
1 parent
608585e
commit 642d6be
Showing
7 changed files
with
137 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -135,6 +135,7 @@ | |
"Shareaza", | ||
"sharktorrent", | ||
"SHLVL", | ||
"skiplist", | ||
"socketaddr", | ||
"sqllite", | ||
"subsec", | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
packages/torrent-repository/src/repository/skip_map_mutex_std.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
use std::collections::BTreeMap; | ||
use std::sync::Arc; | ||
|
||
use crossbeam_skiplist::SkipMap; | ||
use torrust_tracker_configuration::TrackerPolicy; | ||
use torrust_tracker_primitives::info_hash::InfoHash; | ||
use torrust_tracker_primitives::pagination::Pagination; | ||
use torrust_tracker_primitives::swarm_metadata::SwarmMetadata; | ||
use torrust_tracker_primitives::torrent_metrics::TorrentsMetrics; | ||
use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, PersistentTorrents}; | ||
|
||
use super::Repository; | ||
use crate::entry::{Entry, EntrySync}; | ||
use crate::{EntryMutexStd, EntrySingle}; | ||
|
||
#[derive(Default, Debug)] | ||
pub struct CrossbeamSkipList<T> { | ||
torrents: SkipMap<InfoHash, T>, | ||
} | ||
|
||
impl Repository<EntryMutexStd> for CrossbeamSkipList<EntryMutexStd> | ||
where | ||
EntryMutexStd: EntrySync, | ||
EntrySingle: Entry, | ||
{ | ||
fn update_torrent_with_peer_and_get_stats(&self, info_hash: &InfoHash, peer: &peer::Peer) -> (bool, SwarmMetadata) { | ||
let entry = self.torrents.get_or_insert(*info_hash, Arc::default()); | ||
entry.value().insert_or_update_peer_and_get_stats(peer) | ||
} | ||
|
||
fn get(&self, key: &InfoHash) -> Option<EntryMutexStd> { | ||
let maybe_entry = self.torrents.get(key); | ||
maybe_entry.map(|entry| entry.value().clone()) | ||
} | ||
|
||
fn get_metrics(&self) -> TorrentsMetrics { | ||
let mut metrics = TorrentsMetrics::default(); | ||
|
||
for entry in &self.torrents { | ||
let stats = entry.value().lock().expect("it should get a lock").get_stats(); | ||
metrics.complete += u64::from(stats.complete); | ||
metrics.downloaded += u64::from(stats.downloaded); | ||
metrics.incomplete += u64::from(stats.incomplete); | ||
metrics.torrents += 1; | ||
} | ||
|
||
metrics | ||
} | ||
|
||
fn get_paginated(&self, pagination: Option<&Pagination>) -> Vec<(InfoHash, EntryMutexStd)> { | ||
match pagination { | ||
Some(pagination) => self | ||
.torrents | ||
.iter() | ||
.skip(pagination.offset as usize) | ||
.take(pagination.limit as usize) | ||
.map(|entry| (*entry.key(), entry.value().clone())) | ||
.collect(), | ||
None => self | ||
.torrents | ||
.iter() | ||
.map(|entry| (*entry.key(), entry.value().clone())) | ||
.collect(), | ||
} | ||
} | ||
|
||
fn import_persistent(&self, persistent_torrents: &PersistentTorrents) { | ||
for (info_hash, completed) in persistent_torrents { | ||
if self.torrents.contains_key(info_hash) { | ||
continue; | ||
} | ||
|
||
let entry = EntryMutexStd::new( | ||
EntrySingle { | ||
peers: BTreeMap::default(), | ||
downloaded: *completed, | ||
} | ||
.into(), | ||
); | ||
|
||
// Since SkipMap is lock-free the torrent could have been inserted | ||
// after checking if it exists. | ||
self.torrents.get_or_insert(*info_hash, entry); | ||
} | ||
} | ||
|
||
fn remove(&self, key: &InfoHash) -> Option<EntryMutexStd> { | ||
self.torrents.remove(key).map(|entry| entry.value().clone()) | ||
} | ||
|
||
fn remove_inactive_peers(&self, current_cutoff: DurationSinceUnixEpoch) { | ||
for entry in &self.torrents { | ||
entry.value().remove_inactive_peers(current_cutoff); | ||
} | ||
} | ||
|
||
fn remove_peerless_torrents(&self, policy: &TrackerPolicy) { | ||
for entry in &self.torrents { | ||
if entry.value().is_good(policy) { | ||
continue; | ||
} | ||
|
||
entry.remove(); | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters