perf: introduce URN to speed up large directory file updates (#1652)

yazi-core/src/manager/commands/bulk_rename.rs

@@ -6,7 +6,7 @@ use tokio::{fs::{self, OpenOptions}, io::{stdin, AsyncReadExt, AsyncWriteExt}};
 use yazi_config::{OPEN, PREVIEW};
 use yazi_dds::Pubsub;
 use yazi_proxy::{AppProxy, TasksProxy, HIDER, WATCHER};
-use yazi_shared::{fs::{max_common_root, maybe_exists, paths_to_same_file, File, FilesOp, Url}, terminal_clear};
+use yazi_shared::{fs::{max_common_root, maybe_exists, paths_to_same_file, File, Url}, terminal_clear};
 
 use crate::manager::Manager;
 
@@ -98,7 +98,8 @@ impl Manager {
 		// FIXME: consider old and new in the different directories
 		if !succeeded.is_empty() {
 			Pubsub::pub_from_bulk(succeeded.iter().map(|(u, f)| (u, f.url())).collect());
-			FilesOp::Upserting(cwd, succeeded).emit();
+			// FIXME 0
+			// FilesOp::Upserting(cwd, succeeded).emit();
 		}
 		drop(permit);
 

yazi-core/src/manager/commands/create.rs

@@ -1,10 +1,10 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use anyhow::Result;
 use tokio::fs;
 use yazi_config::popup::{ConfirmCfg, InputCfg};
 use yazi_proxy::{ConfirmProxy, InputProxy, TabProxy, WATCHER};
-use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, symlink_realpath, File, FilesOp, Url}};
+use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, realname, File, FilesOp, Url, UrnBuf}};
 
 use crate::manager::Manager;
 
@@ -48,9 +48,9 @@ impl Manager {
 
 		if dir {
 			fs::create_dir_all(&new).await?;
-		} else if let Ok(real) = symlink_realpath(&new).await {
+		} else if let Some(real) = realname(&new).await {
 			ok_or_not_found(fs::remove_file(&new).await)?;
-			FilesOp::Deleting(parent.clone(), vec![Url::from(real)]).emit();
+			FilesOp::Deleting(parent.clone(), HashSet::from_iter([UrnBuf::_from(real)])).emit();
 			fs::File::create(&new).await?;
 		} else {
 			fs::create_dir_all(&parent).await.ok();
@@ -59,7 +59,7 @@ impl Manager {
 		}
 
 		if let Ok(f) = File::from(new.clone()).await {
-			FilesOp::Upserting(parent, HashMap::from_iter([(f.url_owned(), f)])).emit();
+			FilesOp::Upserting(parent, HashMap::from_iter([(f.urn_owned(), f)])).emit();
 			TabProxy::reveal(&new)
 		}
 		Ok(())

yazi-core/src/manager/commands/rename.rs

@@ -1,11 +1,11 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use anyhow::Result;
 use tokio::fs;
 use yazi_config::popup::{ConfirmCfg, InputCfg};
 use yazi_dds::Pubsub;
 use yazi_proxy::{ConfirmProxy, InputProxy, TabProxy, WATCHER};
-use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, paths_to_same_file, symlink_realpath, File, FilesOp, Url}};
+use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, paths_to_same_file, realname, File, FilesOp, Url, UrnBuf}};
 
 use crate::manager::Manager;
 
@@ -74,22 +74,27 @@ impl Manager {
 	}
 
 	async fn rename_do(tab: usize, old: Url, new: Url) -> Result<()> {
-		let Some(p_old) = old.parent_url() else { return Ok(()) };
-		let Some(p_new) = new.parent_url() else { return Ok(()) };
+		let Some((p_old, n_old)) = old.pair() else { return Ok(()) };
+		let Some((p_new, n_new)) = new.pair() else { return Ok(()) };
 		let _permit = WATCHER.acquire().await.unwrap();
 
-		let overwritten = symlink_realpath(&new).await;
+		let overwritten = realname(&new).await;
 		fs::rename(&old, &new).await?;
 
-		if let Ok(o) = overwritten {
-			ok_or_not_found(fs::rename(&o, &new).await)?;
-			FilesOp::Deleting(p_new.clone(), vec![Url::from(o)]).emit();
+		if let Some(o) = overwritten {
+			ok_or_not_found(fs::rename(p_new.join(&o), &new).await)?;
+			FilesOp::Deleting(p_new.clone(), HashSet::from_iter([UrnBuf::_from(o)])).emit();
 		}
 		Pubsub::pub_from_rename(tab, &old, &new);
 
 		let file = File::from(new.clone()).await?;
-		FilesOp::Deleting(p_old, vec![old]).emit();
-		FilesOp::Upserting(p_new, HashMap::from_iter([(new.clone(), file)])).emit();
+		if p_new == p_old {
+			FilesOp::Upserting(p_old, HashMap::from_iter([(n_old, file)])).emit();
+		} else {
+			FilesOp::Deleting(p_old, HashSet::from_iter([n_old])).emit();
+			FilesOp::Upserting(p_new, HashMap::from_iter([(n_new, file)])).emit();
+		}
+
 		Ok(TabProxy::reveal(&new))
 	}
 

yazi-core/src/manager/commands/update_files.rs

@@ -26,7 +26,7 @@ impl Manager {
 
 		let mut ops = vec![opt.op];
 		for u in LINKED.read().from_dir(ops[0].cwd()) {
-			ops.push(ops[0].chroot(u));
+			ops.push(ops[0].rebase(u));
 		}
 
 		for op in ops {
@@ -60,10 +60,7 @@ impl Manager {
 
 	fn update_parent(tab: &mut Tab, op: Cow<FilesOp>) {
 		let urn = tab.cwd().urn_owned();
-		// FIXME
-		let leave = false;
-		// let leave = matches!(*op, FilesOp::Deleting(_, ref urls) if
-		// urls.contains(&urn));
+		let leave = matches!(*op, FilesOp::Deleting(_, ref urns) if urns.contains(&urn));
 
 		if let Some(f) = tab.parent.as_mut() {
 			render!(f.update(op.into_owned()));
@@ -111,8 +108,8 @@ impl Manager {
 	}
 
 	fn update_history(tab: &mut Tab, op: Cow<FilesOp>) {
-		let leave = tab.parent.as_ref().and_then(|f| f.loc.parent_url().map(|p| (&f.loc, p))).is_some_and(
-			|(p, pp)| matches!(*op, FilesOp::Deleting(ref parent, ref urls) if *parent == pp && urls.contains(p)),
+		let leave = tab.parent.as_ref().and_then(|f| f.loc.parent_url().map(|p| (p, f.loc.urn()))).is_some_and(
+			|(p, n)| matches!(*op, FilesOp::Deleting(ref parent, ref urns) if *parent == p && urns.contains(n)),
 		);
 
 		let folder = tab.history.entry(op.cwd().clone()).or_insert_with(|| Folder::from(op.cwd()));

yazi-core/src/manager/watcher.rs

@@ -9,7 +9,7 @@ use tracing::error;
 use yazi_fs::{Files, Folder};
 use yazi_plugin::isolate;
 use yazi_proxy::WATCHER;
-use yazi_shared::{fs::{symlink_realname, Cha, File, FilesOp, Url}, RoCell};
+use yazi_shared::{fs::{realname_unchecked, Cha, File, FilesOp, Url}, RoCell};
 
 use super::Linked;
 
@@ -124,27 +124,26 @@ impl Watcher {
 			let _permit = WATCHER.acquire().await.unwrap();
 			let mut reload = Vec::with_capacity(urls.len());
 
-			for url in urls {
-				let Some(name) = url.file_name() else { continue };
-				let Some(parent) = url.parent_url() else { continue };
-
-				let Ok(file) = File::from(url.clone()).await else {
-					FilesOp::Deleting(parent, vec![url]).emit();
+			for u in urls {
+				let Some((parent, urn)) = u.pair() else { continue };
+				let Ok(file) = File::from(u).await else {
+					FilesOp::Deleting(parent, HashSet::from_iter([urn])).emit();
 					continue;
 				};
 
-				let eq = (!file.is_link() && fs::canonicalize(&url).await.is_ok_and(|p| p == *url))
-					|| symlink_realname(&url, &mut cached).await.is_ok_and(|s| s == name);
+				let u = file.url();
+				let eq = (!file.is_link() && fs::canonicalize(u).await.is_ok_and(|p| p == **u))
+					|| realname_unchecked(u, &mut cached).await.is_ok_and(|s| s == urn._deref()._as_path());
 
 				if !eq {
-					FilesOp::Deleting(parent, vec![url]).emit();
+					FilesOp::Deleting(parent, HashSet::from_iter([urn])).emit();
 					continue;
 				}
 
 				if !file.is_dir() {
 					reload.push(file.clone());
 				}
-				FilesOp::Upserting(parent, HashMap::from_iter([(url, file)])).emit();
+				FilesOp::Upserting(parent, HashMap::from_iter([(urn, file)])).emit();
 			}
 
 			if reload.is_empty() {

yazi-core/src/manager/yanked.rs

@@ -39,17 +39,10 @@ impl Yanked {
 	}
 
 	pub fn apply_op(&mut self, op: &FilesOp) {
-		let (removal, addition) = match op {
-			FilesOp::Deleting(_, urls) => (urls.iter().collect(), vec![]),
-			FilesOp::Updating(_, urls) | FilesOp::Upserting(_, urls) => {
-				urls.iter().filter(|(u, _)| self.contains(u)).map(|(u, f)| (u, f.url_owned())).unzip()
-			}
-			_ => (vec![], vec![]),
-		};
-
+		let (removal, addition) = op.diff_recoverable(|u| self.contains(u));
 		if !removal.is_empty() {
 			let old = self.urls.len();
-			self.urls.retain(|u| !removal.contains(&u));
+			self.urls.retain(|u| !removal.contains(u));
 			self.revision += (old != self.urls.len()) as u64;
 		}
 

yazi-core/src/tab/selected.rs

@@ -18,23 +18,24 @@ impl Selected {
 	#[inline]
 	pub fn add(&mut self, url: &Url) -> bool { self.add_same(&[url]) == 1 }
 
-	pub fn add_many(&mut self, urls: &[&Url], same: bool) -> usize {
+	pub fn add_many(&mut self, urls: &[impl AsRef<Url>], same: bool) -> usize {
 		if same {
 			return self.add_same(urls);
 		}
 
 		let mut grouped: HashMap<_, Vec<_>> = Default::default();
-		for &u in urls {
-			if let Some(p) = u.parent_url() {
+		for u in urls {
+			if let Some(p) = u.as_ref().parent_url() {
 				grouped.entry(p).or_default().push(u);
 			}
 		}
 		grouped.into_values().map(|v| self.add_same(&v)).sum()
 	}
 
-	fn add_same(&mut self, urls: &[&Url]) -> usize {
+	fn add_same(&mut self, urls: &[impl AsRef<Url>]) -> usize {
 		// If it has appeared as a parent
-		let urls: Vec<_> = urls.iter().filter(|&&u| !self.parents.contains_key(u)).collect();
+		let urls: Vec<_> =
+			urls.iter().map(|u| u.as_ref()).filter(|&u| !self.parents.contains_key(u)).collect();
 		if urls.is_empty() {
 			return 0;
 		}
@@ -52,7 +53,7 @@ impl Selected {
 		}
 
 		let (now, len) = (timestamp_us(), self.inner.len());
-		self.inner.extend(urls.iter().enumerate().map(|(i, &&u)| (u.clone(), now + i as u64)));
+		self.inner.extend(urls.iter().enumerate().map(|(i, &u)| (u.clone(), now + i as u64)));
 
 		for u in parents {
 			*self.parents.entry(u).or_insert(0) += self.inner.len() - len;
@@ -103,14 +104,7 @@ impl Selected {
 	}
 
 	pub fn apply_op(&mut self, op: &FilesOp) {
-		let (removal, addition) = match op {
-			FilesOp::Deleting(_, urls) => (urls.iter().collect(), vec![]),
-			FilesOp::Updating(_, urls) | FilesOp::Upserting(_, urls) => {
-				urls.iter().filter(|&(u, _)| self.contains_key(u)).map(|(u, f)| (u, f.url())).unzip()
-			}
-			_ => (vec![], vec![]),
-		};
-
+		let (removal, addition) = op.diff_recoverable(|u| self.contains_key(u));
 		if !removal.is_empty() {
 			self.remove_many(&removal, !op.cwd().is_search());
 		}
@@ -196,7 +190,7 @@ mod tests {
 	fn insert_many_empty_urls_list() {
 		let mut s = Selected::default();
 
-		assert_eq!(0, s.add_same(&[]));
+		assert_eq!(0, s.add_same(&[] as &[&Url]));
 	}
 
 	#[test]

yazi-core/src/tasks/preload.rs

@@ -72,7 +72,7 @@ impl Tasks {
 			targets
 				.iter()
 				.filter(|f| {
-					f.is_dir() && !targets.sizes.contains_key(f.url()) && !loading.contains(f.url())
+					f.is_dir() && !targets.sizes.contains_key(f.urn()) && !loading.contains(f.url())
 				})
 				.map(|f| f.url())
 				.collect()

yazi-fm/src/lives/file.rs

@@ -39,7 +39,7 @@ impl File {
 
 			reg.add_field_method_get("idx", |_, me| Ok(me.idx + 1));
 			reg.add_method("size", |_, me, ()| {
-				Ok(if me.is_dir() { me.folder().files.sizes.get(me.url()).copied() } else { Some(me.len) })
+				Ok(if me.is_dir() { me.folder().files.sizes.get(me.urn()).copied() } else { Some(me.len) })
 			});
 			reg.add_method("mime", |lua, me, ()| {
 				let cx = lua.named_registry_value::<CtxRef>("cx")?;