feat: prefetcher (#1061)

README.md

@@ -12,7 +12,7 @@ Yazi (means "duck") is a terminal file manager written in Rust, based on non-blo
 - 💪 **Powerful Async Task Scheduling and Management**: Provides real-time progress updates, task cancellation, and internal task priority assignment.
 - 🖼️ **Built-in Support for Multiple Image Protocols**: Also integrated with Überzug++, covering almost all terminals.
 - 🌟 **Built-in Code Highlighting and Image Decoding**: Combined with the pre-loading mechanism, greatly accelerates image and normal file loading.
-- 🔌 **Concurrent Plugin System**: UI plugins (rewriting most of the UI), functional plugins, custom previewer, and custom preloader; Just some pieces of Lua.
+- 🔌 **Concurrent Plugin System**: UI plugins (rewriting most of the UI), functional plugins, custom previewer/preloader/prefetcher; Just some pieces of Lua.
 - 📡 **Data Distribution Service**: Built on a client-server architecture (no additional server process required), integrated with a Lua-based publish-subscribe model, achieving cross-instance communication and state persistence.
 - 📦 **Package Manager**: Install plugins and themes with one command, keeping them always up to date, or pin them to a specific version.
 - 🧰 Integration with fd, rg, fzf, zoxide

cspell.json

@@ -1 +1 @@
-{"language":"en","flagWords":[],"words":["Punct","KEYMAP","splitn","crossterm","YAZI","unar","peekable","ratatui","syntect","pbpaste","pbcopy","ffmpegthumbnailer","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","nvim","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","️ Überzug","️ Überzug","Konsole","Alacritty","Überzug","pkgs","paru","unarchiver","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","imagesize","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick"],"version":"0.2"}
+{"words":["Punct","KEYMAP","splitn","crossterm","YAZI","unar","peekable","ratatui","syntect","pbpaste","pbcopy","ffmpegthumbnailer","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","nvim","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","️ Überzug","️ Überzug","Konsole","Alacritty","Überzug","pkgs","paru","unarchiver","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","imagesize","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick","prefetcher","Prework","prefetchers","PREWORKERS"],"version":"0.2","language":"en","flagWords":[]}

yazi-config/preset/yazi.toml

@@ -79,8 +79,11 @@ suppress_preload = false
 
 [plugin]
 
+prefetchers = [
+ # Mimetype
+ { name = "*", cond = "!mime", run = "mime", prio = "high" },
+]
 preloaders = [
- { name = "*", cond = "!mime", run = "mime", next = true, multi = true, prio = "high" },
  # Image
  { mime = "image/svg+xml", run = "magick" },
  { mime = "image/heic", run = "magick" },

yazi-config/src/plugin/mod.rs

@@ -1,9 +1,11 @@
 mod plugin;
+mod prefetcher;
 mod preloader;
 mod previewer;
 
 pub use plugin::*;
+pub use prefetcher::*;
 pub use preloader::*;
 pub use previewer::*;
 
-pub const MAX_PRELOADERS: u8 = 32;
+pub const MAX_PREWORKERS: u8 = 32;

yazi-config/src/plugin/plugin.rs

@@ -3,13 +3,14 @@ use std::path::Path;
 use serde::Deserialize;
 use yazi_shared::MIME_DIR;
 
-use super::{Preloader, Previewer};
-use crate::{plugin::MAX_PRELOADERS, Preset, MERGED_YAZI};
+use super::{Prefetcher, Preloader, Previewer};
+use crate::{plugin::MAX_PREWORKERS, Preset, MERGED_YAZI};
 
 #[derive(Deserialize)]
 pub struct Plugin {
- pub preloaders: Vec<Preloader>,
- pub previewers: Vec<Previewer>,
+ pub prefetchers: Vec<Prefetcher>,
+ pub preloaders: Vec<Preloader>,
+ pub previewers: Vec<Previewer>,
 }
 
 impl Default for Plugin {
@@ -21,6 +22,12 @@ impl Default for Plugin {
 
  #[derive(Deserialize)]
  struct Shadow {
+ prefetchers: Vec<Prefetcher>,
+ #[serde(default)]
+ prepend_prefetchers: Vec<Prefetcher>,
+ #[serde(default)]
+ append_prefetchers: Vec<Prefetcher>,
+
  preloaders: Vec<Preloader>,
  #[serde(default)]
  prepend_preloaders: Vec<Preloader>,
@@ -42,36 +49,53 @@ impl Default for Plugin {
  shadow.previewers.retain(|r| !r.any_dir());
  }
 
+ Preset::mix(&mut shadow.prefetchers, shadow.prepend_prefetchers, shadow.append_prefetchers);
  Preset::mix(&mut shadow.preloaders, shadow.prepend_preloaders, shadow.append_preloaders);
  Preset::mix(&mut shadow.previewers, shadow.prepend_previewers, shadow.append_previewers);
 
- if shadow.preloaders.len() > MAX_PRELOADERS as usize {
- panic!("Too many preloaders");
+ if shadow.prefetchers.len() + shadow.preloaders.len() > MAX_PREWORKERS as usize {
+ panic!("Prefetchers and preloaders exceed the limit of {MAX_PREWORKERS}");
  }
 
- for (i, preloader) in shadow.preloaders.iter_mut().enumerate() {
- preloader.id = i as u8;
+ for (i, p) in shadow.prefetchers.iter_mut().enumerate() {
+ p.id = i as u8;
+ }
+ for (i, p) in shadow.preloaders.iter_mut().enumerate() {
+ p.id = shadow.prefetchers.len() as u8 + i as u8;
  }
 
- Self { preloaders: shadow.preloaders, previewers: shadow.previewers }
+ Self {
+ prefetchers: shadow.prefetchers,
+ preloaders: shadow.preloaders,
+ previewers: shadow.previewers,
+ }
  }
 }
 
 impl Plugin {
- pub fn preloaders(
+ pub fn prefetchers(
  &self,
  path: &Path,
  mime: Option<&str>,
  f: impl Fn(&str) -> bool + Copy,
- ) -> Vec<&Preloader> {
+ ) -> Vec<&Prefetcher> {
+ let is_dir = mime == Some(MIME_DIR);
+ self
+ .prefetchers
+ .iter()
+ .filter(|&p| {
+ p.cond.as_ref().and_then(|c| c.eval(f)) != Some(false)
+ && (p.mime.as_ref().zip(mime).map_or(false, |(p, m)| p.match_mime(m))
+ || p.name.as_ref().is_some_and(|p| p.match_path(path, is_dir)))
+ })
+ .collect()
+ }
+
+ pub fn preloaders(&self, path: &Path, mime: Option<&str>) -> Vec<&Preloader> {
  let is_dir = mime == Some(MIME_DIR);
  let mut preloaders = Vec::with_capacity(1);
 
  for p in &self.preloaders {
- if p.cond.as_ref().and_then(|c| c.eval(f)) == Some(false) {
- continue;
- }
-
  if !p.mime.as_ref().zip(mime).map_or(false, |(p, m)| p.match_mime(m))
  && !p.name.as_ref().is_some_and(|p| p.match_path(path, is_dir))
  {

yazi-config/src/plugin/prefetcher.rs

@@ -0,0 +1,29 @@
+use serde::Deserialize;
+use yazi_shared::{event::Cmd, Condition};
+
+use crate::{Pattern, Priority};
+
+#[derive(Debug, Deserialize)]
+pub struct Prefetcher {
+ #[serde(skip)]
+ pub id: u8,
+ pub cond: Option<Condition>,
+ pub name: Option<Pattern>,
+ pub mime: Option<Pattern>,
+ pub run: Cmd,
+ #[serde(default)]
+ pub prio: Priority,
+}
+
+#[derive(Debug, Clone)]
+pub struct PrefetcherProps {
+ pub id: u8,
+ pub name: String,
+ pub prio: Priority,
+}
+
+impl From<&Prefetcher> for PrefetcherProps {
+ fn from(prefetcher: &Prefetcher) -> Self {
+ Self { id: prefetcher.id, name: prefetcher.run.name.to_owned(), prio: prefetcher.prio }
+ }
+}

yazi-config/src/plugin/preloader.rs

@@ -1,39 +1,30 @@
 use serde::Deserialize;
-use yazi_shared::{event::Cmd, Condition};
+use yazi_shared::event::Cmd;
 
 use crate::{Pattern, Priority};
 
 #[derive(Debug, Deserialize)]
 pub struct Preloader {
  #[serde(skip)]
- pub id: u8,
- pub cond: Option<Condition>,
- pub name: Option<Pattern>,
- pub mime: Option<Pattern>,
- pub run: Cmd,
+ pub id: u8,
+ pub name: Option<Pattern>,
+ pub mime: Option<Pattern>,
+ pub run: Cmd,
  #[serde(default)]
- pub next:  bool,
+ pub next: bool,
  #[serde(default)]
- pub multi: bool,
- #[serde(default)]
- pub prio: Priority,
+ pub prio: Priority,
 }
 
 #[derive(Debug, Clone)]
 pub struct PreloaderProps {
- pub id: u8,
- pub name: String,
- pub multi: bool,
- pub prio: Priority,
+ pub id: u8,
+ pub name: String,
+ pub prio: Priority,
 }
 
 impl From<&Preloader> for PreloaderProps {
  fn from(preloader: &Preloader) -> Self {
- Self {
- id: preloader.id,
- name: preloader.run.name.to_owned(),
- multi: preloader.multi,
- prio: preloader.prio,
- }
+ Self { id: preloader.id, name: preloader.run.name.to_owned(), prio: preloader.prio }
  }
 }

yazi-core/src/manager/commands/open.rs

@@ -63,8 +63,8 @@ impl Manager {
  }
 
  done.extend(files.iter().map(|f| (f.url(), String::new())));
- if let Err(e) = isolate::preload("mime", files, true).await {
- error!("preload in open failed: {e}");
+ if let Err(e) = isolate::prefetch("mime", files).await {
+ error!("prefetch `mime` failed in opening: {e}");
  }
 
  ManagerProxy::open_do(OpenDoOpt { hovered, targets: done, interactive: opt.interactive });

yazi-core/src/manager/commands/refresh.rs

@@ -35,6 +35,6 @@ impl Manager {
  self.hover(None);
  self.update_paged((), tasks);
 
- tasks.preload_sorted(&self.current().files);
+ tasks.prework_sorted(&self.current().files);
  }
 }

yazi-core/src/manager/commands/update_files.rs

@@ -88,7 +88,7 @@ impl Manager {
  ManagerProxy::hover(None); // Re-hover in next loop
  ManagerProxy::update_paged(); // Update for paged files in next loop
  if calc {
- tasks.preload_sorted(&tab.current.files);
+ tasks.prework_sorted(&tab.current.files);
  }
  }
 

yazi-core/src/manager/commands/update_mimetype.rs

@@ -52,7 +52,7 @@ impl Manager {
  self.mimetype.extend(updates);
  self.peek(false);
 
- tasks.preload_affected(&affected, &self.mimetype);
+ tasks.prework_affected(&affected, &self.mimetype);
  render!();
  }
 }

yazi-core/src/manager/commands/update_paged.rs

@@ -32,6 +32,7 @@ impl Manager {
  }
 
  let targets = self.current().paginate(opt.page.unwrap_or(self.current().page));
+ tasks.prefetch_paged(targets, &self.mimetype);
  tasks.preload_paged(targets, &self.mimetype);
  }
 }

yazi-core/src/manager/watcher.rs

@@ -117,8 +117,8 @@ impl Watcher {
  if reload.is_empty() {
  continue;
  }
- if let Err(e) = isolate::preload("mime", reload, true).await {
- error!("preload in watcher failed: {e}");
+ if let Err(e) = isolate::prefetch("mime", reload).await {
+ error!("prefetch `mime` failed in watcher: {e}");
  }
  }
  }

yazi-core/src/tab/commands/sort.rs

@@ -18,6 +18,6 @@ impl Tab {
  self.apply_files_attrs();
  ManagerProxy::update_paged();
 
- tasks.preload_sorted(&self.current.files);
+ tasks.prework_sorted(&self.current.files);
  }
 }