Implement speculative caching.

Ai00-X · Jun 10, 2024 · 5df6f0a · 5df6f0a
1 parent 4cf05db
commit 5df6f0a
Show file tree

Hide file tree

Showing 2 changed files with 147 additions and 77 deletions.
diff --git a/crates/ai00-core/src/lib.rs b/crates/ai00-core/src/lib.rs
@@ -702,7 +702,7 @@ pub async fn model_route(receiver: Receiver<ThreadRequest>) -> Result<()> {
 
  let context = GenerateContext {
  prompt_tokens: tokens.to_vec(),
- prompt_cached: false,
+ prompt_cached: Default::default(),
  prefix: Default::default(),
  suffix: tokens,
  output: None,
@@ -720,8 +720,9 @@ pub async fn model_route(receiver: Receiver<ThreadRequest>) -> Result<()> {
  let queue = queue.clone();
  let sender = sender.clone();
  tokio::spawn(async move {
+ let context = &mut env.read().await.enqueue(context).await;
  let mut queue = queue.lock().await;
- queue.append(&mut env.read().await.enqueue(context).await);
+ queue.append(context);
  let _ = sender.send(());
  });
  }