feat: add streaming

2026-04-10 14:15:27 +02:00
parent 686f9ff747
commit 1a99490e22
4 changed files with 329 additions and 100 deletions
@@ -1,6 +1,9 @@
 use crate::errors::OllamaError;
+use axum::response::sse::Event;
+use futures::StreamExt;
 use reqwest::Client;
 use serde_json::{Value, json};
+use tokio_stream::wrappers::ReceiverStream;

 #[derive(Clone)]
 pub struct OllamaProvider {
@@ -66,6 +69,99 @@ impl OllamaProvider {
        }
    }

+    fn extract_completion_params<'a>(
+        &self,
+        body: &'a Value,
+    ) -> Result<(&'a str, &'a str), OllamaError> {
+        let prompt = body
+            .get("prompt")
+            .and_then(|v| v.as_str())
+            .filter(|s| !s.trim().is_empty())
+            .ok_or(OllamaError::MissingPrompt)?;
+
+        let model = body
+            .get("model")
+            .and_then(|v| v.as_str())
+            .unwrap_or("llama3");
+
+        Ok((prompt, model))
+    }
+
+    fn format_completion_response(&self, res: &Value) -> Value {
+        json!({
+            "id":      "cmpl-ollama",
+            "object":  "text_completion",
+            "model":   res.get("model"),
+            "choices": [{
+                "text":          res.get("response"),
+                "index":         0,
+                "finish_reason": if res.get("done").and_then(|v| v.as_bool()).unwrap_or(false) {
+                    "stop"
+                } else {
+                    "length"
+                },
+            }],
+            "usage": {
+                "prompt_tokens":     res.get("prompt_eval_count"),
+                "completion_tokens": res.get("eval_count"),
+                "total_tokens":      null,
+            }
+        })
+    }
+
+    fn extract_chat_params<'a>(
+        &self,
+        body: &'a Value,
+    ) -> Result<(&'a str, &'a Vec<Value>), OllamaError> {
+        let model = body
+            .get("model")
+            .and_then(|v| v.as_str())
+            .unwrap_or("llama3");
+
+        let messages = body
+            .get("messages")
+            .and_then(|v| v.as_array())
+            .filter(|arr| !arr.is_empty())
+            .ok_or(OllamaError::MissingMessages)?;
+
+        let has_user_msg = messages
+            .iter()
+            .any(|m| m.get("role").and_then(|r| r.as_str()) == Some("user"));
+
+        if !has_user_msg {
+            return Err(OllamaError::MissingMessages);
+        }
+
+        Ok((model, messages))
+    }
+
+    fn format_chat_response(&self, res: &Value) -> Value {
+        json!({
+            "id":      "chatcmpl-ollama",
+            "object":  "chat.completion",
+            "model":   res.get("model"),
+            "choices": [{
+                "index": 0,
+                "message": {
+                    "role":    res.get("message").and_then(|m| m.get("role")),
+                    "content": res.get("message").and_then(|m| m.get("content")),
+                },
+                "finish_reason": res
+                    .get("done_reason")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("stop"),
+            }],
+            "usage": {
+                "prompt_tokens":     res.get("prompt_eval_count"),
+                "completion_tokens": res.get("eval_count"),
+                "total_tokens": res.get("prompt_eval_count")
+                    .and_then(|p| p.as_u64())
+                    .zip(res.get("eval_count").and_then(|e| e.as_u64()))
+                    .map(|(p, e)| p + e),
+            }
+        })
+    }
+
    // ── public endpoints ─────────────────────────────────────────────────────

    pub async fn list_models(&self) -> Result<Value, OllamaError> {
@@ -133,19 +229,10 @@ impl OllamaProvider {
    }

    pub async fn completions(&self, body: Value) -> Result<Value, OllamaError> {
-        let prompt = body
-            .get("prompt")
-            .and_then(|v| v.as_str())
-            .filter(|s| !s.trim().is_empty())
-            .ok_or(OllamaError::MissingPrompt)?;
-
-        let model = body
-            .get("model")
-            .and_then(|v| v.as_str())
-            .unwrap_or("llama3");
+        let (prompt, model) = self.extract_completion_params(&body)?;
        self.validate_model(model).await?;

-        let ollama_payload = json!({
+        let payload = json!({
            "model":   model,
            "prompt":  prompt,
            "stream":  false,
@@ -155,56 +242,80 @@ impl OllamaProvider {
        let res = self
            .client
            .post(format!("{}/api/generate", self.base_url))
-            .json(&ollama_payload)
+            .json(&payload)
            .send()
            .await?
            .json::<Value>()
            .await?;

-        Ok(json!({
-            "id":      "cmpl-ollama",
-            "object":  "text_completion",
-            "model":   res.get("model"),
-            "choices": [{
-                "text":          res.get("response"),
-                "index":         0,
-                "finish_reason": if res.get("done").and_then(|v| v.as_bool()).unwrap_or(false) {
-                    "stop"
-                } else {
-                    "length"
-                },
-            }],
-            "usage": {
-                "prompt_tokens":     res.get("prompt_eval_count"),
-                "completion_tokens": res.get("eval_count"),
-                "total_tokens":      null,
+        Ok(self.format_completion_response(&res))
+    }
+
+    pub async fn completions_stream(
+        &self,
+        body: Value,
+    ) -> Result<ReceiverStream<Result<Event, OllamaError>>, OllamaError> {
+        let (prompt, model) = self.extract_completion_params(&body)?;
+        self.validate_model(model).await?;
+
+        let payload = json!({
+            "model":   model,
+            "prompt":  prompt,
+            "stream":  true,
+            "options": Self::build_options(&body),
+        });
+
+        let mut byte_stream = self
+            .client
+            .post(format!("{}/api/generate", self.base_url))
+            .json(&payload)
+            .send()
+            .await?
+            .bytes_stream();
+
+        let (tx, rx) = tokio::sync::mpsc::channel(32);
+
+        tokio::spawn(async move {
+            while let Some(chunk) = byte_stream.next().await {
+                let chunk = match chunk {
+                    Ok(b) => b,
+                    Err(e) => {
+                        let _ = tx.send(Err(OllamaError::Http(e))).await;
+                        break;
+                    }
+                };
+
+                if let Ok(json) = serde_json::from_slice::<Value>(&chunk) {
+                    let token = json.get("response").and_then(|v| v.as_str()).unwrap_or("");
+                    let done = json.get("done").and_then(|v| v.as_bool()).unwrap_or(false);
+
+                    // OpenAI-compatible SSE chunk
+                    let event_data = serde_json::to_string(&json!({
+                        "id":      "cmpl-ollama",
+                        "object":  "text_completion",
+                        "choices": [{ "text": token, "index": 0, "finish_reason": null }],
+                    }))
+                    .unwrap_or_default();
+
+                    let _ = tx.send(Ok(Event::default().data(event_data))).await;
+
+                    if done {
+                        // Final [DONE] sentinel — matches OpenAI streaming protocol
+                        let _ = tx.send(Ok(Event::default().data("[DONE]"))).await;
+                        break;
+                    }
+                }
            }
-        }))
+        });
+
+        Ok(ReceiverStream::new(rx))
    }

    pub async fn chat_completions(&self, body: Value) -> Result<Value, OllamaError> {
-        let model = body
-            .get("model")
-            .and_then(|v| v.as_str())
-            .unwrap_or("llama3");
-
-        let messages = body
-            .get("messages")
-            .and_then(|v| v.as_array())
-            .filter(|arr| !arr.is_empty())
-            .ok_or(OllamaError::MissingMessages)?;
-
-        let has_user_msg = messages
-            .iter()
-            .any(|m| m.get("role").and_then(|r| r.as_str()) == Some("user"));
-
-        if !has_user_msg {
-            return Err(OllamaError::MissingMessages);
-        }
-
+        let (model, messages) = self.extract_chat_params(&body)?;
        self.validate_model(model).await?;

-        let ollama_payload = json!({
+        let payload = json!({
            "model":    model,
            "messages": messages,
            "stream":   false,
@@ -214,35 +325,76 @@ impl OllamaProvider {
        let res = self
            .client
            .post(format!("{}/api/chat", self.base_url))
-            .json(&ollama_payload)
+            .json(&payload)
            .send()
            .await?
            .json::<Value>()
            .await?;

-        Ok(json!({
-            "id":      "chatcmpl-ollama",
-            "object":  "chat.completion",
-            "model":   res.get("model"),
-            "choices": [{
-                "index": 0,
-                "message": {
-                    "role":    res.get("message").and_then(|m| m.get("role")),
-                    "content": res.get("message").and_then(|m| m.get("content")),
-                },
-                "finish_reason": res
-                    .get("done_reason")
-                    .and_then(|v| v.as_str())
-                    .unwrap_or("stop"),
-            }],
-            "usage": {
-                "prompt_tokens":     res.get("prompt_eval_count"),
-                "completion_tokens": res.get("eval_count"),
-                "total_tokens": res.get("prompt_eval_count")
-                    .and_then(|p| p.as_u64())
-                    .zip(res.get("eval_count").and_then(|e| e.as_u64()))
-                    .map(|(p, e)| p + e),
+        Ok(self.format_chat_response(&res))
+    }
+
+    pub async fn chat_completions_stream(
+        &self,
+        body: Value,
+    ) -> Result<ReceiverStream<Result<Event, OllamaError>>, OllamaError> {
+        let (model, messages) = self.extract_chat_params(&body)?;
+        self.validate_model(model).await?;
+
+        let payload = json!({
+            "model":    model,
+            "messages": messages,
+            "stream":   true,
+            "options":  Self::build_options(&body),
+        });
+
+        let mut byte_stream = self
+            .client
+            .post(format!("{}/api/chat", self.base_url))
+            .json(&payload)
+            .send()
+            .await?
+            .bytes_stream();
+
+        let (tx, rx) = tokio::sync::mpsc::channel(32);
+
+        tokio::spawn(async move {
+            while let Some(chunk) = byte_stream.next().await {
+                let chunk = match chunk {
+                    Ok(b) => b,
+                    Err(e) => {
+                        let _ = tx.send(Err(OllamaError::Http(e))).await;
+                        break;
+                    }
+                };
+
+                if let Ok(json) = serde_json::from_slice::<Value>(&chunk) {
+                    let done = json.get("done").and_then(|v| v.as_bool()).unwrap_or(false);
+
+                    let event_data = serde_json::to_string(&json!({
+                        "id":      "chatcmpl-ollama",
+                        "object":  "chat.completion.chunk",
+                        "choices": [{
+                            "index": 0,
+                            "delta": {
+                                "role":    json.get("message").and_then(|m| m.get("role")),
+                                "content": json.get("message").and_then(|m| m.get("content")),
+                            },
+                            "finish_reason": if done { json!("stop") } else { json!(null) },
+                        }],
+                    }))
+                    .unwrap_or_default();
+
+                    let _ = tx.send(Ok(Event::default().data(event_data))).await;
+
+                    if done {
+                        let _ = tx.send(Ok(Event::default().data("[DONE]"))).await;
+                        break;
+                    }
+                }
            }
-        }))
+        });
+
+        Ok(ReceiverStream::new(rx))
    }
 }