diff --git a/Cargo.toml b/Cargo.toml index 258914cd70..9e496ec9a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "screenpipe-audio", "screenpipe-server", "screenpipe-integrations", - "screenpipe-actions", "screenpipe-events", ] exclude = ["screenpipe-app-tauri/src-tauri"] diff --git a/screenpipe-actions/Cargo.toml b/screenpipe-actions/Cargo.toml deleted file mode 100644 index 7f4babb4ba..0000000000 --- a/screenpipe-actions/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "screenpipe-actions" -version = { workspace = true } -authors = { workspace = true } -description = { workspace = true } -repository = { workspace = true } -license = { workspace = true } -edition = { workspace = true } - -[lib] -name = "screenpipe_actions" -path = "src/lib.rs" - -[[bin]] -name = "screenpipe-actions" -path = "src/main.rs" - -[dependencies] -anyhow = "1.0" -reqwest = { version = "0.11", features = ["json"] } -serde_json = "1.0" -tokio = { version = "1.26", features = ["full"] } -rdev = "0.5.3" -enigo = "0.1.3" -serde = { version = "1.0", features = ["derive"] } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } diff --git a/screenpipe-actions/ai-proxy/bun.lockb b/screenpipe-actions/ai-proxy/bun.lockb deleted file mode 100755 index fc36b6d83a..0000000000 Binary files a/screenpipe-actions/ai-proxy/bun.lockb and /dev/null differ diff --git a/screenpipe-actions/src/call_ai.rs b/screenpipe-actions/src/call_ai.rs deleted file mode 100644 index e9396349ab..0000000000 --- a/screenpipe-actions/src/call_ai.rs +++ /dev/null @@ -1,259 +0,0 @@ -use anyhow::{Result, Context}; -use reqwest::Client; -use serde_json::{json, Value}; -// use base64::{engine::general_purpose, Engine as _}; -// use image::{DynamicImage, ImageFormat}; -// use std::io::Cursor; - -pub async fn call_ai(prompt: String, context: String, expect_json: bool) -> Result { - let client = Client::new(); - - let messages = vec![ - json!({ - "role": "system", - "content": context - }), - json!({ - "role": "user", - "content": prompt - }) - ]; - - let mut body = json!({ - "model": "gpt-4o", - "messages": messages, - "temperature": 0.2, - "stream": false - }); - - if expect_json { - body["response_format"] = json!({"type": "json_object"}); - } - - let response: Value = client.post("https://ai-proxy.i-f9f.workers.dev/v1/chat/completions") - .json(&body) - .send() - .await? - .json() - .await?; - - // Log the entire response for debugging - // println!("raw api response: {}", serde_json::to_string_pretty(&response)?); - - if response.get("error").is_some() { - // If there's an error, return it as a string - let error_message = response["error"]["message"] - .as_str() - .unwrap_or("Unknown error") - .to_string(); - return Err(anyhow::anyhow!(error_message)); - } - - let content = response["choices"] - .get(0) - .and_then(|choice| choice["message"]["content"].as_str()) - .context("failed to extract content from response")? - .to_string(); - - if expect_json { - let json_value: Value = serde_json::from_str(&content) - .context("response content is not valid JSON")?; - - if let Some(array) = json_value["response"].as_array() { - // If "response" is an array, join its elements with newlines - Ok(array.iter() - .filter_map(|v| v.as_str()) - .collect::>() - .join("\n")) - } else if let Some(response_str) = json_value["response"].as_str() { - // If "response" is a string, return it directly - Ok(response_str.to_string()) - } else { - // If "response" is neither an array nor a string, return the whole JSON - Ok(content) - } - } else { - Ok(content) - } -} - -// Commented out unused enum and functions -// pub enum AIProvider { -// OpenAI, -// Claude, -// } - -// pub async fn call_ai_with_screenshot(provider: AIProvider, prompt: String, expect_json: bool, screenshot: DynamicImage) -> Result { -// match provider { -// AIProvider::OpenAI => call_openai_with_screenshot(prompt, expect_json, screenshot).await, -// AIProvider::Claude => call_claude_with_screenshot(prompt, expect_json, screenshot).await, -// } -// } - -// pub async fn call_openai_with_screenshot(prompt: String, expect_json: bool, screenshot: DynamicImage) -> Result { -// dotenv().ok(); // Load .env file -// let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY not set"); - -// let client = Client::new(); - -// // Convert image to base64 -// let mut image_buffer = Vec::new(); -// screenshot.write_to(&mut Cursor::new(&mut image_buffer), ImageFormat::Png) -// .context("failed to write image to buffer")?; -// let base64_image = general_purpose::STANDARD.encode(&image_buffer); - -// let messages = vec![ -// json!({ -// "role": "user", -// "content": [ -// { -// "type": "text", -// "text": prompt -// }, -// { -// "type": "image_url", -// "image_url": { -// "url": format!("data:image/png;base64,{}", base64_image) -// } -// } -// ] -// }) -// ]; - -// let mut body = json!({ -// "model": "gpt-4o", -// "messages": messages, -// "temperature": 0.2 -// }); - -// if expect_json { -// body["response_format"] = json!({"type": "json_object"}); -// } - -// let response: Value = client.post("https://api.openai.com/v1/chat/completions") -// .header("Authorization", format!("Bearer {}", api_key)) -// .json(&body) -// .send() -// .await? -// .json() -// .await?; - -// // Log the entire response for debugging -// // println!("raw api response: {}", serde_json::to_string_pretty(&response)?); - -// if response.get("error").is_some() { -// let error_message = response["error"]["message"] -// .as_str() -// .unwrap_or("unknown error") -// .to_string(); -// return Err(anyhow::anyhow!(error_message)); -// } - -// let choice = response["choices"] -// .get(0) -// .ok_or_else(|| anyhow::anyhow!("no choices in response"))?; - -// if let Some(refusal) = choice["message"]["refusal"].as_str() { -// return Ok(refusal.to_string()); -// } - -// let content = choice["message"]["content"] -// .as_str() -// .context("failed to extract content from response")? -// .to_string(); - -// if expect_json { -// serde_json::from_str::(&content) -// .context("response content is not valid JSON")?; -// } - -// Ok(content) -// } - -// pub async fn call_claude_with_screenshot(prompt: String, expect_json: bool, screenshot: DynamicImage) -> Result { -// dotenv().ok(); // load .env file -// let api_key = std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY not set")?; - -// let client = Client::new(); - -// // Convert image to base64 -// let mut image_buffer = Vec::new(); -// screenshot.write_to(&mut Cursor::new(&mut image_buffer), ImageFormat::Png) -// .context("failed to write image to buffer")?; -// let base64_image = general_purpose::STANDARD.encode(&image_buffer); - -// let system_message = if expect_json { -// "You must respond in valid JSON format. Always wrap your response in a JSON object with a 'response' key." -// } else { -// "" -// }; - -// let user_message = if expect_json { -// format!("{}\nRemember to format your entire response as a JSON object with a 'response' key.", prompt) -// } else { -// prompt -// }; - -// let body = json!({ -// "model": "claude-3-5-sonnet-20240620", -// "max_tokens": 1024, -// "temperature": 0.2, -// "system": system_message, -// "messages": [ -// { -// "role": "user", -// "content": [ -// { -// "type": "image", -// "source": { -// "type": "base64", -// "media_type": "image/png", -// "data": base64_image -// } -// }, -// { -// "type": "text", -// "text": user_message -// } -// ] -// } -// ] -// }); - -// let response: Value = client.post("https://api.anthropic.com/v1/messages") -// .header("x-api-key", &api_key) -// .header("anthropic-version", "2023-06-01") -// .header("content-type", "application/json") -// .json(&body) -// .send() -// .await? -// .json() -// .await?; - -// // log the entire response for debugging -// println!("raw api response: {}", serde_json::to_string_pretty(&response)?); - -// if response.get("error").is_some() { -// let error_message = response["error"]["message"] -// .as_str() -// .unwrap_or("unknown error") -// .to_string(); -// return Err(anyhow::anyhow!(error_message)); -// } - -// let content = response["content"] -// .get(0) -// .and_then(|content| content["text"].as_str()) -// .context("failed to extract content from response")?; - -// if expect_json { -// // Validate that the content is valid JSON -// serde_json::from_str::(content) -// .context("response content is not valid JSON")?; - -// // Return the full JSON string -// Ok(content.to_string()) -// } else { -// Ok(content.to_string()) -// } -// } diff --git a/screenpipe-actions/src/lib.rs b/screenpipe-actions/src/lib.rs deleted file mode 100644 index b50ed4e496..0000000000 --- a/screenpipe-actions/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod call_ai; -mod monitor_keystroke_commands; -mod run; -pub mod type_and_animate; - -pub use call_ai::call_ai; -pub use monitor_keystroke_commands::{run_keystroke_monitor, KeystrokeCommand}; -pub use run::run; -pub use type_and_animate::{delete_characters, trigger_keyboard_permission, type_slowly}; // Export the run function from the run module diff --git a/screenpipe-actions/src/main.rs b/screenpipe-actions/src/main.rs deleted file mode 100644 index 411757c97a..0000000000 --- a/screenpipe-actions/src/main.rs +++ /dev/null @@ -1,11 +0,0 @@ -use screenpipe_actions::run; -use tracing::Level; - -fn main() -> anyhow::Result<()> { - // try to enable tracing subscriber if not already running - let subscriber = tracing_subscriber::fmt::Subscriber::builder() - .with_max_level(Level::INFO) - .finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - tokio::runtime::Runtime::new()?.block_on(run()) -} diff --git a/screenpipe-actions/src/monitor_keystroke_commands.rs b/screenpipe-actions/src/monitor_keystroke_commands.rs deleted file mode 100644 index 9f61173cb5..0000000000 --- a/screenpipe-actions/src/monitor_keystroke_commands.rs +++ /dev/null @@ -1,101 +0,0 @@ -use rdev::{listen, Event, EventType, Key}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use tokio::sync::{mpsc, Mutex}; -use tracing::error; - -pub enum KeystrokeCommand { - DoubleSlash, - Stop, - // Add other commands as needed -} - -pub struct KeystrokeMonitor { - tx: mpsc::Sender, - last_slash: Arc>>, - shift_pressed: Arc, - ctrl_pressed: Arc, -} - -impl KeystrokeMonitor { - pub fn new(tx: mpsc::Sender) -> Self { - KeystrokeMonitor { - tx, - last_slash: Arc::new(Mutex::new(None)), - shift_pressed: Arc::new(AtomicBool::new(false)), - ctrl_pressed: Arc::new(AtomicBool::new(false)), - } - } - - pub async fn start_monitoring(&self) -> anyhow::Result<()> { - let tx = self.tx.clone(); - let last_slash = self.last_slash.clone(); - let shift_pressed = self.shift_pressed.clone(); - let ctrl_pressed = self.ctrl_pressed.clone(); - tokio::task::spawn_blocking(move || { - if let Err(error) = listen(move |event| { - let _ = handle_event(event, &tx, &last_slash, &shift_pressed, &ctrl_pressed); - }) { - error!("error: {:?}", error) - } - }); - - Ok(()) - } -} - -fn handle_event( - event: Event, - tx: &mpsc::Sender, - last_slash: &Arc>>, - shift_pressed: &Arc, - ctrl_pressed: &Arc, -) -> anyhow::Result<()> { - match event.event_type { - EventType::KeyPress(key) => { - match key { - Key::ShiftLeft | Key::ShiftRight => { - shift_pressed.store(true, Ordering::SeqCst); - } - Key::ControlLeft | Key::ControlRight => { - ctrl_pressed.store(true, Ordering::SeqCst); - } - Key::KeyQ => { - tx.blocking_send(KeystrokeCommand::Stop)?; - return Ok(()); - } - Key::Slash | Key::Num7 if shift_pressed.load(Ordering::SeqCst) => { - // Monitoring for slash (/) or Shift+7 combination - let mut last_slash_guard = last_slash.blocking_lock(); - let now = std::time::Instant::now(); - if let Some(last) = *last_slash_guard { - if now.duration_since(last).as_millis() < 500 { - tx.blocking_send(KeystrokeCommand::DoubleSlash)?; - *last_slash_guard = None; - return Ok(()); - } - } - *last_slash_guard = Some(now); - } - _ => {} - } - } - EventType::KeyRelease(key) => match key { - Key::ShiftLeft | Key::ShiftRight => { - shift_pressed.store(false, Ordering::SeqCst); - } - Key::ControlLeft | Key::ControlRight => { - ctrl_pressed.store(false, Ordering::SeqCst); - } - _ => {} - }, - _ => {} - } - Ok(()) -} - -pub async fn run_keystroke_monitor(tx: mpsc::Sender) -> anyhow::Result<()> { - let monitor = KeystrokeMonitor::new(tx); - monitor.start_monitoring().await?; - Ok(()) -} diff --git a/screenpipe-actions/src/print_all_attributes.swift b/screenpipe-actions/src/print_all_attributes.swift deleted file mode 100644 index 4b753e0201..0000000000 --- a/screenpipe-actions/src/print_all_attributes.swift +++ /dev/null @@ -1,169 +0,0 @@ -import Cocoa -import ApplicationServices -import Foundation - -class QueueElement { - let element: AXUIElement - let depth: Int - - init(_ element: AXUIElement, depth: Int) { - self.element = element - self.depth = depth - } -} - -func printAllAttributeValues(_ startElement: AXUIElement) { - var elements: [(CGPoint, CGSize, String)] = [] - var visitedElements = Set() - let unwantedValues = ["0", "", "", "3", ""] - let unwantedLabels = [ - "window", "application", "group", "button", "image", "text", - "pop up button", "region", "notifications", "table", "column", - "html content" - ] - - func traverseHierarchy(_ element: AXUIElement, depth: Int) { - guard !visitedElements.contains(element) else { return } - visitedElements.insert(element) - - var attributeNames: CFArray? - let result = AXUIElementCopyAttributeNames(element, &attributeNames) - - guard result == .success, let attributes = attributeNames as? [String] else { return } - - var position: CGPoint = .zero - var size: CGSize = .zero - - // Get position - if let positionValue = getAttributeValue(element, forAttribute: kAXPositionAttribute) as! AXValue?, - AXValueGetType(positionValue) == .cgPoint { - AXValueGetValue(positionValue, .cgPoint, &position) - } - - // Get size - if let sizeValue = getAttributeValue(element, forAttribute: kAXSizeAttribute) as! AXValue?, - AXValueGetType(sizeValue) == .cgSize { - AXValueGetValue(sizeValue, .cgSize, &size) - } - - for attr in attributes { - if ["AXDescription", "AXValue", "AXLabel", "AXRoleDescription", "AXHelp"].contains(attr) { - if let value = getAttributeValue(element, forAttribute: attr) { - let valueStr = describeValue(value) - if !valueStr.isEmpty && !unwantedValues.contains(valueStr) && valueStr.count > 1 && - !unwantedLabels.contains(valueStr.lowercased()) { - elements.append((position, size, valueStr)) - } - } - } - - // Traverse child elements - if let childrenValue = getAttributeValue(element, forAttribute: attr) { - if let elementArray = childrenValue as? [AXUIElement] { - for childElement in elementArray { - traverseHierarchy(childElement, depth: depth + 1) - } - } else if let childElement = childrenValue as! AXUIElement? { - traverseHierarchy(childElement, depth: depth + 1) - } - } - } - } - - traverseHierarchy(startElement, depth: 0) - - // Sort elements from top to bottom, then left to right - elements.sort { (a, b) in - if a.0.y != b.0.y { - return a.0.y < b.0.y - } else { - return a.0.x < b.0.x - } - } - - // Deduplicate and print sorted elements to stdout, excluding coordinates - var uniqueValues = Set() - for (_, _, valueStr) in elements { - if uniqueValues.insert(valueStr).inserted { - print(valueStr) - } - } -} - -func formatCoordinates(_ position: CGPoint, _ size: CGSize) -> String { - return String(format: "(x:%.0f,y:%.0f,w:%.0f,h:%.0f)", position.x, position.y, size.width, size.height) -} - -func describeValue(_ value: AnyObject?) -> String { - switch value { - case let string as String: - return string - case let number as NSNumber: - return number.stringValue - case let point as NSPoint: - return "(\(point.x), \(point.y))" - case let size as NSSize: - return "w=\(size.width) h=\(size.height)" - case let rect as NSRect: - return "x=\(rect.origin.x) y=\(rect.origin.y) w=\(rect.size.width) h=\(rect.size.height)" - case let range as NSRange: - return "loc=\(range.location) len=\(range.length)" - case let url as URL: - return url.absoluteString - case let array as [AnyObject]: - return array.isEmpty ? "Empty array" : "Array with \(array.count) elements" - case let axValue as AXValue: - return describeAXValue(axValue) - case is AXUIElement: - return "AXUIElement" - case .none: - return "None" - default: - return String(describing: value) - } -} - -func describeAXValue(_ axValue: AXValue) -> String { - let type = AXValueGetType(axValue) - switch type { - case .cgPoint: - var point = CGPoint.zero - AXValueGetValue(axValue, .cgPoint, &point) - return "(\(point.x), \(point.y))" - case .cgSize: - var size = CGSize.zero - AXValueGetValue(axValue, .cgSize, &size) - return "w=\(size.width) h=\(size.height)" - case .cgRect: - var rect = CGRect.zero - AXValueGetValue(axValue, .cgRect, &rect) - return "x=\(rect.origin.x) y=\(rect.origin.y) w=\(rect.size.width) h=\(rect.size.height)" - case .cfRange: - var range = CFRange(location: 0, length: 0) - AXValueGetValue(axValue, .cfRange, &range) - return "loc=\(range.location) len=\(range.length)" - default: - return "Unknown AXValue type" - } -} - -func getAttributeValue(_ element: AXUIElement, forAttribute attr: String) -> AnyObject? { - var value: AnyObject? - let result = AXUIElementCopyAttributeValue(element, attr as CFString, &value) - return result == .success ? value : nil -} - -func printAllAttributeValuesForCurrentApp() { - guard let app = NSWorkspace.shared.frontmostApplication else { - return - } - - let pid = app.processIdentifier - let axApp = AXUIElementCreateApplication(pid) - - print("attribute values for \(app.localizedName ?? "unknown app"):") - printAllAttributeValues(axApp) -} - -// usage -printAllAttributeValuesForCurrentApp() diff --git a/screenpipe-actions/src/run.rs b/screenpipe-actions/src/run.rs deleted file mode 100644 index 9c68ddfbb6..0000000000 --- a/screenpipe-actions/src/run.rs +++ /dev/null @@ -1,491 +0,0 @@ -use crate::type_and_animate::{delete_characters, type_slowly, EnigoCommand}; -use crate::{call_ai, run_keystroke_monitor, KeystrokeCommand}; -use std::path::Path; -use std::string::ToString; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use tokio::io::AsyncWriteExt; -use tokio::process::Command; -use tokio::sync::mpsc; -use tokio::task; -use tokio::time::Instant; -use tracing::{error, info}; - -// TODO: make this function not prevent ctrl+c in the future -pub async fn run() -> anyhow::Result<()> { - info!("starting keystroke monitor. press '//' to print attributes for the current app and call openai. press 'ctrl+q' to stop."); - let (tx, mut rx) = mpsc::channel(100); - - let (_enigo_tx, mut enigo_rx) = mpsc::channel(100); - - // Spawn the Enigo handler thread - task::spawn_blocking(move || { - use enigo::KeyboardControllable; - let mut enigo = enigo::Enigo::new(); - while let Some(command) = enigo_rx.blocking_recv() { - match command { - EnigoCommand::TypeCharacter(c) => { - enigo.key_click(enigo::Key::Layout(c)); - } - EnigoCommand::TypeString(s) => { - enigo.key_sequence(&s); - } - EnigoCommand::DeleteCharacter => { - enigo.key_click(enigo::Key::Backspace); - } - EnigoCommand::Shutdown => { - info!("Shutting down Enigo thread."); - break; - } - } - } - }); - - // Spawn the keystroke monitor - let tx_clone = tx.clone(); - task::spawn(async move { - if let Err(e) = run_keystroke_monitor(tx_clone).await { - eprintln!("Error in keystroke monitoring: {:?}", e); - } - }); - - let stop_signal = Arc::new(AtomicBool::new(false)); - - loop { - tokio::select! { - Some(command) = rx.recv() => { - match command { - KeystrokeCommand::DoubleSlash => { - // Reset the stop signal at the start of a new action - stop_signal.store(false, Ordering::SeqCst); - - info!("double slash detected. calling ai..."); - - type_slowly("thinking".to_string(), stop_signal.clone()).await?; - let swift_output = run_swift_script().await?; - - // Use swift_output directly here - // For example, pass it to the LLM or process it further - - info!("swift output: {}", swift_output); - - let prompt = format!( - r#"Based on the following Swift output, - you need to continue where the "//" is. - Output your response in JSON format as follows: - {{ - "response": "Your response text here" - }} - The response should match the length, tone, and style of the message/prompt we are responding to. - It should not look like it was written by AI. - You are responding on behalf of the user, try to understand what is actually happening. - We also provide you with detailed print out of the entire desktop content for your reference below: - "{}" - "#, - swift_output - ); - - let start = Instant::now(); - let stop_signal_clone = stop_signal.clone(); - match call_ai(prompt, String::new(), true).await { - Ok(response) => { - let duration = start.elapsed(); - info!("{:.1?} - first call_ai", duration); - delete_characters("thinking".len()).await?; - delete_characters(2).await?; // Delete the double slash - info!("ai response: {}", response); - - // Spawn a new task for typing - let typing_task = task::spawn(async move { - let _ = type_slowly("[GENERIC_RESPONSE]".to_string(), stop_signal_clone.clone()).await; - let _ = type_slowly("\n".to_string(), stop_signal_clone.clone()).await; - let _ = type_slowly(response, stop_signal_clone).await; - }); - - // Wait for the typing task to complete or be interrupted - tokio::select! { - _ = typing_task => {}, - _ = rx.recv() => { - // If we receive any command while typing, stop the typing - stop_signal.store(true, Ordering::SeqCst); - } - } - } - Err(e) => { - let duration = start.elapsed(); - error!("{:.1?} - failed first call_ai", duration); - delete_characters("thinking".len()).await?; - error!("failed to get response from ai: {}", e); - continue; // skip the rest of the loop iteration - } - } - type_slowly("\n".to_string(), stop_signal.clone()).await?; - type_slowly("\n".to_string(), stop_signal.clone()).await?; - type_slowly("[RESPONSE_WITH_CONTEXT]".to_string(), stop_signal.clone()).await?; - - let context = format!("Swift output: {}", swift_output); - let prompt = r#"Based on the following Swift output, find double slash '//' to see where users curor is, - Now, provide 3 search queries to find relevant context in user files to continue the conversation. follow these guidelines: - - 1. focus on specificity, avoid generic words - 2. use noun phrases, target key concepts - 3. employ technical terms when appropriate - 4. consider synonyms and related concepts - 5. utilize proper nouns (names, places, products) - 6. incorporate timeframes if relevant - 7. prioritize unusual or unique words - 8. consider file types if applicable - - return a json object with an array of 3 queries, each query should be one to three words max. format: - { - "response": [ - "query1", - "query2", - "query3" - ] - }"#.to_string(); - - let start = Instant::now(); - match call_ai(prompt, context, true).await { - Ok(response) => { - let duration = start.elapsed(); - info!("{:.1?} - second call_ai", duration); - // println!("ai response: {}", response); - - // Split the response by newlines to get individual queries - let queries: Vec = response - .split('\n') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - - let mut search_results = Vec::new(); - - for query in &queries { - match search_localhost(query).await { - // used to be &query - Ok(search_result) => { - let truncated_result = - search_result.chars().take(100).collect::(); - let capitalized_query = query.to_uppercase(); - type_slowly(format!("[{}] ", capitalized_query), stop_signal.clone()).await?; - info!("search result for '{}': {}", query, truncated_result); - search_results.push(search_result); - } - Err(e) => { - error!("error searching localhost for '{}': {:?}", query, e) - } - } - } - - type_slowly("\n".to_string(), stop_signal.clone()).await?; - type_slowly("analyzing".to_string(), stop_signal.clone()).await?; - - // Final LLM call - let final_prompt = r#"Based on the desktop text and search results of my computer, draft a concise response. - Provide the most relevant message - Also provide who your message is addressed to ("TO_YOU" or "ON_YOUR_BEHALF") - You might be writing: a response, a follow-up, a suggestion, a clarification, or you might be puzzled - Output your response in JSON format as follows: - { - "response": "[ADDRESSED TO] Your response text here" - } - Ensure the response matches the length and tone of the message we are responding to. - Make the response concise and to the point. - The response should be casual, social media style. - It should not look like it was written by AI. - "#.to_string(); - let final_context = format!( - "swift output: {}\nfirst llm response: {}\nsearch results: {}", - swift_output, - response, - search_results.join("\n") - ); - - let start = Instant::now(); - match call_ai(final_prompt, final_context, true).await { - Ok(final_response) => { - let duration = start.elapsed(); - info!("{:.1?} - final call_ai", duration); - delete_characters("analyzing".len()).await?; - info!("<<>>: {}", final_response); - type_slowly(final_response, stop_signal.clone()).await?; - } - Err(e) => { - error!("error in final openai call: {:?}", e); - continue; - } - } - } - Err(e) => { - error!("error in second openai call: {}", e); - continue; - } - } - }, - KeystrokeCommand::Stop => { - info!("stop command received. stopping current action..."); - stop_signal.store(true, Ordering::SeqCst); - } - } - }, - else => break, - } - } - - // Cleanup code - info!("shutting down enigo thread..."); - if let Err(e) = _enigo_tx.send(EnigoCommand::Shutdown).await { - error!("failed to send shutdown command to enigo thread: {:?}", e); - } - - Ok(()) -} - -async fn run_swift_script() -> anyhow::Result { - let start = Instant::now(); - - let script_content = r#" -import Cocoa -import ApplicationServices -import Foundation - -class QueueElement { - let element: AXUIElement - let depth: Int - - init(_ element: AXUIElement, depth: Int) { - self.element = element - self.depth = depth - } -} - -func printAllAttributeValues(_ startElement: AXUIElement) { - var elements: [(CGPoint, CGSize, String)] = [] - var visitedElements = Set() - let unwantedValues = ["0", "", "", "3", ""] - let unwantedLabels = [ - "window", "application", "group", "button", "image", "text", - "pop up button", "region", "notifications", "table", "column", - "html content" - ] - - func traverseHierarchy(_ element: AXUIElement, depth: Int) { - guard !visitedElements.contains(element) else { return } - visitedElements.insert(element) - - var attributeNames: CFArray? - let result = AXUIElementCopyAttributeNames(element, &attributeNames) - - guard result == .success, let attributes = attributeNames as? [String] else { return } - - var position: CGPoint = .zero - var size: CGSize = .zero - - // Get position - if let positionValue = getAttributeValue(element, forAttribute: kAXPositionAttribute) as! AXValue?, - AXValueGetType(positionValue) == .cgPoint { - AXValueGetValue(positionValue, .cgPoint, &position) - } - - // Get size - if let sizeValue = getAttributeValue(element, forAttribute: kAXSizeAttribute) as! AXValue?, - AXValueGetType(sizeValue) == .cgSize { - AXValueGetValue(sizeValue, .cgSize, &size) - } - - for attr in attributes { - if ["AXDescription", "AXValue", "AXLabel", "AXRoleDescription", "AXHelp"].contains(attr) { - if let value = getAttributeValue(element, forAttribute: attr) { - let valueStr = describeValue(value) - if !valueStr.isEmpty && !unwantedValues.contains(valueStr) && valueStr.count > 1 && - !unwantedLabels.contains(valueStr.lowercased()) { - elements.append((position, size, valueStr)) - } - } - } - - // Traverse child elements - if let childrenValue = getAttributeValue(element, forAttribute: attr) { - if let elementArray = childrenValue as? [AXUIElement] { - for childElement in elementArray { - traverseHierarchy(childElement, depth: depth + 1) - } - } else if let childElement = childrenValue as! AXUIElement? { - traverseHierarchy(childElement, depth: depth + 1) - } - } - } - } - - traverseHierarchy(startElement, depth: 0) - - // Sort elements from top to bottom, then left to right - elements.sort { (a, b) in - if a.0.y != b.0.y { - return a.0.y < b.0.y - } else { - return a.0.x < b.0.x - } - } - - // Deduplicate and print sorted elements to stdout, excluding coordinates - var uniqueValues = Set() - for (_, _, valueStr) in elements { - if uniqueValues.insert(valueStr).inserted { - print(valueStr) - } - } -} - -func formatCoordinates(_ position: CGPoint, _ size: CGSize) -> String { - return String(format: "(x:%.0f,y:%.0f,w:%.0f,h:%.0f)", position.x, position.y, size.width, size.height) -} - -func describeValue(_ value: AnyObject?) -> String { - switch value { - case let string as String: - return string - case let number as NSNumber: - return number.stringValue - case let point as NSPoint: - return "(\(point.x), \(point.y))" - case let size as NSSize: - return "w=\(size.width) h=\(size.height)" - case let rect as NSRect: - return "x=\(rect.origin.x) y=\(rect.origin.y) w=\(rect.size.width) h=\(rect.size.height)" - case let range as NSRange: - return "loc=\(range.location) len=\(range.length)" - case let url as URL: - return url.absoluteString - case let array as [AnyObject]: - return array.isEmpty ? "Empty array" : "Array with \(array.count) elements" - case let axValue as AXValue: - return describeAXValue(axValue) - case is AXUIElement: - return "AXUIElement" - case .none: - return "None" - default: - return String(describing: value) - } -} - -func describeAXValue(_ axValue: AXValue) -> String { - let type = AXValueGetType(axValue) - switch type { - case .cgPoint: - var point = CGPoint.zero - AXValueGetValue(axValue, .cgPoint, &point) - return "(\(point.x), \(point.y))" - case .cgSize: - var size = CGSize.zero - AXValueGetValue(axValue, .cgSize, &size) - return "w=\(size.width) h=\(size.height)" - case .cgRect: - var rect = CGRect.zero - AXValueGetValue(axValue, .cgRect, &rect) - return "x=\(rect.origin.x) y=\(rect.origin.y) w=\(rect.size.width) h=\(rect.size.height)" - case .cfRange: - var range = CFRange(location: 0, length: 0) - AXValueGetValue(axValue, .cfRange, &range) - return "loc=\(range.location) len=\(range.length)" - default: - return "Unknown AXValue type" - } -} - -func getAttributeValue(_ element: AXUIElement, forAttribute attr: String) -> AnyObject? { - var value: AnyObject? - let result = AXUIElementCopyAttributeValue(element, attr as CFString, &value) - return result == .success ? value : nil -} - -func printAllAttributeValuesForCurrentApp() { - guard let app = NSWorkspace.shared.frontmostApplication else { - return - } - - let pid = app.processIdentifier - let axApp = AXUIElementCreateApplication(pid) - - print("attribute values for \(app.localizedName ?? "unknown app"):") - printAllAttributeValues(axApp) -} - -// usage -printAllAttributeValuesForCurrentApp() -"#; - - info!("running swift script"); - - // Check multiple possible Swift paths - let swift_paths = [ - "/usr/bin/swift", // Common path for both Intel and Apple Silicon - "/usr/local/bin/swift", // Possible alternative location - "/opt/homebrew/bin/swift", // Homebrew path on Apple Silicon - ]; - - let swift_path = swift_paths - .iter() - .find(|&path| Path::new(path).exists()) - .ok_or_else(|| { - anyhow::anyhow!("Swift executable not found in any of the expected locations") - })?; - - info!("using swift at: {}", swift_path); - - let mut child = Command::new(swift_path) - .arg("-") // Read from stdin - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn()?; - - // Write to stdin - if let Some(mut stdin) = child.stdin.take() { - stdin.write_all(script_content.as_bytes()).await?; - stdin.flush().await?; - } - - // Wait for the command to complete and get the output - let output = child.wait_with_output().await?; - - let duration = start.elapsed(); - info!("{:.1?} - run_swift_script", duration); - - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - - info!("debug: swift stdout length: {}", stdout.len()); - info!("debug: swift stderr length: {}", stderr.len()); - - if !output.status.success() { - info!( - "debug: swift script failed with status: {:?}", - output.status - ); - anyhow::bail!("swift script failed: {}", stderr); - } - - if stdout.is_empty() { - info!("debug: swift stdout is empty, returning stderr"); - Ok(stderr.into_owned()) - } else { - info!("debug: returning swift stdout"); - Ok(stdout.into_owned()) - } -} - -async fn search_localhost(query: &str) -> anyhow::Result { - let start = Instant::now(); - let client = reqwest::Client::new(); - let url = format!( - "http://localhost:3030/search?q={}&content_type=all&limit=5&offset=0", - query - ); - let response = client.get(&url).send().await?.text().await?; - let duration = start.elapsed(); - info!("{:.1?} - search_localhost for '{}'", duration, query); - Ok(response) -} diff --git a/screenpipe-actions/src/type_and_animate.rs b/screenpipe-actions/src/type_and_animate.rs deleted file mode 100644 index 328029a265..0000000000 --- a/screenpipe-actions/src/type_and_animate.rs +++ /dev/null @@ -1,93 +0,0 @@ -use anyhow::Result; -use enigo::{Enigo, Key, KeyboardControllable}; -use serde::Serialize; -use std::cell::RefCell; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use tokio::time::{sleep, Duration}; - -#[allow(dead_code)] -#[derive(Debug)] -pub enum EnigoCommand { - TypeCharacter(char), - TypeString(String), - DeleteCharacter, - Shutdown, -} - -#[derive(Debug, Serialize)] -pub struct EnigoResponse { - pub success: bool, - pub message: Option, -} - -thread_local! { - static ENIGO: RefCell> = const { RefCell::new(None) }; -} - -fn with_enigo(f: F) -> R -where - F: FnOnce(&mut Enigo) -> R, -{ - ENIGO.with(|cell| { - let mut enigo = cell.borrow_mut(); - if enigo.is_none() { - *enigo = Some(Enigo::new()); - } - f(enigo.as_mut().unwrap()) - }) -} - -pub async fn delete_characters(count: usize) -> Result<()> { - tokio::task::spawn_blocking(move || { - with_enigo(|enigo| { - enigo.key_sequence(&"\u{8}".repeat(count)); - }); - }) - .await?; - Ok(()) -} - -pub async fn type_slowly(text: String, stop_signal: Arc) -> Result<()> { - let text_len = text.len(); - for line in text.split('\n') { - if !line.is_empty() { - for char in line.chars() { - if stop_signal.load(Ordering::SeqCst) { - return Ok(()); - } - tokio::task::spawn_blocking(move || { - with_enigo(|enigo| { - enigo.key_sequence(&char.to_string()); - }); - }) - .await?; - sleep(Duration::from_millis(50)).await; - } - } - if text.contains('\n') { - if stop_signal.load(Ordering::SeqCst) { - return Ok(()); - } - tokio::task::spawn_blocking(move || { - with_enigo(|enigo| { - enigo.key_down(Key::Shift); - enigo.key_click(Key::Return); - enigo.key_up(Key::Shift); - }); - }) - .await?; - } - } - sleep(Duration::from_millis(text_len as u64)).await; - Ok(()) -} - -pub fn trigger_keyboard_permission() -> anyhow::Result<()> { - with_enigo(|enigo| { - // Perform a no-op key press to trigger the permission request - enigo.key_down(enigo::Key::Shift); - enigo.key_up(enigo::Key::Shift); - }); - Ok(()) -} diff --git a/screenpipe-app-tauri/components/pipe-store.tsx b/screenpipe-app-tauri/components/pipe-store.tsx index 1c157711b5..72e315e5a1 100644 --- a/screenpipe-app-tauri/components/pipe-store.tsx +++ b/screenpipe-app-tauri/components/pipe-store.tsx @@ -68,7 +68,7 @@ export const PipeStore: React.FC = () => { return downloadsB - downloadsA; } // Then by creation date - return new Date(b.created_at).getTime() - new Date(a.created_at).getTime(); + return new Date(b.created_at as string).getTime() - new Date(a.created_at as string).getTime(); }); // Add debounced search tracking diff --git a/screenpipe-app-tauri/src-tauri/Cargo.toml b/screenpipe-app-tauri/src-tauri/Cargo.toml index 3c36f89551..b204bd13eb 100644 --- a/screenpipe-app-tauri/src-tauri/Cargo.toml +++ b/screenpipe-app-tauri/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "screenpipe-app" -version = "0.36.5" +version = "0.36.6" description = "" authors = ["you"] license = "" diff --git a/screenpipe-audio/Cargo.toml b/screenpipe-audio/Cargo.toml index 783803e75b..9caa89807c 100644 --- a/screenpipe-audio/Cargo.toml +++ b/screenpipe-audio/Cargo.toml @@ -11,7 +11,6 @@ edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [build-dependencies] reqwest = { workspace = true } -which = "7.0.0" [dependencies] serde = { version = "1.0", features = ["derive"] } @@ -101,7 +100,6 @@ objc = "0.2.7" tempfile = "3.3.0" infer = "0.15" criterion = { workspace = true } -memory-stats = "1.0" strsim = "0.10.0" futures = "0.3.31" tracing-subscriber = "0.3.16" diff --git a/screenpipe-core/Cargo.toml b/screenpipe-core/Cargo.toml index dba450a18c..fc461ba6da 100644 --- a/screenpipe-core/Cargo.toml +++ b/screenpipe-core/Cargo.toml @@ -22,9 +22,7 @@ hf-hub = { workspace = true, features = [ "tokio", "native-tls", ] } -screenpipe-actions = { path = "../screenpipe-actions", optional = true } http-cache-reqwest = "0.15.0" -reqwest = { workspace = true } reqwest-middleware = "0.4.0" tokio = { workspace = true } @@ -51,11 +49,13 @@ sentry = { workspace = true } zip = "0.6.2" tokio-stream = "0.1.17" +[dev-dependencies] +reqwest = { workspace = true } + [features] default = ["security"] security = ["dep:regex", "dep:lazy_static"] metal = ["candle/metal", "candle-nn/metal", "candle-transformers/metal"] cuda = ["candle/cuda", "candle-nn/cuda", "candle-transformers/cuda"] mkl = ["candle/mkl", "candle-nn/mkl", "candle-transformers/mkl"] -beta = ["dep:screenpipe-actions"] llm = [] diff --git a/screenpipe-core/tests/pipes_test.rs b/screenpipe-core/tests/pipes_test.rs index c3647a11f2..821fddd969 100644 --- a/screenpipe-core/tests/pipes_test.rs +++ b/screenpipe-core/tests/pipes_test.rs @@ -1,4 +1,3 @@ -#[cfg(feature = "pipes")] #[cfg(test)] mod tests { use chrono::{TimeZone, Utc}; @@ -26,123 +25,6 @@ mod tests { }); } - async fn setup_test_pipe(temp_dir: &TempDir, pipe_name: &str, code: &str) -> PathBuf { - init(); - let pipe_dir = temp_dir.path().join(pipe_name); - create_dir_all(&pipe_dir).await.unwrap(); - let file_path = pipe_dir.join("pipe.ts"); - tokio::fs::write(&file_path, code).await.unwrap(); - pipe_dir - } - - #[tokio::test] - #[ignore] - async fn test_simple_pipe() { - let temp_dir = TempDir::new().unwrap(); - let screenpipe_dir = temp_dir.path().to_path_buf(); - - let code = r#" - console.log("Hello from simple pipe!"); - const result = 2 + 3; - console.log(`Result: ${result}`); - "#; - - let pipe_dir = setup_test_pipe(&temp_dir, "simple_pipe", code).await; - - let result = run_pipe(&pipe_dir.to_string_lossy(), screenpipe_dir).await; - assert!(result.is_ok()); - } - - #[tokio::test] - #[ignore] // TODO: fix this test (not implemented yet) - async fn test_pipe_with_http_request() { - let temp_dir = TempDir::new().unwrap(); - let screenpipe_dir = temp_dir.path().to_path_buf(); - - let code = r#" - console.log("Fetching data from API..."); - const response = await pipe.get("https://jsonplaceholder.typicode.com/todos/1"); - console.log(JSON.stringify(response, null, 2)); - "#; - - let pipe_dir = setup_test_pipe(&temp_dir, "http_pipe", code).await; - - let result = run_pipe(&pipe_dir.to_string_lossy(), screenpipe_dir).await; - assert!(result.is_ok()); - } - - #[tokio::test] - #[ignore] // TODO: fix this test (not implemented yet) - async fn test_pipe_with_error() { - let temp_dir = TempDir::new().unwrap(); - let screenpipe_dir = temp_dir.path().to_path_buf(); - - let code = r#" - console.log("This pipe will throw an error"); - throw new Error("Intentional error"); - "#; - - let pipe_dir = setup_test_pipe(&temp_dir, "error_pipe", code).await; - - let result = run_pipe(&pipe_dir.to_string_lossy(), screenpipe_dir).await; - assert!(result.is_err()); - } - - #[tokio::test] - #[ignore] // TODO: fix this test (file operations work but not in this test for some reason) - async fn test_pipe_with_file_operations() { - let temp_dir = TempDir::new().unwrap(); - let screenpipe_dir = temp_dir.path().to_path_buf(); - - let code = r#" - console.log("Writing to a file..."); - await pipe.writeFile("output.txt", "Hello, Screenpipe!"); - const content = await pipe.readFile("output.txt"); - console.log(`File content: ${content}`); - "#; - - let pipe_dir = setup_test_pipe(&temp_dir, "file_pipe", code).await; - - let result = run_pipe(&pipe_dir.to_string_lossy(), screenpipe_dir).await; - assert!(result.is_ok()); - - // Verify that the file was created and contains the expected content - let output_file = pipe_dir.join("output.txt"); - assert!(output_file.exists()); - let content = tokio::fs::read_to_string(output_file).await.unwrap(); - assert_eq!(content, "Hello, Screenpipe!"); - } - - #[tokio::test] - #[ignore] // Github said NO - async fn test_download_pipe_github_folder() { - init(); - let temp_dir = TempDir::new().unwrap(); - let screenpipe_dir = temp_dir.path().to_path_buf(); - - let github_url = - "https://github.com/mediar-ai/screenpipe/tree/main/pipes/pipe-stream-ocr-text"; - let result = download_pipe(github_url, screenpipe_dir.clone()).await; - - assert!( - result.is_ok(), - "Failed to download GitHub folder: {:?}", - result - ); - let pipe_dir = result.unwrap(); - assert!(pipe_dir.exists(), "Pipe directory does not exist"); - - let has_main_or_pipe_file = std::fs::read_dir(&pipe_dir).unwrap().any(|entry| { - let file_name = entry.unwrap().file_name().into_string().unwrap(); - (file_name.starts_with("main") || file_name.starts_with("pipe")) - && (file_name.ends_with(".ts") || file_name.ends_with(".js")) - }); - - assert!( - has_main_or_pipe_file, - "No main.ts, main.js, pipe.ts, or pipe.js file found" - ); - } #[tokio::test] async fn test_download_pipe_invalid_url() { @@ -156,142 +38,7 @@ mod tests { assert!(result.is_err(), "Expected an error for invalid URL"); } - #[tokio::test] - #[ignore] - async fn test_send_email() { - let temp_dir = TempDir::new().unwrap(); - let screenpipe_dir = temp_dir.path().to_path_buf(); - let to = std::env::var("EMAIL_TO").expect("EMAIL_TO not set"); - let from = std::env::var("EMAIL_FROM").expect("EMAIL_FROM not set"); - let password = std::env::var("EMAIL_PASSWORD").expect("EMAIL_PASSWORD not set"); - - println!("to: {}", to); - println!("from: {}", from); - println!("password: {}", password); - - // Test plain text email - let plain_text_code = format!( - r#" - (async () => {{ - const result = await pipe.sendEmail({{ - to: "{to}", - from: "{from}", - password: "{password}", - subject: "screenpipe test - plain text", - body: "yo louis, this is a plain text email test!", - contentType: "text/plain" - }}); - console.log("Plain text email result:", result); - if (!result) {{ - throw new Error("Failed to send plain text email"); - }} - }})(); - "# - ); - - // Test HTML email - let html_code = format!( - r#" - (async () => {{ - const result = await pipe.sendEmail({{ - to: "{to}", - from: "{from}", - password: "{password}", - subject: "screenpipe test - html", - body: ` - - -

yo louis, you absolute madlad!

-

this is an html email test from screenpipe.

-
    -
  • item 1
    • -
  • item 2
    • -
  • item 3
    • -
- - - `, - contentType: "text/html" - }}); - console.log("HTML email result:", result); - if (!result) {{ - throw new Error("Failed to send HTML email"); - }} - }})(); - "# - ); - - let pipe_dir = setup_test_pipe(&temp_dir, "email_test_pipe_plain", &plain_text_code).await; - std::env::set_current_dir(&pipe_dir).unwrap(); - let result = run_pipe(&pipe_dir.to_string_lossy(), screenpipe_dir.clone()).await; - assert!(result.is_ok(), "Plain text email test failed: {:?}", result); - - let pipe_dir = setup_test_pipe(&temp_dir, "email_test_pipe_html", &html_code).await; - std::env::set_current_dir(&pipe_dir).unwrap(); - let result = run_pipe(&pipe_dir.to_string_lossy(), screenpipe_dir).await; - assert!(result.is_ok(), "HTML email test failed: {:?}", result); - } - - #[tokio::test] - #[ignore] // works when run on click in cursor but not in cli so weird haha - async fn test_directory_functions() { - let temp_dir = TempDir::new().unwrap(); - let screenpipe_dir = temp_dir.path().to_path_buf(); - - let code = r#" - (async () => { - // Test mkdir - await fs.mkdir('test_dir'); - console.log('Directory created'); - - // Test writeFile - await fs.writeFile('test_dir/test_file.txt', 'Hello, World!'); - console.log('File written'); - - // Test readFile - const content = await fs.readFile('test_dir/test_file.txt'); - console.log('File content:', content); - if (content !== 'Hello, World!') { - throw new Error('File content mismatch'); - } - - // Test readdir - const files = await fs.readdir('test_dir'); - console.log('Directory contents:', files); - if (!files.includes('test_file.txt')) { - throw new Error('File not found in directory'); - } - - // Test path.join - const joinedPath = path.join('test_dir', 'nested', 'file.txt'); - console.log('Joined path:', joinedPath); - const expectedPath = process.env.OS === 'windows' ? 'test_dir\\nested\\file.txt' : 'test_dir/nested/file.txt'; - if (joinedPath !== expectedPath) { - throw new Error('Path join mismatch'); - } - - console.log('All directory function tests passed'); - })(); - "#; - - let pipe_dir = setup_test_pipe(&temp_dir, "directory_functions_test", code).await; - - // Change the working directory to the pipe directory - std::env::set_current_dir(&pipe_dir).unwrap(); - - let result = run_pipe(&pipe_dir.to_string_lossy(), screenpipe_dir).await; - assert!(result.is_ok(), "Pipe execution failed: {:?}", result); - - // Additional checks - let test_dir = pipe_dir.join("test_dir"); - assert!(test_dir.exists(), "Test directory was not created"); - - let test_file = test_dir.join("test_file.txt"); - assert!(test_file.exists(), "Test file was not created"); - - let file_content = std::fs::read_to_string(test_file).unwrap(); - assert_eq!(file_content, "Hello, World!", "File content mismatch"); - } + #[tokio::test] #[ignore] diff --git a/screenpipe-events/Cargo.toml b/screenpipe-events/Cargo.toml index a67dbfaf76..eec914a108 100644 --- a/screenpipe-events/Cargo.toml +++ b/screenpipe-events/Cargo.toml @@ -17,12 +17,12 @@ tokio-stream = { version = "0.1.17", features = ["sync"] } once_cell = { version = "1.18" } tracing.workspace = true parking_lot = "0.12.3" -serial_test = "3.2.0" chrono = "0.4.39" [dev-dependencies] serde = { version = "1.0", features = ["derive"] } criterion = { version = "0.5", features = ["async_tokio"] } +serial_test = "3.2.0" [[bench]] name = "events" diff --git a/screenpipe-actions/ai-proxy/.editorconfig b/screenpipe-js/ai-proxy/.editorconfig similarity index 100% rename from screenpipe-actions/ai-proxy/.editorconfig rename to screenpipe-js/ai-proxy/.editorconfig diff --git a/screenpipe-actions/ai-proxy/.gitignore b/screenpipe-js/ai-proxy/.gitignore similarity index 100% rename from screenpipe-actions/ai-proxy/.gitignore rename to screenpipe-js/ai-proxy/.gitignore diff --git a/screenpipe-actions/ai-proxy/.prettierrc b/screenpipe-js/ai-proxy/.prettierrc similarity index 100% rename from screenpipe-actions/ai-proxy/.prettierrc rename to screenpipe-js/ai-proxy/.prettierrc diff --git a/screenpipe-actions/ai-proxy/package-lock.json b/screenpipe-js/ai-proxy/package-lock.json similarity index 100% rename from screenpipe-actions/ai-proxy/package-lock.json rename to screenpipe-js/ai-proxy/package-lock.json diff --git a/screenpipe-actions/ai-proxy/package.json b/screenpipe-js/ai-proxy/package.json similarity index 100% rename from screenpipe-actions/ai-proxy/package.json rename to screenpipe-js/ai-proxy/package.json diff --git a/screenpipe-actions/ai-proxy/src/index.ts b/screenpipe-js/ai-proxy/src/index.ts similarity index 99% rename from screenpipe-actions/ai-proxy/src/index.ts rename to screenpipe-js/ai-proxy/src/index.ts index 32666f2da4..eaaec8626b 100644 --- a/screenpipe-actions/ai-proxy/src/index.ts +++ b/screenpipe-js/ai-proxy/src/index.ts @@ -598,7 +598,7 @@ export default Sentry.withSentry( /* terminal 1 -cd screenpipe-actions/ai-proxy +cd screenpipe-js/ai-proxy wrangler dev diff --git a/screenpipe-actions/ai-proxy/src/providers/anthropic.ts b/screenpipe-js/ai-proxy/src/providers/anthropic.ts similarity index 100% rename from screenpipe-actions/ai-proxy/src/providers/anthropic.ts rename to screenpipe-js/ai-proxy/src/providers/anthropic.ts diff --git a/screenpipe-actions/ai-proxy/src/providers/base.ts b/screenpipe-js/ai-proxy/src/providers/base.ts similarity index 100% rename from screenpipe-actions/ai-proxy/src/providers/base.ts rename to screenpipe-js/ai-proxy/src/providers/base.ts diff --git a/screenpipe-actions/ai-proxy/src/providers/gemini.ts b/screenpipe-js/ai-proxy/src/providers/gemini.ts similarity index 100% rename from screenpipe-actions/ai-proxy/src/providers/gemini.ts rename to screenpipe-js/ai-proxy/src/providers/gemini.ts diff --git a/screenpipe-actions/ai-proxy/src/providers/index.ts b/screenpipe-js/ai-proxy/src/providers/index.ts similarity index 100% rename from screenpipe-actions/ai-proxy/src/providers/index.ts rename to screenpipe-js/ai-proxy/src/providers/index.ts diff --git a/screenpipe-actions/ai-proxy/src/providers/openai.ts b/screenpipe-js/ai-proxy/src/providers/openai.ts similarity index 100% rename from screenpipe-actions/ai-proxy/src/providers/openai.ts rename to screenpipe-js/ai-proxy/src/providers/openai.ts diff --git a/screenpipe-actions/ai-proxy/src/types.ts b/screenpipe-js/ai-proxy/src/types.ts similarity index 100% rename from screenpipe-actions/ai-proxy/src/types.ts rename to screenpipe-js/ai-proxy/src/types.ts diff --git a/screenpipe-actions/ai-proxy/tsconfig.json b/screenpipe-js/ai-proxy/tsconfig.json similarity index 100% rename from screenpipe-actions/ai-proxy/tsconfig.json rename to screenpipe-js/ai-proxy/tsconfig.json diff --git a/screenpipe-actions/ai-proxy/worker-configuration.d.ts b/screenpipe-js/ai-proxy/worker-configuration.d.ts similarity index 100% rename from screenpipe-actions/ai-proxy/worker-configuration.d.ts rename to screenpipe-js/ai-proxy/worker-configuration.d.ts diff --git a/screenpipe-actions/ai-proxy/wrangler.toml b/screenpipe-js/ai-proxy/wrangler.toml similarity index 100% rename from screenpipe-actions/ai-proxy/wrangler.toml rename to screenpipe-js/ai-proxy/wrangler.toml diff --git a/screenpipe-server/Cargo.toml b/screenpipe-server/Cargo.toml index 5c487849e6..2d2f22b606 100644 --- a/screenpipe-server/Cargo.toml +++ b/screenpipe-server/Cargo.toml @@ -17,7 +17,6 @@ screenpipe-events = { path = "../screenpipe-events" } screenpipe-vision = { path = "../screenpipe-vision" } screenpipe-audio = { path = "../screenpipe-audio" } screenpipe-core = { path = "../screenpipe-core", features = ["security"] } -screenpipe-actions = { path = "../screenpipe-actions", optional = true } killport = { version = "1.1.0" } # Image processing @@ -131,7 +130,6 @@ tokio-tungstenite = "0.19.0" criterion = { workspace = true } rand = "0.8" -axum-test = "15.3.0" [[bench]] name = "db_benchmarks" @@ -147,7 +145,6 @@ metal = ["candle/metal", "candle-nn/metal", "candle-transformers/metal"] cuda = ["candle/cuda", "candle-nn/cuda", "candle-transformers/cuda"] mkl = ["candle/mkl", "candle-nn/mkl", "candle-transformers/mkl"] llm = [] -beta = ["screenpipe-core/beta", "dep:screenpipe-actions"] experimental = ["enigo"] debug-console = ["console-subscriber"] diff --git a/screenpipe-server/src/bin/screenpipe-server.rs b/screenpipe-server/src/bin/screenpipe-server.rs index d5272b39b5..eff369e43c 100644 --- a/screenpipe-server/src/bin/screenpipe-server.rs +++ b/screenpipe-server/src/bin/screenpipe-server.rs @@ -261,21 +261,7 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } #[allow(unused_variables)] - Command::Setup { enable_beta } => { - #[cfg(feature = "beta")] - if enable_beta { - use screenpipe_actions::type_and_animate::trigger_keyboard_permission; - - // Trigger keyboard permission request - if let Err(e) = trigger_keyboard_permission() { - warn!("failed to trigger keyboard permission: {:?}", e); - warn!("please grant keyboard permission manually in System Preferences."); - } else { - info!( - "keyboard permission requested. please grant permission if prompted." - ); - } - } + Command::Setup {} => { use screenpipe_audio::{ trigger_audio_permission, vad_engine::SileroVad, whisper::WhisperModel, }; @@ -1015,32 +1001,6 @@ async fn main() -> anyhow::Result<()> { let ctrl_c_future = signal::ctrl_c(); pin_mut!(ctrl_c_future); - // only in beta and on macos - #[cfg(feature = "beta")] - { - if cli.enable_beta && cfg!(target_os = "macos") { - use screenpipe_actions::run; - - info!("beta feature enabled, starting screenpipe actions"); - - let shutdown_tx_clone = shutdown_tx.clone(); - tokio::spawn(async move { - let mut shutdown_rx = shutdown_tx_clone.subscribe(); - - tokio::select! { - result = run() => { - if let Err(e) = result { - error!("Error running screenpipe actions: {}", e); - } - } - _ = shutdown_rx.recv() => { - info!("Received shutdown signal, stopping screenpipe actions"); - } - } - }); - } - } - // Start the UI monitoring task #[cfg(target_os = "macos")] if cli.enable_ui_monitoring { diff --git a/screenpipe-server/src/cli.rs b/screenpipe-server/src/cli.rs index b53d3b2a7c..498877e7c9 100644 --- a/screenpipe-server/src/cli.rs +++ b/screenpipe-server/src/cli.rs @@ -241,11 +241,6 @@ pub struct Cli { #[arg(long, default_value_t = false)] pub enable_llm: bool, - /// Enable beta features - #[cfg(feature = "beta")] - #[arg(long, default_value_t = false)] - pub enable_beta: bool, - /// Enable UI monitoring (macOS only) #[arg(long, default_value_t = false)] pub enable_ui_monitoring: bool, @@ -329,11 +324,7 @@ pub enum Command { use_embedding: bool, }, /// Setup screenpipe environment - Setup { - /// Enable beta features - #[arg(long, default_value_t = false)] - enable_beta: bool, - }, + Setup, /// Run database migrations Migrate, /// Generate shell completions diff --git a/screenpipe-vision/Cargo.toml b/screenpipe-vision/Cargo.toml index 67fe9e12ee..365b856e15 100644 --- a/screenpipe-vision/Cargo.toml +++ b/screenpipe-vision/Cargo.toml @@ -53,9 +53,6 @@ reqwest = { workspace = true } [dev-dependencies] tempfile = "3.3.0" criterion = { workspace = true } -assert_cmd = "2.0.14" -predicates = "3.1.0" -assert_fs = "1.1.1" strsim = "0.10.0" memory-stats = "1.2.0" @@ -64,9 +61,6 @@ futures-util = "0.3" tokio-tungstenite = "0.20" serde = "1.0.200" -[build-dependencies] -cc = "1.0" - [package.metadata.osx] framework = ["Vision", "AppKit"]