1

src-tauri/Cargo.toml

@@ -16,14 +16,15 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 
 [build-dependencies]
 tauri-build = { version = "2.0.3", features = [] }
+tonic-build = "0.12.3"
 
 [dependencies]
 tauri = { version = "2.1.1", features = ["unstable"] }
 tauri-plugin-shell = "2.0.2"
 serde = { version = "1.0.215", features = ["derive"] }
 serde_json = "1.0.133"
-tauri-plugin-fs = "2.0.3"
-tauri-plugin-dialog = "2"
+tauri-plugin-fs = "2.1.0"
+tauri-plugin-dialog = "2.0.4"
 tauri-plugin-store = "2.1.0"
 tauri-plugin-os = "2"
 
@@ -36,24 +37,25 @@ crabgrab = { git = "https://github.com/qzd1989/CrabGrab.git", features = [
 ] }
 futures = "0.3"
 wgpu = "23.0.0"
-tokio = { version = "1.41.1", features = ["rt", "macros", "rt-multi-thread"] }
+tokio = { version = "1.42.0", features = ["rt", "macros", "rt-multi-thread"] }
 tokio-tungstenite = "0.24.0"
 crossbeam-channel = "0.5.13"
 lazy_static = "1.5.0"
-anyhow = "1.0.93"
+anyhow = "1.0.94"
 image = { version = "0.25.5", features = ["jpeg", "png"] }
 base64 = "0.22.1"
 rusty-tesseract = { path = "libs/rusty-tesseract" }
 enigo = "0.3.0"
-libc = "0.2.164" #kill command on macos && linux
-opencv = { version = "0.93.4" }
-
+libc = "0.2.167" #kill command on macos && linux
+opencv = { version = "0.93.5" }
+tonic = "0.12.3"
+prost = "0.13.3"
 
 [target.'cfg(windows)'.dependencies]
 windows = "0.58"
 
 [dependencies.pyo3]
-version = "0.23.1"
+version = "0.23.3"
 features = ["abi3-py310"]
 
 [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]

src-tauri/build.rs

@@ -1,3 +1,4 @@
 fn main() {
+    tonic_build::compile_protos("src/grpc/proto/common.proto").unwrap();
     tauri_build::build()
 }

src-tauri/examples/grpc_client.rs

@@ -0,0 +1,17 @@
+use std::time::Instant;
+
+use kiwi_lib::grpc::FindImageRequest;
+
+fn main() {
+    let time = Instant::now();
+    let path = "payment".to_string();
+    let start_x = 15;
+    let start_y = 20;
+    let end_x = 100;
+    let end_y = 100;
+    let threshold = 0.957;
+    let request = FindImageRequest::new(path, start_x, start_y, end_x, end_y, threshold);
+    let result = request.send();
+    println!("{:?}", result);
+    println!("eslaped: {:?}", time.elapsed());
+}

src-tauri/examples/grpc_server.rs

@@ -0,0 +1,10 @@
+use std::time::Duration;
+
+use kiwi_lib::grpc::{self, RUN_TIME};
+
+fn main() {
+    RUN_TIME.spawn(async move {
+        grpc::run().await.unwrap();
+    });
+    std::thread::sleep(Duration::from_secs(1000));
+}

src-tauri/src/common.rs

@@ -14,8 +14,6 @@ use std::collections::VecDeque;
 use std::sync::Mutex;
 
 lazy_static! {
-    pub static ref VERSION: String = String::from("1.0.0");
-    pub static ref PYTHON_VERSION: String = String::from("3.10");
     pub static ref PROJECTS_DIR: String = {
         directories::UserDirs::new()
             .unwrap()
@@ -28,24 +26,6 @@ lazy_static! {
             .to_string()
     };
     pub static ref PROJECT_DIR: Mutex<Option<String>> = Mutex::new(None);
-    pub static ref PYTHON_EXEC_FILE: String = {
-        #[cfg(target_os = "macos")]
-        {
-            format!(
-                "/Library/Frameworks/Python.framework/Versions/{}/bin/python{}",
-                *PYTHON_VERSION, *PYTHON_VERSION
-            )
-        }
-        #[cfg(target_os = "windows")]
-        {
-            utils::fs::current_dir()
-                .join("python")
-                .join("pythonw.exe")
-                .to_str()
-                .unwrap()
-                .to_string()
-        }
-    };
 }
 
 #[pyclass]

src-tauri/src/frontend_commands/install/mod.rs

@@ -1,4 +1,4 @@
-use crate::{common::VERSION, utils};
+use crate::utils;
 use lazy_static::lazy_static;
 #[cfg(target_os = "macos")]
 mod macos;
@@ -10,6 +10,7 @@ mod windows;
 pub use windows::*;
 
 lazy_static! {
+    pub static ref VERSION: String = String::from("1.0.0");
     pub static ref TESSERACT_INSTALL_FILE: String = {
         #[cfg(target_os = "macos")]
         {

src-tauri/src/frontend_commands/mod.rs

@@ -6,6 +6,7 @@ pub mod fs;
 pub mod install;
 pub mod project;
 
+use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 use tauri::{AppHandle, Emitter};
 
@@ -74,3 +75,25 @@ impl Emit {
         Self { data, time }
     }
 }
+
+lazy_static! {
+    pub static ref PYTHON_VERSION: String = String::from("3.10");
+    pub static ref PYTHON_EXEC_FILE: String = {
+        #[cfg(target_os = "macos")]
+        {
+            format!(
+                "/Library/Frameworks/Python.framework/Versions/{}/bin/python{}",
+                *PYTHON_VERSION, *PYTHON_VERSION
+            )
+        }
+        #[cfg(target_os = "windows")]
+        {
+            utils::fs::current_dir()
+                .join("python")
+                .join("pythonw.exe")
+                .to_str()
+                .unwrap()
+                .to_string()
+        }
+    };
+}

src-tauri/src/frontend_commands/project.rs

@@ -1,7 +1,8 @@
 use super::AppHandleExt as _;
+use super::PYTHON_EXEC_FILE;
 use crate::{
     capture::{listen_primary_display, CAPTURE_SWITCH, FRAME},
-    common::{PROJECT_DIR, PYTHON_EXEC_FILE},
+    common::PROJECT_DIR,
 };
 use lazy_static::lazy_static;
 use std::{

src-tauri/src/grpc/find.rs

@@ -0,0 +1,143 @@
+use crate::capture::FRAME;
+use crate::common::PROJECT_DIR;
+use crate::find as system_find;
+use crate::utils::fs::exists;
+
+use super::RUN_TIME;
+use anyhow::Result;
+use find::{
+    find_service_client::FindServiceClient, find_service_server::FindService, FindImageReply,
+    FindImageRequest,
+};
+use std::path::PathBuf;
+use std::sync::LazyLock;
+use std::sync::Mutex;
+use tonic::transport::channel::Channel;
+use tonic::{Request, Response, Status};
+
+use super::EMPTY_WEIGHT_POINT;
+pub mod find {
+    tonic::include_proto!("find_kiwi");
+}
+#[derive(Debug, Default)]
+pub struct FindServiceInstance {}
+#[tonic::async_trait]
+impl FindService for FindServiceInstance {
+    async fn find_image(
+        &self,
+        request: Request<FindImageRequest>,
+    ) -> Result<Response<FindImageReply>, Status> {
+        if PROJECT_DIR.lock().unwrap().is_none() {
+            return Ok(FindImageReply::empty().response());
+        }
+        if FRAME.lock().unwrap().is_none() {
+            return Ok(FindImageReply::empty().response());
+        }
+        let request = request.into_inner();
+        let frame = FRAME.lock().unwrap().clone().unwrap();
+        let project_path = PROJECT_DIR.lock().unwrap().clone().unwrap();
+        let project_pathbuf = PathBuf::from(project_path);
+        let full_path = format!(
+            "{}{}",
+            project_pathbuf
+                .join("resources")
+                .join(request.path)
+                .to_str()
+                .unwrap()
+                .to_string(),
+            ".png"
+        );
+        println!("full path is {:?}", full_path.clone());
+        if let Err(error) = exists(full_path.clone()) {
+            println!("exists caused error: {}", error.to_string());
+            return Ok(FindImageReply::empty().response());
+        }
+        if !exists(full_path.clone()).unwrap() {
+            println!("file not exist");
+            return Ok(FindImageReply::empty().response());
+        }
+        let (width, height) = (
+            request.end_x - request.start_x,
+            request.end_y - request.start_y,
+        );
+        if width <= 0 || height <= 0 {
+            println!("width ({}) or height ({}) is zero", width, height);
+            return Ok(FindImageReply::empty().response());
+        }
+        let template = image::open(full_path.clone()).unwrap().to_rgba8();
+        if let Ok(weight_point) = system_find::image::find_one(
+            frame,
+            template,
+            request.start_x,
+            request.start_y,
+            width,
+            height,
+            request.threshold,
+        ) {
+            let (x, y, weight) = (
+                weight_point.point.x as i32,
+                weight_point.point.y as i32,
+                weight_point.weight,
+            );
+            return Ok(FindImageReply::new(x, y, weight).response());
+        }
+        Ok(FindImageReply::empty().response())
+    }
+}
+
+impl FindImageReply {
+    pub fn new(x: i32, y: i32, threshold: f64) -> Self {
+        Self { x, y, threshold }
+    }
+    pub fn empty() -> Self {
+        let (x, y, threshold) = EMPTY_WEIGHT_POINT;
+        Self { x, y, threshold }
+    }
+    pub fn response(self) -> Response<Self> {
+        Response::new(self)
+    }
+
+    pub fn to_tuple(self) -> (i32, i32, f64) {
+        (self.x, self.y, self.threshold)
+    }
+}
+
+impl FindImageRequest {
+    pub fn new(
+        path: String,
+        start_x: u32,
+        start_y: u32,
+        end_x: u32,
+        end_y: u32,
+        threshold: f64,
+    ) -> Self {
+        Self {
+            path,
+            start_x,
+            start_y,
+            end_x,
+            end_y,
+            threshold,
+        }
+    }
+    pub fn send(self) -> FindImageReply {
+        let result = RUN_TIME.block_on(async move {
+            if CLIENT.lock().unwrap().is_none() {
+                init_client().await;
+            }
+            let mut client = CLIENT.lock().unwrap().clone().unwrap();
+            client.find_image(self).await.unwrap()
+        });
+        result.into_inner()
+    }
+}
+
+async fn init_client() {
+    let client = FindServiceClient::connect(format!("http://{}", super::ADDR))
+        .await
+        .unwrap();
+    *CLIENT.lock().unwrap() = Some(client);
+}
+
+pub static CLIENT: LazyLock<Mutex<Option<FindServiceClient<Channel>>>> =
+    LazyLock::new(|| Mutex::new(None));

src-tauri/src/grpc/mod.rs

@@ -0,0 +1,18 @@
+mod find;
+mod server;
+
+use crate::common::HexColor;
+use std::sync::LazyLock;
+use tokio::runtime::Runtime;
+
+pub use find::find::FindImageRequest;
+pub use server::{run, run_spawn};
+
+const ADDR: &str = "[::1]:50555";
+pub static RUN_TIME: LazyLock<Runtime> = LazyLock::new(|| tokio::runtime::Runtime::new().unwrap());
+
+const EMPTY_WEIGHT_POINT: (i32, i32, f64) = (-1, -1, 0.0);
+const EMPTY_WEIGHT_POINTS: Vec<(i32, i32, f64)> = Vec::new();
+const EMPTY_POINT: (i32, i32) = (-1, -1);
+const EMPTY_LOCATING_COLORS: Vec<(i32, i32, HexColor)> = Vec::new();
+const EMPTY_TEXT: &str = "";

src-tauri/src/grpc/proto/common.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+package find_kiwi;
+service FindService {
+  rpc FindImage(FindImageRequest) returns (FindImageReply);
+}
+message FindImageRequest {
+  string path = 1;
+  uint32 start_x = 2;
+  uint32 start_y = 3;
+  uint32 end_x = 4;
+  uint32 end_y = 5;
+  double threshold = 6;
+}
+message FindImageReply {
+  int32 x = 1;
+  int32 y = 2;
+  double threshold = 3;
+}

src-tauri/src/grpc/server.rs

@@ -0,0 +1,21 @@
+use super::find::{find::find_service_server::FindServiceServer, FindServiceInstance};
+use super::RUN_TIME;
+use anyhow::Result;
+use tonic::transport::Server;
+
+pub async fn run() -> Result<()> {
+    let addr = super::ADDR.parse()?;
+    let instance = FindServiceInstance::default();
+    Server::builder()
+        .add_service(FindServiceServer::new(instance))
+        .serve(addr)
+        .await?;
+    Ok(())
+}
+
+pub fn run_spawn() {
+    RUN_TIME.spawn(async {
+        run().await.unwrap();
+    });
+}
+//grpcurl -plaintext -import-path /Users/kiwi/Documents/Rust/kiwi/src-tauri/src/grpc/proto -proto common.proto -d '{"path": "payment", "startX": 457, "startY": 88, "endX": 711, "endY": 946, "threshold": 590.2428442764207}' '[::1]:50555' find_kiwi.FindService/FindImage

src-tauri/src/lib.rs

@@ -1,6 +1,7 @@
 pub mod capture;
 pub mod common;
 pub mod find;
+pub mod grpc;
 pub mod input;
 pub mod utils;
 
@@ -13,6 +14,10 @@ use pyo3::prelude::*;
 #[cfg_attr(mobile, tauri::mobile_entry_point)]
 pub fn run() {
     env_logger::init();
+
+    //response calling from python or other requests
+    grpc::run_spawn();
+
     let frontend = tauri::Builder::default()
         .plugin(tauri_plugin_os::init())
         .plugin(tauri_plugin_global_shortcut::Builder::new().build())