chore(chrome): add streaming ws

spider-rs · Dec 11, 2024 · f645643 · f645643
1 parent 88a3f83
commit f645643
Show file tree

Hide file tree

Showing 11 changed files with 45 additions and 64 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/examples/real_world.rs b/examples/real_world.rs
@@ -7,13 +7,12 @@ use spider::website::Website;
 use spider::{
     configuration::WaitForIdleNetwork, features::chrome_common::RequestInterceptConfiguration,
 };
-use spider_utils::spider_transformations::transformation::content::{
-    transform_content, ReturnFormat, TransformConfig,
-};
 use std::io::Result;
 use std::time::Duration;
 
 async fn crawl_website(url: &str) -> Result<()> {
+    let mut stdout = tokio::io::stdout();
+
     let mut website: Website = Website::new(url)
         .with_limit(1)
         .with_chrome_intercept(RequestInterceptConfiguration::new(true))
@@ -24,15 +23,12 @@ async fn crawl_website(url: &str) -> Result<()> {
         .with_stealth(true)
         .with_return_page_links(true)
         .with_fingerprint(true)
-        .with_proxies(Some(vec!["http://localhost:8888".into()]))
+        // .with_proxies(Some(vec!["http://localhost:8888".into()]))
         .with_chrome_connection(Some("http://127.0.0.1:9222/json/version".into()))
         .build()
         .unwrap();
 
     let mut rx2 = website.subscribe(16).unwrap();
-    let mut stdout = tokio::io::stdout();
-    let mut conf = TransformConfig::default();
-    conf.return_format = ReturnFormat::Markdown;
 
     tokio::spawn(async move {
         while let Ok(page) = rx2.recv().await {
@@ -51,12 +47,6 @@ async fn crawl_website(url: &str) -> Result<()> {
                     .as_bytes(),
                 )
                 .await;
-
-            let markup = transform_content(&page, &conf, &None, &None, &None);
-
-            let _ = stdout
-                .write_all(format!("- {}\n {}\n", page.get_url(), markup).as_bytes())
-                .await;
         }
     });
 

diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.21.8"
+version = "2.21.9"
 authors = [
     "j-mendez <[email protected]>"
 ]
@@ -108,8 +108,8 @@ version = "2"
 path = "../spider_chrome"
 optional = true
 features = [
-    "tokio-runtime",
     "bytes",
+    "stream"
 ]
 
 [features]

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.21.8"
+version = "2.21.9"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"
@@ -18,7 +18,7 @@ categories = ["web-programming", "api-bindings", "development-tools::testing"]
 name = "chromiumoxide"
 
 [dependencies]
-async-tungstenite = "0.28"
+tokio-tungstenite = "0.24"
 serde = { version = "1", features = ["derive"] }
 futures = "0.3"
 chromiumoxide_types = { version = "0.7" }
@@ -63,8 +63,8 @@ tracing-subscriber = "0.3"
 tokio = { version = "1", features = ["rt-multi-thread", "time", "macros"] }
 
 [features]
-default = ["tokio-runtime", "bytes"]
-tokio-runtime = ["async-tungstenite/tokio-runtime"]
+default = ["bytes"]
+stream = ["tokio-tungstenite/stream"]
 fetcher = []
 bytes = ["dep:bytes"]
 serde0 = []
@@ -77,24 +77,19 @@ _fetcher-native-tokio = ["fetcher", "chromiumoxide_fetcher/_native-tokio"]
 
 [[example]]
 name = "wiki-tokio"
-required-features = ["tokio-runtime"]
 
 [[example]]
 name = "iframe-workaround"
-required-features = ["tokio-runtime", "tokio"]
 
 [[example]]
 name = "storage-cookie"
-required-features = ["tokio-runtime"]
 
 [[example]]
 name = "console-logs"
-required-features = ["tokio-runtime"]
 
 [[example]]
 name = "httpfuture"
-required-features = ["tokio-runtime"]
 
 [[example]]
 name = "fetcher-tokio"
-required-features = ["tokio-runtime", "_fetcher-native-tokio"]
+required-features = ["_fetcher-native-tokio"]
diff --git a/spider_chrome/README.md b/spider_chrome/README.md
@@ -75,10 +75,8 @@ Use `chromiumoxide` with the `async-std` runtime:
 chromiumoxide = { git = "https://github.com/mattsse/chromiumoxide", branch = "main"}
 ```
 
-To use the `tokio` runtime instead add `features = ["tokio-runtime"]` and set `default-features = false` to disable the default runtime (`async-std`):
-
 ```toml
-chromiumoxide = { git = "https://github.com/mattsse/chromiumoxide", features = ["tokio-runtime"], default-features = false, branch = "main"}
+chromiumoxide = { git = "https://github.com/mattsse/chromiumoxide", default-features = false, branch = "main"}
 ```
 
 This configuration is made possible primarily by the websocket crate of choice: [`async-tungstenite`](https://github.com/sdroege/async-tungstenite).

diff --git a/spider_chrome/src/conn.rs b/spider_chrome/src/conn.rs
@@ -3,18 +3,20 @@ use std::marker::PhantomData;
 use std::pin::Pin;
 use std::task::ready;
 
-use async_tungstenite::tungstenite::Message as WsMessage;
-use async_tungstenite::{tungstenite::protocol::WebSocketConfig, WebSocketStream};
 use futures::stream::Stream;
 use futures::task::{Context, Poll};
 use futures::{SinkExt, StreamExt};
+use tokio_tungstenite::tungstenite::Message as WsMessage;
+use tokio_tungstenite::MaybeTlsStream;
+use tokio_tungstenite::{tungstenite::protocol::WebSocketConfig, WebSocketStream};
 
 use chromiumoxide_cdp::cdp::browser_protocol::target::SessionId;
 use chromiumoxide_types::{CallId, EventMessage, Message, MethodCall, MethodId};
 
 use crate::error::CdpError;
 use crate::error::Result;
-use async_tungstenite::tokio::ConnectStream;
+
+type ConnectStream = MaybeTlsStream<tokio::net::TcpStream>;
 
 /// Exchanges the messages with the websocket
 #[must_use = "streams do nothing unless polled"]
@@ -40,11 +42,9 @@ impl<T: EventMessage + Unpin> Connection<T> {
             ..Default::default()
         };
 
-        let (ws, _) = async_tungstenite::tokio::connect_async_with_config(
-            debug_ws_url.as_ref(),
-            Some(config),
-        )
-        .await?;
+        let (ws, _) =
+            tokio_tungstenite::connect_async_with_config(debug_ws_url.as_ref(), Some(config), true)
+                .await?;
 
         Ok(Self {
             pending_commands: Default::default(),

diff --git a/spider_chrome/src/error.rs b/spider_chrome/src/error.rs
@@ -3,12 +3,12 @@ use std::io;
 use std::process::ExitStatus;
 use std::time::Instant;
 
-use async_tungstenite::tungstenite;
-use async_tungstenite::tungstenite::Message;
 use base64::DecodeError;
 use futures::channel::mpsc::SendError;
 use futures::channel::oneshot::Canceled;
 use thiserror::Error;
+use tokio_tungstenite::tungstenite;
+use tokio_tungstenite::tungstenite::Message;
 
 use chromiumoxide_cdp::cdp::browser_protocol::page::FrameId;
 

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.21.8"
+version = "2.21.9"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.21.8"
+version = "2.21.9"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.21.8"
+version = "2.21.9"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.21.8"
+version = "2.21.9"
 authors = [
     "j-mendez <[email protected]>"
 ]