1use crate::listeners::{EventListenerRequest, EventListeners};
2use chromiumoxide_cdp::cdp::browser_protocol::browser::*;
3use chromiumoxide_cdp::cdp::browser_protocol::target::*;
4use chromiumoxide_cdp::cdp::events::CdpEvent;
5use chromiumoxide_cdp::cdp::events::CdpEventMessage;
6use chromiumoxide_types::{CallId, Message, Method, Response};
7use chromiumoxide_types::{MethodId, Request as CdpRequest};
8use fnv::FnvHashMap;
9use futures_util::Stream;
10use hashbrown::{HashMap, HashSet};
11use spider_network_blocker::intercept_manager::NetworkInterceptManager;
12use std::pin::Pin;
13use std::task::{Context, Poll};
14use std::time::{Duration, Instant};
15use tokio::sync::mpsc::Receiver;
16use tokio::sync::oneshot::Sender as OneshotSender;
17use tokio_tungstenite::tungstenite::error::ProtocolError;
18use tokio_tungstenite::tungstenite::Error;
19
20use std::sync::Arc;
21use tokio::sync::Notify;
22
23use crate::cmd::{to_command_response, CommandMessage};
24use crate::conn::Connection;
25use crate::error::{CdpError, Result};
26use crate::handler::browser::BrowserContext;
27use crate::handler::frame::FrameRequestedNavigation;
28use crate::handler::frame::{NavigationError, NavigationId, NavigationOk};
29use crate::handler::job::PeriodicJob;
30use crate::handler::session::Session;
31use crate::handler::target::TargetEvent;
32use crate::handler::target::{Target, TargetConfig};
33use crate::handler::viewport::Viewport;
34use crate::page::Page;
35pub(crate) use page::PageInner;
36
37pub const REQUEST_TIMEOUT: u64 = 30_000;
39
40pub mod blockers;
41pub mod browser;
42pub mod commandfuture;
43pub mod domworld;
44pub mod emulation;
45pub mod frame;
46pub mod http;
47pub mod httpfuture;
48mod job;
49pub mod network;
50pub mod network_utils;
51pub mod page;
52pub mod sender;
53mod session;
54pub mod target;
55pub mod target_message_future;
56pub mod viewport;
57
58#[must_use = "streams do nothing unless polled"]
61#[derive(Debug)]
62pub struct Handler {
63 pub default_browser_context: BrowserContext,
64 pub browser_contexts: HashSet<BrowserContext>,
65 pending_commands: FnvHashMap<CallId, (PendingRequest, MethodId, Instant)>,
69 from_browser: Receiver<HandlerMessage>,
71 target_ids: Vec<TargetId>,
73 targets: HashMap<TargetId, Target>,
75 navigations: FnvHashMap<NavigationId, NavigationRequest>,
77 sessions: HashMap<SessionId, Session>,
81 conn: Option<Connection<CdpEventMessage>>,
84 evict_command_timeout: PeriodicJob,
86 next_navigation_id: usize,
88 config: HandlerConfig,
90 event_listeners: EventListeners,
92 closing: bool,
94 remaining_bytes: Option<u64>,
96 budget_exhausted: bool,
98 attached_targets: HashSet<TargetId>,
100 page_wake: Option<Arc<Notify>>,
103}
104
105lazy_static::lazy_static! {
106 static ref DISCOVER_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
108 let discover = SetDiscoverTargetsParams::new(true);
109 (discover.identifier(), serde_json::to_value(discover).expect("valid discover target params"))
110 };
111 static ref TARGET_PARAMS_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
113 let msg = GetTargetsParams { filter: None };
114 (msg.identifier(), serde_json::to_value(msg).expect("valid paramtarget"))
115 };
116 static ref CLOSE_PARAMS_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
118 let close_msg = CloseParams::default();
119 (close_msg.identifier(), serde_json::to_value(close_msg).expect("valid close params"))
120 };
121}
122
123fn maybe_store_attach_session_id(target: &mut Target, method: &MethodId, resp: &Response) {
124 if method.as_ref() != AttachToTargetParams::IDENTIFIER {
125 return;
126 }
127
128 if let Ok(resp) = to_command_response::<AttachToTargetParams>(resp.clone(), method.clone()) {
129 target.set_session_id(resp.result.session_id);
130 }
131}
132
133impl Handler {
134 pub(crate) fn new(
137 mut conn: Connection<CdpEventMessage>,
138 rx: Receiver<HandlerMessage>,
139 config: HandlerConfig,
140 ) -> Self {
141 let discover = DISCOVER_ID.clone();
142 let _ = conn.submit_command(discover.0, None, discover.1);
143 let conn = Some(conn);
144
145 let browser_contexts = config
146 .context_ids
147 .iter()
148 .map(|id| BrowserContext::from(id.clone()))
149 .collect();
150
151 Self {
152 pending_commands: Default::default(),
153 from_browser: rx,
154 default_browser_context: Default::default(),
155 browser_contexts,
156 target_ids: Default::default(),
157 targets: Default::default(),
158 navigations: Default::default(),
159 sessions: Default::default(),
160 conn,
161 evict_command_timeout: PeriodicJob::new(config.request_timeout),
162 next_navigation_id: 0,
163 config,
164 event_listeners: Default::default(),
165 closing: false,
166 remaining_bytes: None,
167 budget_exhausted: false,
168 attached_targets: Default::default(),
169 page_wake: None,
170 }
171 }
172
173 #[inline]
176 fn conn(&mut self) -> Result<&mut Connection<CdpEventMessage>> {
177 self.conn
178 .as_mut()
179 .ok_or_else(|| CdpError::msg("connection consumed by Handler::run()"))
180 }
181
182 pub fn get_target(&self, target_id: &TargetId) -> Option<&Target> {
184 self.targets.get(target_id)
185 }
186
187 pub fn targets(&self) -> impl Iterator<Item = &Target> + '_ {
189 self.targets.values()
190 }
191
192 pub fn default_browser_context(&self) -> &BrowserContext {
194 &self.default_browser_context
195 }
196
197 pub fn browser_contexts(&self) -> impl Iterator<Item = &BrowserContext> + '_ {
199 self.browser_contexts.iter()
200 }
201
202 fn on_navigation_response(&mut self, id: NavigationId, resp: Response) {
204 if let Some(nav) = self.navigations.remove(&id) {
205 match nav {
206 NavigationRequest::Navigate(mut nav) => {
207 if nav.navigated {
208 let _ = nav.tx.send(Ok(resp));
209 } else {
210 nav.set_response(resp);
211 self.navigations
212 .insert(id, NavigationRequest::Navigate(nav));
213 }
214 }
215 }
216 }
217 }
218
219 fn on_navigation_lifecycle_completed(&mut self, res: Result<NavigationOk, NavigationError>) {
221 match res {
222 Ok(ok) => {
223 let id = *ok.navigation_id();
224 if let Some(nav) = self.navigations.remove(&id) {
225 match nav {
226 NavigationRequest::Navigate(mut nav) => {
227 if let Some(resp) = nav.response.take() {
228 let _ = nav.tx.send(Ok(resp));
229 } else {
230 nav.set_navigated();
231 self.navigations
232 .insert(id, NavigationRequest::Navigate(nav));
233 }
234 }
235 }
236 }
237 }
238 Err(err) => {
239 if let Some(nav) = self.navigations.remove(err.navigation_id()) {
240 match nav {
241 NavigationRequest::Navigate(nav) => {
242 let _ = nav.tx.send(Err(err.into()));
243 }
244 }
245 }
246 }
247 }
248 }
249
250 fn on_response(&mut self, resp: Response) {
252 if let Some((req, method, _)) = self.pending_commands.remove(&resp.id) {
253 match req {
254 PendingRequest::CreateTarget(tx) => {
255 match to_command_response::<CreateTargetParams>(resp, method) {
256 Ok(resp) => {
257 if let Some(target) = self.targets.get_mut(&resp.target_id) {
258 target.set_initiator(tx);
259 } else {
260 let _ = tx.send(Err(CdpError::NotFound)).ok();
261 }
262 }
263 Err(err) => {
264 let _ = tx.send(Err(err)).ok();
265 }
266 }
267 }
268 PendingRequest::GetTargets(tx) => {
269 match to_command_response::<GetTargetsParams>(resp, method) {
270 Ok(resp) => {
271 let targets = resp.result.target_infos;
272 let results = targets.clone();
273
274 for target_info in targets {
275 let event: EventTargetCreated = EventTargetCreated { target_info };
276 self.on_target_created(event);
277 }
278
279 let _ = tx.send(Ok(results)).ok();
280 }
281 Err(err) => {
282 let _ = tx.send(Err(err)).ok();
283 }
284 }
285 }
286 PendingRequest::Navigate(id) => {
287 self.on_navigation_response(id, resp);
288 if self.config.only_html && !self.config.created_first_target {
289 self.config.created_first_target = true;
290 }
291 }
292 PendingRequest::ExternalCommand { tx, .. } => {
293 let _ = tx.send(Ok(resp)).ok();
294 }
295 PendingRequest::InternalCommand(target_id) => {
296 if let Some(target) = self.targets.get_mut(&target_id) {
297 maybe_store_attach_session_id(target, &method, &resp);
298 target.on_response(resp, method.as_ref());
299 }
300 }
301 PendingRequest::CloseBrowser(tx) => {
302 self.closing = true;
303 let _ = tx.send(Ok(CloseReturns {})).ok();
304 }
305 }
306 }
307 }
308
309 pub(crate) fn submit_external_command(
311 &mut self,
312 msg: CommandMessage,
313 now: Instant,
314 ) -> Result<()> {
315 let target_id = msg
320 .session_id
321 .as_ref()
322 .and_then(|sid| self.sessions.get(sid.as_ref()))
323 .map(|s| s.target_id().clone());
324 let call_id =
325 self.conn()?
326 .submit_command(msg.method.clone(), msg.session_id, msg.params)?;
327 self.pending_commands.insert(
328 call_id,
329 (
330 PendingRequest::ExternalCommand {
331 tx: msg.sender,
332 target_id,
333 },
334 msg.method,
335 now,
336 ),
337 );
338 Ok(())
339 }
340
341 pub(crate) fn submit_internal_command(
342 &mut self,
343 target_id: TargetId,
344 req: CdpRequest,
345 now: Instant,
346 ) -> Result<()> {
347 let call_id = self.conn()?.submit_command(
348 req.method.clone(),
349 req.session_id.map(Into::into),
350 req.params,
351 )?;
352 self.pending_commands.insert(
353 call_id,
354 (PendingRequest::InternalCommand(target_id), req.method, now),
355 );
356 Ok(())
357 }
358
359 fn submit_fetch_targets(&mut self, tx: OneshotSender<Result<Vec<TargetInfo>>>, now: Instant) {
360 let msg = TARGET_PARAMS_ID.clone();
361
362 if let Some(conn) = self.conn.as_mut() {
363 if let Ok(call_id) = conn.submit_command(msg.0.clone(), None, msg.1) {
364 self.pending_commands
365 .insert(call_id, (PendingRequest::GetTargets(tx), msg.0, now));
366 }
367 }
368 }
369
370 fn submit_navigation(&mut self, id: NavigationId, req: CdpRequest, now: Instant) {
373 if let Some(conn) = self.conn.as_mut() {
374 if let Ok(call_id) = conn.submit_command(
375 req.method.clone(),
376 req.session_id.map(Into::into),
377 req.params,
378 ) {
379 self.pending_commands
380 .insert(call_id, (PendingRequest::Navigate(id), req.method, now));
381 }
382 }
383 }
384
385 fn submit_close(&mut self, tx: OneshotSender<Result<CloseReturns>>, now: Instant) {
386 let close_msg = CLOSE_PARAMS_ID.clone();
387
388 if let Some(conn) = self.conn.as_mut() {
389 if let Ok(call_id) = conn.submit_command(close_msg.0.clone(), None, close_msg.1) {
390 self.pending_commands.insert(
391 call_id,
392 (PendingRequest::CloseBrowser(tx), close_msg.0, now),
393 );
394 }
395 }
396 }
397
398 fn on_target_message(&mut self, target: &mut Target, msg: CommandMessage, now: Instant) {
400 if msg.is_navigation() {
401 let (req, tx) = msg.split();
402 let id = self.next_navigation_id();
403
404 target.goto(FrameRequestedNavigation::new(
405 id,
406 req,
407 self.config.request_timeout,
408 ));
409
410 self.navigations.insert(
411 id,
412 NavigationRequest::Navigate(NavigationInProgress::new(tx)),
413 );
414 } else {
415 let _ = self.submit_external_command(msg, now);
416 }
417 }
418
419 fn next_navigation_id(&mut self) -> NavigationId {
421 let id = NavigationId(self.next_navigation_id);
422 self.next_navigation_id = self.next_navigation_id.wrapping_add(1);
423 id
424 }
425
426 fn create_page(&mut self, params: CreateTargetParams, tx: OneshotSender<Result<Page>>) {
437 let about_blank = params.url == "about:blank";
438 let http_check =
439 !about_blank && params.url.starts_with("http") || params.url.starts_with("file://");
440
441 if about_blank || http_check {
442 let method = params.identifier();
443
444 let Some(conn) = self.conn.as_mut() else {
445 let _ = tx.send(Err(CdpError::msg("connection consumed"))).ok();
446 return;
447 };
448 match serde_json::to_value(params) {
449 Ok(params) => match conn.submit_command(method.clone(), None, params) {
450 Ok(call_id) => {
451 self.pending_commands.insert(
452 call_id,
453 (PendingRequest::CreateTarget(tx), method, Instant::now()),
454 );
455 }
456 Err(err) => {
457 let _ = tx.send(Err(err.into())).ok();
458 }
459 },
460 Err(err) => {
461 let _ = tx.send(Err(err.into())).ok();
462 }
463 }
464 } else {
465 let _ = tx.send(Err(CdpError::NotFound)).ok();
466 }
467 }
468
469 fn on_event(&mut self, event: CdpEventMessage) {
471 if let Some(session_id) = &event.session_id {
472 if let Some(session) = self.sessions.get(session_id.as_str()) {
473 if let Some(target) = self.targets.get_mut(session.target_id()) {
474 return target.on_event(event);
475 }
476 }
477 }
478 let CdpEventMessage { params, method, .. } = event;
479
480 match params {
481 CdpEvent::TargetTargetCreated(ref ev) => self.on_target_created((**ev).clone()),
482 CdpEvent::TargetAttachedToTarget(ref ev) => self.on_attached_to_target(ev.clone()),
483 CdpEvent::TargetTargetDestroyed(ref ev) => self.on_target_destroyed(ev.clone()),
484 CdpEvent::TargetTargetCrashed(ref ev) => self.on_target_crashed(ev.clone()),
485 CdpEvent::TargetDetachedFromTarget(ref ev) => self.on_detached_from_target(ev.clone()),
486 _ => {}
487 }
488
489 chromiumoxide_cdp::consume_event!(match params {
490 |ev| self.event_listeners.start_send(ev),
491 |json| { let _ = self.event_listeners.try_send_custom(&method, json);}
492 });
493 }
494
495 fn on_target_created(&mut self, event: EventTargetCreated) {
499 if !self.browser_contexts.is_empty() {
500 if let Some(ref context_id) = event.target_info.browser_context_id {
501 let bc = BrowserContext {
502 id: Some(context_id.clone()),
503 };
504 if !self.browser_contexts.contains(&bc) {
505 return;
506 }
507 }
508 }
509 let browser_ctx = event
510 .target_info
511 .browser_context_id
512 .clone()
513 .map(BrowserContext::from)
514 .unwrap_or_else(|| self.default_browser_context.clone());
515 let target = Target::new(
516 event.target_info,
517 TargetConfig {
518 ignore_https_errors: self.config.ignore_https_errors,
519 request_timeout: self.config.request_timeout,
520 viewport: self.config.viewport.clone(),
521 request_intercept: self.config.request_intercept,
522 cache_enabled: self.config.cache_enabled,
523 service_worker_enabled: self.config.service_worker_enabled,
524 ignore_visuals: self.config.ignore_visuals,
525 ignore_stylesheets: self.config.ignore_stylesheets,
526 ignore_javascript: self.config.ignore_javascript,
527 ignore_analytics: self.config.ignore_analytics,
528 ignore_prefetch: self.config.ignore_prefetch,
529 extra_headers: self.config.extra_headers.clone(),
530 only_html: self.config.only_html && self.config.created_first_target,
531 intercept_manager: self.config.intercept_manager,
532 max_bytes_allowed: self.config.max_bytes_allowed,
533 max_redirects: self.config.max_redirects,
534 whitelist_patterns: self.config.whitelist_patterns.clone(),
535 blacklist_patterns: self.config.blacklist_patterns.clone(),
536 #[cfg(feature = "adblock")]
537 adblock_filter_rules: self.config.adblock_filter_rules.clone(),
538 page_wake: self.page_wake.clone(),
539 },
540 browser_ctx,
541 );
542
543 let tid = target.target_id().clone();
544 self.target_ids.push(tid.clone());
545 self.targets.insert(tid, target);
546 }
547
548 fn on_attached_to_target(&mut self, event: Box<EventAttachedToTarget>) {
550 let session = Session::new(event.session_id.clone(), event.target_info.target_id);
551 if let Some(target) = self.targets.get_mut(session.target_id()) {
552 target.set_session_id(session.session_id().clone())
553 }
554 self.sessions.insert(event.session_id, session);
555 }
556
557 fn on_detached_from_target(&mut self, event: EventDetachedFromTarget) {
561 if let Some(session) = self.sessions.remove(&event.session_id) {
563 if let Some(target) = self.targets.get_mut(session.target_id()) {
564 target.session_id_mut().take();
565 }
566 }
567 }
568
569 fn on_target_destroyed(&mut self, event: EventTargetDestroyed) {
571 self.attached_targets.remove(&event.target_id);
572
573 if let Some(target) = self.targets.remove(&event.target_id) {
574 if let Some(session) = target.session_id() {
576 self.sessions.remove(session);
577 }
578 }
579 }
580
581 fn on_target_crashed(&mut self, event: EventTargetCrashed) {
604 let crashed_id = event.target_id.clone();
605 let status = event.status.clone();
606 let error_code = event.error_code;
607
608 let to_cancel: Vec<CallId> = self
614 .pending_commands
615 .iter()
616 .filter_map(|(&call_id, (req, _, _))| match req {
617 PendingRequest::ExternalCommand {
618 target_id: Some(tid),
619 ..
620 } if *tid == crashed_id => Some(call_id),
621 PendingRequest::InternalCommand(tid) if *tid == crashed_id => Some(call_id),
622 _ => None,
623 })
624 .collect();
625
626 for call_id in to_cancel {
627 if let Some((req, _, _)) = self.pending_commands.remove(&call_id) {
628 match req {
629 PendingRequest::ExternalCommand { tx, .. } => {
630 let _ = tx.send(Err(CdpError::msg(format!(
631 "target {:?} crashed: {} (errorCode={})",
632 crashed_id, status, error_code
633 ))));
634 }
635 PendingRequest::InternalCommand(_) => {
636 }
639 _ => {}
640 }
641 }
642 }
643
644 self.attached_targets.remove(&crashed_id);
646 if let Some(target) = self.targets.remove(&crashed_id) {
647 if let Some(session) = target.session_id() {
648 self.sessions.remove(session);
649 }
650 }
651 }
652
653 fn evict_timed_out_commands(&mut self, now: Instant) {
658 let deadline = match now.checked_sub(self.config.request_timeout) {
659 Some(d) => d,
660 None => return,
661 };
662
663 let timed_out: Vec<_> = self
664 .pending_commands
665 .iter()
666 .filter(|(_, (_, _, timestamp))| *timestamp < deadline)
667 .map(|(k, _)| *k)
668 .collect();
669
670 for call in timed_out {
671 if let Some((req, _, _)) = self.pending_commands.remove(&call) {
672 match req {
673 PendingRequest::CreateTarget(tx) => {
674 let _ = tx.send(Err(CdpError::Timeout));
675 }
676 PendingRequest::GetTargets(tx) => {
677 let _ = tx.send(Err(CdpError::Timeout));
678 }
679 PendingRequest::Navigate(nav) => {
680 if let Some(nav) = self.navigations.remove(&nav) {
681 match nav {
682 NavigationRequest::Navigate(nav) => {
683 let _ = nav.tx.send(Err(CdpError::Timeout));
684 }
685 }
686 }
687 }
688 PendingRequest::ExternalCommand { tx, .. } => {
689 let _ = tx.send(Err(CdpError::Timeout));
690 }
691 PendingRequest::InternalCommand(_) => {}
692 PendingRequest::CloseBrowser(tx) => {
693 let _ = tx.send(Err(CdpError::Timeout));
694 }
695 }
696 }
697 }
698 }
699
700 pub fn event_listeners_mut(&mut self) -> &mut EventListeners {
701 &mut self.event_listeners
702 }
703
704 pub async fn run(mut self) -> Result<()> {
734 use chromiumoxide_types::Message;
735 use tokio::time::MissedTickBehavior;
736 use tokio_tungstenite::tungstenite::{self, error::ProtocolError};
737
738 let page_wake = Arc::new(Notify::new());
740 self.page_wake = Some(page_wake.clone());
741
742 let conn = self
744 .conn
745 .take()
746 .ok_or_else(|| CdpError::msg("Handler::run() called with no connection"))?;
747 let async_conn = conn.into_async();
748 let mut ws_reader = async_conn.reader;
749 let ws_tx = async_conn.cmd_tx;
750 let mut writer_handle = async_conn.writer_handle;
751 let mut next_call_id = async_conn.next_id;
752
753 let mut alloc_call_id = || {
755 let id = chromiumoxide_types::CallId::new(next_call_id);
756 next_call_id = next_call_id.wrapping_add(1);
757 id
758 };
759
760 let mut evict_timer = tokio::time::interval_at(
762 tokio::time::Instant::now() + self.config.request_timeout,
763 self.config.request_timeout,
764 );
765 evict_timer.set_missed_tick_behavior(MissedTickBehavior::Delay);
766
767 macro_rules! ws_submit {
769 ($method:expr, $session_id:expr, $params:expr) => {{
770 let id = alloc_call_id();
771 let call = chromiumoxide_types::MethodCall {
772 id,
773 method: $method,
774 session_id: $session_id,
775 params: $params,
776 };
777 match ws_tx.try_send(call) {
778 Ok(()) => Ok::<_, CdpError>(id),
779 Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => {
780 tracing::warn!("WS command channel full — dropping command");
781 Err(CdpError::msg("WS command channel full"))
782 }
783 Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {
784 Err(CdpError::msg("WS writer closed"))
785 }
786 }
787 }};
788 }
789
790 loop {
792 let now = std::time::Instant::now();
793
794 const PER_TARGET_DRAIN_BUDGET: usize = 128;
800
801 for n in (0..self.target_ids.len()).rev() {
802 let target_id = self.target_ids.swap_remove(n);
803
804 if let Some((id, mut target)) = self.targets.remove_entry(&target_id) {
805 {
807 let mut msgs = Vec::new();
808 if let Some(handle) = target.page_mut() {
809 while msgs.len() < PER_TARGET_DRAIN_BUDGET {
810 match handle.rx.try_recv() {
811 Ok(msg) => msgs.push(msg),
812 Err(_) => break,
813 }
814 }
815 }
816 for msg in msgs {
817 target.on_page_message(msg);
818 }
819 }
820
821 while let Some(event) = target.advance(now) {
823 match event {
824 TargetEvent::Request(req) => {
825 if let Ok(call_id) =
826 ws_submit!(req.method.clone(), req.session_id, req.params)
827 {
828 self.pending_commands.insert(
829 call_id,
830 (
831 PendingRequest::InternalCommand(
832 target.target_id().clone(),
833 ),
834 req.method,
835 now,
836 ),
837 );
838 }
839 }
840 TargetEvent::Command(msg) => {
841 if msg.is_navigation() {
842 let (req, tx) = msg.split();
843 let nav_id = self.next_navigation_id();
844 target.goto(FrameRequestedNavigation::new(
845 nav_id,
846 req.clone(),
847 self.config.request_timeout,
848 ));
849 if let Ok(call_id) =
850 ws_submit!(req.method.clone(), req.session_id, req.params)
851 {
852 self.pending_commands.insert(
853 call_id,
854 (PendingRequest::Navigate(nav_id), req.method, now),
855 );
856 }
857 self.navigations.insert(
858 nav_id,
859 NavigationRequest::Navigate(NavigationInProgress::new(tx)),
860 );
861 } else if let Ok(call_id) = ws_submit!(
862 msg.method.clone(),
863 msg.session_id.map(Into::into),
864 msg.params
865 ) {
866 let target_id = Some(target.target_id().clone());
870 self.pending_commands.insert(
871 call_id,
872 (
873 PendingRequest::ExternalCommand {
874 tx: msg.sender,
875 target_id,
876 },
877 msg.method,
878 now,
879 ),
880 );
881 }
882 }
883 TargetEvent::NavigationRequest(nav_id, req) => {
884 if let Ok(call_id) =
885 ws_submit!(req.method.clone(), req.session_id, req.params)
886 {
887 self.pending_commands.insert(
888 call_id,
889 (PendingRequest::Navigate(nav_id), req.method, now),
890 );
891 }
892 }
893 TargetEvent::NavigationResult(res) => {
894 self.on_navigation_lifecycle_completed(res);
895 }
896 TargetEvent::BytesConsumed(n) => {
897 if let Some(rem) = self.remaining_bytes.as_mut() {
898 *rem = rem.saturating_sub(n);
899 if *rem == 0 {
900 self.budget_exhausted = true;
901 }
902 }
903 }
904 }
905 }
906
907 target.event_listeners_mut().flush();
909
910 self.targets.insert(id, target);
911 self.target_ids.push(target_id);
912 }
913 }
914
915 self.event_listeners.flush();
917
918 if self.budget_exhausted {
919 for t in self.targets.values_mut() {
920 t.network_manager.set_block_all(true);
921 }
922 }
923
924 if self.closing {
925 break;
926 }
927
928 tokio::select! {
930 msg = self.from_browser.recv() => {
931 match msg {
932 Some(msg) => {
933 match msg {
934 HandlerMessage::Command(cmd) => {
935 let target_id = cmd
938 .session_id
939 .as_ref()
940 .and_then(|sid| self.sessions.get(sid.as_ref()))
941 .map(|s| s.target_id().clone());
942 if let Ok(call_id) = ws_submit!(
943 cmd.method.clone(),
944 cmd.session_id.map(Into::into),
945 cmd.params
946 ) {
947 self.pending_commands.insert(
948 call_id,
949 (
950 PendingRequest::ExternalCommand {
951 tx: cmd.sender,
952 target_id,
953 },
954 cmd.method,
955 now,
956 ),
957 );
958 }
959 }
960 HandlerMessage::FetchTargets(tx) => {
961 let msg = TARGET_PARAMS_ID.clone();
962 if let Ok(call_id) = ws_submit!(msg.0.clone(), None, msg.1) {
963 self.pending_commands.insert(
964 call_id,
965 (PendingRequest::GetTargets(tx), msg.0, now),
966 );
967 }
968 }
969 HandlerMessage::CloseBrowser(tx) => {
970 let close_msg = CLOSE_PARAMS_ID.clone();
971 if let Ok(call_id) = ws_submit!(close_msg.0.clone(), None, close_msg.1) {
972 self.pending_commands.insert(
973 call_id,
974 (PendingRequest::CloseBrowser(tx), close_msg.0, now),
975 );
976 }
977 }
978 HandlerMessage::CreatePage(params, tx) => {
979 if let Some(ref id) = params.browser_context_id {
980 self.browser_contexts.insert(BrowserContext::from(id.clone()));
981 }
982 self.create_page_async(params, tx, &mut alloc_call_id, &ws_tx, now);
983 }
984 HandlerMessage::GetPages(tx) => {
985 let pages: Vec<_> = self.targets.values_mut()
986 .filter(|p| p.is_page())
987 .filter_map(|target| target.get_or_create_page())
988 .map(|page| Page::from(page.clone()))
989 .collect();
990 let _ = tx.send(pages);
991 }
992 HandlerMessage::InsertContext(ctx) => {
993 if self.default_browser_context.id().is_none() {
994 self.default_browser_context = ctx.clone();
995 }
996 self.browser_contexts.insert(ctx);
997 }
998 HandlerMessage::DisposeContext(ctx) => {
999 self.browser_contexts.remove(&ctx);
1000 self.attached_targets.retain(|tid| {
1001 self.targets.get(tid)
1002 .and_then(|t| t.browser_context_id())
1003 .map(|id| Some(id) != ctx.id())
1004 .unwrap_or(true)
1005 });
1006 self.closing = true;
1007 }
1008 HandlerMessage::GetPage(target_id, tx) => {
1009 let page = self.targets.get_mut(&target_id)
1010 .and_then(|target| target.get_or_create_page())
1011 .map(|page| Page::from(page.clone()));
1012 let _ = tx.send(page);
1013 }
1014 HandlerMessage::AddEventListener(req) => {
1015 self.event_listeners.add_listener(req);
1016 }
1017 }
1018 }
1019 None => break, }
1021 }
1022
1023 frame = ws_reader.next_message() => {
1024 match frame {
1025 Some(Ok(boxed_msg)) => match *boxed_msg {
1026 Message::Response(resp) => {
1027 self.on_response(resp);
1028 }
1029 Message::Event(ev) => {
1030 self.on_event(ev);
1031 }
1032 },
1033 Some(Err(err)) => {
1034 tracing::error!("WS Connection error: {:?}", err);
1035 if let CdpError::Ws(ref ws_error) = err {
1036 match ws_error {
1037 tungstenite::Error::AlreadyClosed => break,
1038 tungstenite::Error::Protocol(detail)
1039 if detail == &ProtocolError::ResetWithoutClosingHandshake =>
1040 {
1041 break;
1042 }
1043 _ => return Err(err),
1044 }
1045 } else {
1046 return Err(err);
1047 }
1048 }
1049 None => break, }
1051 }
1052
1053 _ = page_wake.notified() => {
1054 }
1056
1057 _ = evict_timer.tick() => {
1058 self.evict_timed_out_commands(now);
1059 for t in self.targets.values_mut() {
1060 t.network_manager.evict_stale_entries(now);
1061 t.frame_manager_mut().evict_stale_context_ids();
1062 }
1063 }
1064
1065 result = &mut writer_handle => {
1066 match result {
1068 Ok(Ok(())) => break,
1069 Ok(Err(e)) => return Err(e),
1070 Err(e) => return Err(CdpError::msg(format!("WS writer panicked: {e}"))),
1071 }
1072 }
1073 }
1074 }
1075
1076 writer_handle.abort();
1077 Ok(())
1078 }
1079
1080 fn create_page_async(
1082 &mut self,
1083 params: CreateTargetParams,
1084 tx: OneshotSender<Result<Page>>,
1085 alloc_call_id: &mut impl FnMut() -> chromiumoxide_types::CallId,
1086 ws_tx: &tokio::sync::mpsc::Sender<chromiumoxide_types::MethodCall>,
1087 now: std::time::Instant,
1088 ) {
1089 let about_blank = params.url == "about:blank";
1090 let http_check =
1091 !about_blank && params.url.starts_with("http") || params.url.starts_with("file://");
1092
1093 if about_blank || http_check {
1094 let method = params.identifier();
1095 match serde_json::to_value(params) {
1096 Ok(params) => {
1097 let id = alloc_call_id();
1098 let call = chromiumoxide_types::MethodCall {
1099 id,
1100 method: method.clone(),
1101 session_id: None,
1102 params,
1103 };
1104 match ws_tx.try_send(call) {
1105 Ok(()) => {
1106 self.pending_commands
1107 .insert(id, (PendingRequest::CreateTarget(tx), method, now));
1108 }
1109 Err(_) => {
1110 let _ = tx
1111 .send(Err(CdpError::msg("WS command channel full or closed")))
1112 .ok();
1113 }
1114 }
1115 }
1116 Err(err) => {
1117 let _ = tx.send(Err(err.into())).ok();
1118 }
1119 }
1120 } else {
1121 let _ = tx.send(Err(CdpError::NotFound)).ok();
1122 }
1123 }
1124}
1125
1126impl Stream for Handler {
1127 type Item = Result<()>;
1128
1129 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
1130 let pin = self.get_mut();
1131
1132 let mut dispose = false;
1133
1134 let now = Instant::now();
1135
1136 loop {
1137 while let Poll::Ready(Some(msg)) = pin.from_browser.poll_recv(cx) {
1141 match msg {
1142 HandlerMessage::Command(cmd) => {
1143 pin.submit_external_command(cmd, now)?;
1144 }
1145 HandlerMessage::FetchTargets(tx) => {
1146 pin.submit_fetch_targets(tx, now);
1147 }
1148 HandlerMessage::CloseBrowser(tx) => {
1149 pin.submit_close(tx, now);
1150 }
1151 HandlerMessage::CreatePage(params, tx) => {
1152 if let Some(ref id) = params.browser_context_id {
1153 pin.browser_contexts
1154 .insert(BrowserContext::from(id.clone()));
1155 }
1156 pin.create_page(params, tx);
1157 }
1158 HandlerMessage::GetPages(tx) => {
1159 let pages: Vec<_> = pin
1160 .targets
1161 .values_mut()
1162 .filter(|p: &&mut Target| p.is_page())
1163 .filter_map(|target| target.get_or_create_page())
1164 .map(|page| Page::from(page.clone()))
1165 .collect();
1166 let _ = tx.send(pages);
1167 }
1168 HandlerMessage::InsertContext(ctx) => {
1169 if pin.default_browser_context.id().is_none() {
1170 pin.default_browser_context = ctx.clone();
1171 }
1172 pin.browser_contexts.insert(ctx);
1173 }
1174 HandlerMessage::DisposeContext(ctx) => {
1175 pin.browser_contexts.remove(&ctx);
1176 pin.attached_targets.retain(|tid| {
1177 pin.targets
1178 .get(tid)
1179 .and_then(|t| t.browser_context_id()) .map(|id| Some(id) != ctx.id())
1181 .unwrap_or(true)
1182 });
1183 pin.closing = true;
1184 dispose = true;
1185 }
1186 HandlerMessage::GetPage(target_id, tx) => {
1187 let page = pin
1188 .targets
1189 .get_mut(&target_id)
1190 .and_then(|target| target.get_or_create_page())
1191 .map(|page| Page::from(page.clone()));
1192 let _ = tx.send(page);
1193 }
1194 HandlerMessage::AddEventListener(req) => {
1195 pin.event_listeners.add_listener(req);
1196 }
1197 }
1198 }
1199
1200 for n in (0..pin.target_ids.len()).rev() {
1201 let target_id = pin.target_ids.swap_remove(n);
1202
1203 if let Some((id, mut target)) = pin.targets.remove_entry(&target_id) {
1204 while let Some(event) = target.poll(cx, now) {
1205 match event {
1206 TargetEvent::Request(req) => {
1207 let _ = pin.submit_internal_command(
1208 target.target_id().clone(),
1209 req,
1210 now,
1211 );
1212 }
1213 TargetEvent::Command(msg) => {
1214 pin.on_target_message(&mut target, msg, now);
1215 }
1216 TargetEvent::NavigationRequest(id, req) => {
1217 pin.submit_navigation(id, req, now);
1218 }
1219 TargetEvent::NavigationResult(res) => {
1220 pin.on_navigation_lifecycle_completed(res)
1221 }
1222 TargetEvent::BytesConsumed(n) => {
1223 if let Some(rem) = pin.remaining_bytes.as_mut() {
1224 *rem = rem.saturating_sub(n);
1225 if *rem == 0 {
1226 pin.budget_exhausted = true;
1227 }
1228 }
1229 }
1230 }
1231 }
1232
1233 target.event_listeners_mut().poll(cx);
1235
1236 pin.targets.insert(id, target);
1237 pin.target_ids.push(target_id);
1238 }
1239 }
1240
1241 pin.event_listeners_mut().poll(cx);
1244
1245 let mut done = true;
1246
1247 let mut ws_msgs = Vec::new();
1250 let mut ws_err = None;
1251 {
1252 let Some(conn) = pin.conn.as_mut() else {
1253 return Poll::Ready(Some(Err(CdpError::msg(
1254 "connection consumed by Handler::run()",
1255 ))));
1256 };
1257 while let Poll::Ready(Some(ev)) = Pin::new(&mut *conn).poll_next(cx) {
1258 match ev {
1259 Ok(msg) => ws_msgs.push(msg),
1260 Err(err) => {
1261 ws_err = Some(err);
1262 break;
1263 }
1264 }
1265 }
1266 }
1267
1268 for boxed_msg in ws_msgs {
1269 match *boxed_msg {
1270 Message::Response(resp) => {
1271 pin.on_response(resp);
1272 if pin.closing {
1273 return Poll::Ready(None);
1274 }
1275 }
1276 Message::Event(ev) => {
1277 pin.on_event(ev);
1278 }
1279 }
1280 done = false;
1281 }
1282
1283 if let Some(err) = ws_err {
1284 tracing::error!("WS Connection error: {:?}", err);
1285 if let CdpError::Ws(ref ws_error) = err {
1286 match ws_error {
1287 Error::AlreadyClosed => {
1288 pin.closing = true;
1289 dispose = true;
1290 }
1291 Error::Protocol(detail)
1292 if detail == &ProtocolError::ResetWithoutClosingHandshake =>
1293 {
1294 pin.closing = true;
1295 dispose = true;
1296 }
1297 _ => return Poll::Ready(Some(Err(err))),
1298 }
1299 } else {
1300 return Poll::Ready(Some(Err(err)));
1301 }
1302 }
1303
1304 if pin.evict_command_timeout.poll_ready(cx) {
1305 pin.evict_timed_out_commands(now);
1307 for t in pin.targets.values_mut() {
1310 t.network_manager.evict_stale_entries(now);
1311 t.frame_manager_mut().evict_stale_context_ids();
1312 }
1313 }
1314
1315 if pin.budget_exhausted {
1316 for t in pin.targets.values_mut() {
1317 t.network_manager.set_block_all(true);
1318 }
1319 }
1320
1321 if dispose {
1322 return Poll::Ready(None);
1323 }
1324
1325 if done {
1326 return Poll::Pending;
1328 }
1329 }
1330 }
1331}
1332
1333#[derive(Debug, Clone)]
1335pub struct HandlerConfig {
1336 pub ignore_https_errors: bool,
1338 pub viewport: Option<Viewport>,
1340 pub context_ids: Vec<BrowserContextId>,
1342 pub request_timeout: Duration,
1344 pub request_intercept: bool,
1346 pub cache_enabled: bool,
1348 pub service_worker_enabled: bool,
1350 pub ignore_visuals: bool,
1352 pub ignore_stylesheets: bool,
1354 pub ignore_javascript: bool,
1356 pub ignore_analytics: bool,
1358 pub ignore_prefetch: bool,
1360 pub ignore_ads: bool,
1362 pub extra_headers: Option<std::collections::HashMap<String, String>>,
1364 pub only_html: bool,
1366 pub created_first_target: bool,
1368 pub intercept_manager: NetworkInterceptManager,
1370 pub max_bytes_allowed: Option<u64>,
1372 pub max_redirects: Option<usize>,
1379 pub whitelist_patterns: Option<Vec<String>>,
1381 pub blacklist_patterns: Option<Vec<String>>,
1383 #[cfg(feature = "adblock")]
1385 pub adblock_filter_rules: Option<Vec<String>>,
1386 pub channel_capacity: usize,
1389 pub connection_retries: u32,
1392}
1393
1394impl Default for HandlerConfig {
1395 fn default() -> Self {
1396 Self {
1397 ignore_https_errors: true,
1398 viewport: Default::default(),
1399 context_ids: Vec::new(),
1400 request_timeout: Duration::from_millis(REQUEST_TIMEOUT),
1401 request_intercept: false,
1402 cache_enabled: true,
1403 service_worker_enabled: true,
1404 ignore_visuals: false,
1405 ignore_stylesheets: false,
1406 ignore_ads: false,
1407 ignore_javascript: false,
1408 ignore_analytics: true,
1409 ignore_prefetch: true,
1410 only_html: false,
1411 extra_headers: Default::default(),
1412 created_first_target: false,
1413 intercept_manager: NetworkInterceptManager::Unknown,
1414 max_bytes_allowed: None,
1415 max_redirects: None,
1416 whitelist_patterns: None,
1417 blacklist_patterns: None,
1418 #[cfg(feature = "adblock")]
1419 adblock_filter_rules: None,
1420 channel_capacity: 4096,
1421 connection_retries: crate::conn::DEFAULT_CONNECTION_RETRIES,
1422 }
1423 }
1424}
1425
1426#[derive(Debug)]
1428pub struct NavigationInProgress<T> {
1429 navigated: bool,
1431 response: Option<Response>,
1433 tx: OneshotSender<T>,
1435}
1436
1437impl<T> NavigationInProgress<T> {
1438 fn new(tx: OneshotSender<T>) -> Self {
1439 Self {
1440 navigated: false,
1441 response: None,
1442 tx,
1443 }
1444 }
1445
1446 fn set_response(&mut self, resp: Response) {
1448 self.response = Some(resp);
1449 }
1450
1451 fn set_navigated(&mut self) {
1453 self.navigated = true;
1454 }
1455}
1456
1457#[derive(Debug)]
1459enum NavigationRequest {
1460 Navigate(NavigationInProgress<Result<Response>>),
1462 }
1464
1465#[derive(Debug)]
1468enum PendingRequest {
1469 CreateTarget(OneshotSender<Result<Page>>),
1472 GetTargets(OneshotSender<Result<Vec<TargetInfo>>>),
1474 Navigate(NavigationId),
1481 ExternalCommand {
1490 tx: OneshotSender<Result<Response>>,
1491 target_id: Option<TargetId>,
1492 },
1493 InternalCommand(TargetId),
1496 CloseBrowser(OneshotSender<Result<CloseReturns>>),
1498}
1499
1500#[derive(Debug)]
1504pub(crate) enum HandlerMessage {
1505 CreatePage(CreateTargetParams, OneshotSender<Result<Page>>),
1506 FetchTargets(OneshotSender<Result<Vec<TargetInfo>>>),
1507 InsertContext(BrowserContext),
1508 DisposeContext(BrowserContext),
1509 GetPages(OneshotSender<Vec<Page>>),
1510 Command(CommandMessage),
1511 GetPage(TargetId, OneshotSender<Option<Page>>),
1512 AddEventListener(EventListenerRequest),
1513 CloseBrowser(OneshotSender<Result<CloseReturns>>),
1514}
1515
1516#[cfg(test)]
1517mod tests {
1518 use super::*;
1519 use chromiumoxide_cdp::cdp::browser_protocol::target::{AttachToTargetReturns, TargetInfo};
1520
1521 #[test]
1522 fn attach_to_target_response_sets_session_id_before_event_arrives() {
1523 let info = TargetInfo::builder()
1524 .target_id("target-1".to_string())
1525 .r#type("page")
1526 .title("")
1527 .url("about:blank")
1528 .attached(false)
1529 .can_access_opener(false)
1530 .build()
1531 .expect("target info");
1532 let mut target = Target::new(info, TargetConfig::default(), BrowserContext::default());
1533 let method: MethodId = AttachToTargetParams::IDENTIFIER.into();
1534 let result = serde_json::to_value(AttachToTargetReturns::new("session-1".to_string()))
1535 .expect("attach result");
1536 let resp = Response {
1537 id: CallId::new(1),
1538 result: Some(result),
1539 error: None,
1540 };
1541
1542 maybe_store_attach_session_id(&mut target, &method, &resp);
1543
1544 assert_eq!(
1545 target.session_id().map(AsRef::as_ref),
1546 Some("session-1"),
1547 "attach response should seed the flat session id even before Target.attachedToTarget"
1548 );
1549 }
1550}