1use crate::listeners::{EventListenerRequest, EventListeners};
2use chromiumoxide_cdp::cdp::browser_protocol::browser::*;
3use chromiumoxide_cdp::cdp::browser_protocol::target::*;
4use chromiumoxide_cdp::cdp::events::CdpEvent;
5use chromiumoxide_cdp::cdp::events::CdpEventMessage;
6use chromiumoxide_types::{CallId, Message, Method, Response};
7use chromiumoxide_types::{MethodId, Request as CdpRequest};
8use fnv::FnvHashMap;
9use futures_util::Stream;
10use hashbrown::{HashMap, HashSet};
11use spider_network_blocker::intercept_manager::NetworkInterceptManager;
12use std::pin::Pin;
13use std::task::{Context, Poll};
14use std::time::{Duration, Instant};
15use tokio::sync::mpsc::Receiver;
16use tokio::sync::oneshot::Sender as OneshotSender;
17use tokio_tungstenite::tungstenite::error::ProtocolError;
18use tokio_tungstenite::tungstenite::Error;
19
20use std::sync::Arc;
21use tokio::sync::Notify;
22
23use crate::cmd::{to_command_response, CommandMessage};
24use crate::conn::Connection;
25use crate::error::{CdpError, Result};
26use crate::handler::browser::BrowserContext;
27use crate::handler::frame::FrameRequestedNavigation;
28use crate::handler::frame::{NavigationError, NavigationId, NavigationOk};
29use crate::handler::job::PeriodicJob;
30use crate::handler::session::Session;
31use crate::handler::target::TargetEvent;
32use crate::handler::target::{Target, TargetConfig};
33use crate::handler::viewport::Viewport;
34use crate::page::Page;
35pub(crate) use page::PageInner;
36
37pub const REQUEST_TIMEOUT: u64 = 30_000;
39
40pub mod blockers;
41pub mod browser;
42pub mod commandfuture;
43pub mod domworld;
44pub mod emulation;
45pub mod frame;
46pub mod http;
47pub mod httpfuture;
48mod job;
49pub mod network;
50pub mod network_utils;
51pub mod page;
52pub mod sender;
53mod session;
54pub mod target;
55pub mod target_message_future;
56pub mod viewport;
57
58#[must_use = "streams do nothing unless polled"]
61#[derive(Debug)]
62pub struct Handler {
63 pub default_browser_context: BrowserContext,
64 pub browser_contexts: HashSet<BrowserContext>,
65 pending_commands: FnvHashMap<CallId, (PendingRequest, MethodId, Instant)>,
69 from_browser: Receiver<HandlerMessage>,
71 target_ids: Vec<TargetId>,
73 targets: HashMap<TargetId, Target>,
75 navigations: FnvHashMap<NavigationId, NavigationRequest>,
77 sessions: HashMap<SessionId, Session>,
81 conn: Option<Connection<CdpEventMessage>>,
84 evict_command_timeout: PeriodicJob,
86 next_navigation_id: usize,
88 config: HandlerConfig,
90 event_listeners: EventListeners,
92 closing: bool,
94 remaining_bytes: Option<u64>,
96 budget_exhausted: bool,
98 attached_targets: HashSet<TargetId>,
100 page_wake: Option<Arc<Notify>>,
103}
104
105lazy_static::lazy_static! {
106 static ref DISCOVER_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
108 let discover = SetDiscoverTargetsParams::new(true);
109 (discover.identifier(), serde_json::to_value(discover).expect("valid discover target params"))
110 };
111 static ref TARGET_PARAMS_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
113 let msg = GetTargetsParams { filter: None };
114 (msg.identifier(), serde_json::to_value(msg).expect("valid paramtarget"))
115 };
116 static ref CLOSE_PARAMS_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
118 let close_msg = CloseParams::default();
119 (close_msg.identifier(), serde_json::to_value(close_msg).expect("valid close params"))
120 };
121}
122
123fn maybe_store_attach_session_id(target: &mut Target, method: &MethodId, resp: &Response) {
124 if method.as_ref() != AttachToTargetParams::IDENTIFIER {
125 return;
126 }
127
128 if let Ok(resp) = to_command_response::<AttachToTargetParams>(resp.clone(), method.clone()) {
129 target.set_session_id(resp.result.session_id);
130 }
131}
132
133impl Handler {
134 pub(crate) fn new(
137 mut conn: Connection<CdpEventMessage>,
138 rx: Receiver<HandlerMessage>,
139 config: HandlerConfig,
140 ) -> Self {
141 let discover = DISCOVER_ID.clone();
142 let _ = conn.submit_command(discover.0, None, discover.1);
143 let conn = Some(conn);
144
145 let browser_contexts = config
146 .context_ids
147 .iter()
148 .map(|id| BrowserContext::from(id.clone()))
149 .collect();
150
151 Self {
152 pending_commands: Default::default(),
153 from_browser: rx,
154 default_browser_context: Default::default(),
155 browser_contexts,
156 target_ids: Default::default(),
157 targets: Default::default(),
158 navigations: Default::default(),
159 sessions: Default::default(),
160 conn,
161 evict_command_timeout: PeriodicJob::new(config.request_timeout),
162 next_navigation_id: 0,
163 config,
164 event_listeners: Default::default(),
165 closing: false,
166 remaining_bytes: None,
167 budget_exhausted: false,
168 attached_targets: Default::default(),
169 page_wake: None,
170 }
171 }
172
173 #[inline]
176 fn conn(&mut self) -> Result<&mut Connection<CdpEventMessage>> {
177 self.conn
178 .as_mut()
179 .ok_or_else(|| CdpError::msg("connection consumed by Handler::run()"))
180 }
181
182 pub fn get_target(&self, target_id: &TargetId) -> Option<&Target> {
184 self.targets.get(target_id)
185 }
186
187 pub fn targets(&self) -> impl Iterator<Item = &Target> + '_ {
189 self.targets.values()
190 }
191
192 pub fn default_browser_context(&self) -> &BrowserContext {
194 &self.default_browser_context
195 }
196
197 pub fn browser_contexts(&self) -> impl Iterator<Item = &BrowserContext> + '_ {
199 self.browser_contexts.iter()
200 }
201
202 fn on_navigation_response(&mut self, id: NavigationId, resp: Response) {
204 if let Some(nav) = self.navigations.remove(&id) {
205 match nav {
206 NavigationRequest::Navigate(mut nav) => {
207 if nav.navigated {
208 let _ = nav.tx.send(Ok(resp));
209 } else {
210 nav.set_response(resp);
211 self.navigations
212 .insert(id, NavigationRequest::Navigate(nav));
213 }
214 }
215 }
216 }
217 }
218
219 fn on_navigation_lifecycle_completed(&mut self, res: Result<NavigationOk, NavigationError>) {
221 match res {
222 Ok(ok) => {
223 let id = *ok.navigation_id();
224 if let Some(nav) = self.navigations.remove(&id) {
225 match nav {
226 NavigationRequest::Navigate(mut nav) => {
227 if let Some(resp) = nav.response.take() {
228 let _ = nav.tx.send(Ok(resp));
229 } else {
230 nav.set_navigated();
231 self.navigations
232 .insert(id, NavigationRequest::Navigate(nav));
233 }
234 }
235 }
236 }
237 }
238 Err(err) => {
239 if let Some(nav) = self.navigations.remove(err.navigation_id()) {
240 match nav {
241 NavigationRequest::Navigate(nav) => {
242 let _ = nav.tx.send(Err(err.into()));
243 }
244 }
245 }
246 }
247 }
248 }
249
250 fn on_response(&mut self, resp: Response) {
252 if let Some((req, method, _)) = self.pending_commands.remove(&resp.id) {
253 match req {
254 PendingRequest::CreateTarget(tx) => {
255 match to_command_response::<CreateTargetParams>(resp, method) {
256 Ok(resp) => {
257 if let Some(target) = self.targets.get_mut(&resp.target_id) {
258 target.set_initiator(tx);
259 } else {
260 let _ = tx.send(Err(CdpError::NotFound)).ok();
261 }
262 }
263 Err(err) => {
264 let _ = tx.send(Err(err)).ok();
265 }
266 }
267 }
268 PendingRequest::GetTargets(tx) => {
269 match to_command_response::<GetTargetsParams>(resp, method) {
270 Ok(resp) => {
271 let targets = resp.result.target_infos;
272 let results = targets.clone();
273
274 for target_info in targets {
275 let event: EventTargetCreated = EventTargetCreated { target_info };
276 self.on_target_created(event);
277 }
278
279 let _ = tx.send(Ok(results)).ok();
280 }
281 Err(err) => {
282 let _ = tx.send(Err(err)).ok();
283 }
284 }
285 }
286 PendingRequest::Navigate(id) => {
287 self.on_navigation_response(id, resp);
288 if self.config.only_html && !self.config.created_first_target {
289 self.config.created_first_target = true;
290 }
291 }
292 PendingRequest::ExternalCommand { tx, .. } => {
293 let _ = tx.send(Ok(resp)).ok();
294 }
295 PendingRequest::InternalCommand(target_id) => {
296 if let Some(target) = self.targets.get_mut(&target_id) {
297 maybe_store_attach_session_id(target, &method, &resp);
298 target.on_response(resp, method.as_ref());
299 }
300 }
301 PendingRequest::CloseBrowser(tx) => {
302 self.closing = true;
303 let _ = tx.send(Ok(CloseReturns {})).ok();
304 }
305 }
306 }
307 }
308
309 pub(crate) fn submit_external_command(
311 &mut self,
312 msg: CommandMessage,
313 now: Instant,
314 ) -> Result<()> {
315 let target_id = msg
320 .session_id
321 .as_ref()
322 .and_then(|sid| self.sessions.get(sid.as_ref()))
323 .map(|s| s.target_id().clone());
324 let call_id =
325 self.conn()?
326 .submit_command(msg.method.clone(), msg.session_id, msg.params)?;
327 self.pending_commands.insert(
328 call_id,
329 (
330 PendingRequest::ExternalCommand {
331 tx: msg.sender,
332 target_id,
333 },
334 msg.method,
335 now,
336 ),
337 );
338 Ok(())
339 }
340
341 pub(crate) fn submit_internal_command(
342 &mut self,
343 target_id: TargetId,
344 req: CdpRequest,
345 now: Instant,
346 ) -> Result<()> {
347 let call_id = self.conn()?.submit_command(
348 req.method.clone(),
349 req.session_id.map(Into::into),
350 req.params,
351 )?;
352 self.pending_commands.insert(
353 call_id,
354 (PendingRequest::InternalCommand(target_id), req.method, now),
355 );
356 Ok(())
357 }
358
359 fn submit_fetch_targets(&mut self, tx: OneshotSender<Result<Vec<TargetInfo>>>, now: Instant) {
360 let msg = TARGET_PARAMS_ID.clone();
361
362 if let Some(conn) = self.conn.as_mut() {
363 if let Ok(call_id) = conn.submit_command(msg.0.clone(), None, msg.1) {
364 self.pending_commands
365 .insert(call_id, (PendingRequest::GetTargets(tx), msg.0, now));
366 }
367 }
368 }
369
370 fn submit_navigation(&mut self, id: NavigationId, req: CdpRequest, now: Instant) {
373 if let Some(conn) = self.conn.as_mut() {
374 if let Ok(call_id) = conn.submit_command(
375 req.method.clone(),
376 req.session_id.map(Into::into),
377 req.params,
378 ) {
379 self.pending_commands
380 .insert(call_id, (PendingRequest::Navigate(id), req.method, now));
381 }
382 }
383 }
384
385 fn submit_close(&mut self, tx: OneshotSender<Result<CloseReturns>>, now: Instant) {
386 let close_msg = CLOSE_PARAMS_ID.clone();
387
388 if let Some(conn) = self.conn.as_mut() {
389 if let Ok(call_id) = conn.submit_command(close_msg.0.clone(), None, close_msg.1) {
390 self.pending_commands.insert(
391 call_id,
392 (PendingRequest::CloseBrowser(tx), close_msg.0, now),
393 );
394 }
395 }
396 }
397
398 fn on_target_message(&mut self, target: &mut Target, msg: CommandMessage, now: Instant) {
400 if msg.is_navigation() {
401 let (req, tx) = msg.split();
402 let id = self.next_navigation_id();
403
404 target.goto(FrameRequestedNavigation::new(
405 id,
406 req,
407 self.config.request_timeout,
408 ));
409
410 self.navigations.insert(
411 id,
412 NavigationRequest::Navigate(NavigationInProgress::new(tx)),
413 );
414 } else {
415 let _ = self.submit_external_command(msg, now);
416 }
417 }
418
419 fn next_navigation_id(&mut self) -> NavigationId {
421 let id = NavigationId(self.next_navigation_id);
422 self.next_navigation_id = self.next_navigation_id.wrapping_add(1);
423 id
424 }
425
426 fn create_page(&mut self, params: CreateTargetParams, tx: OneshotSender<Result<Page>>) {
437 let about_blank = params.url == "about:blank";
438 let http_check =
439 !about_blank && params.url.starts_with("http") || params.url.starts_with("file://");
440
441 if about_blank || http_check {
442 let method = params.identifier();
443
444 let Some(conn) = self.conn.as_mut() else {
445 let _ = tx.send(Err(CdpError::msg("connection consumed"))).ok();
446 return;
447 };
448 match serde_json::to_value(params) {
449 Ok(params) => match conn.submit_command(method.clone(), None, params) {
450 Ok(call_id) => {
451 self.pending_commands.insert(
452 call_id,
453 (PendingRequest::CreateTarget(tx), method, Instant::now()),
454 );
455 }
456 Err(err) => {
457 let _ = tx.send(Err(err.into())).ok();
458 }
459 },
460 Err(err) => {
461 let _ = tx.send(Err(err.into())).ok();
462 }
463 }
464 } else {
465 let _ = tx.send(Err(CdpError::NotFound)).ok();
466 }
467 }
468
469 fn on_event(&mut self, event: CdpEventMessage) {
471 if let Some(session_id) = &event.session_id {
472 if let Some(session) = self.sessions.get(session_id.as_str()) {
473 if let Some(target) = self.targets.get_mut(session.target_id()) {
474 return target.on_event(event);
475 }
476 }
477 }
478 let CdpEventMessage { params, method, .. } = event;
479
480 match params {
481 CdpEvent::TargetTargetCreated(ref ev) => self.on_target_created((**ev).clone()),
482 CdpEvent::TargetAttachedToTarget(ref ev) => self.on_attached_to_target(ev.clone()),
483 CdpEvent::TargetTargetDestroyed(ref ev) => self.on_target_destroyed(ev.clone()),
484 CdpEvent::TargetTargetCrashed(ref ev) => self.on_target_crashed(ev.clone()),
485 CdpEvent::TargetDetachedFromTarget(ref ev) => self.on_detached_from_target(ev.clone()),
486 _ => {}
487 }
488
489 chromiumoxide_cdp::consume_event!(match params {
490 |ev| self.event_listeners.start_send(ev),
491 |json| { let _ = self.event_listeners.try_send_custom(&method, json);}
492 });
493 }
494
495 fn on_target_created(&mut self, event: EventTargetCreated) {
499 if !self.browser_contexts.is_empty() {
500 if let Some(ref context_id) = event.target_info.browser_context_id {
501 let bc = BrowserContext {
502 id: Some(context_id.clone()),
503 };
504 if !self.browser_contexts.contains(&bc) {
505 return;
506 }
507 }
508 }
509 let browser_ctx = event
510 .target_info
511 .browser_context_id
512 .clone()
513 .map(BrowserContext::from)
514 .unwrap_or_else(|| self.default_browser_context.clone());
515 let target = Target::new(
516 event.target_info,
517 TargetConfig {
518 ignore_https_errors: self.config.ignore_https_errors,
519 request_timeout: self.config.request_timeout,
520 viewport: self.config.viewport.clone(),
521 request_intercept: self.config.request_intercept,
522 cache_enabled: self.config.cache_enabled,
523 service_worker_enabled: self.config.service_worker_enabled,
524 ignore_visuals: self.config.ignore_visuals,
525 ignore_stylesheets: self.config.ignore_stylesheets,
526 ignore_javascript: self.config.ignore_javascript,
527 ignore_analytics: self.config.ignore_analytics,
528 ignore_prefetch: self.config.ignore_prefetch,
529 extra_headers: self.config.extra_headers.clone(),
530 only_html: self.config.only_html && self.config.created_first_target,
531 intercept_manager: self.config.intercept_manager,
532 max_bytes_allowed: self.config.max_bytes_allowed,
533 max_redirects: self.config.max_redirects,
534 max_main_frame_navigations: self.config.max_main_frame_navigations,
535 whitelist_patterns: self.config.whitelist_patterns.clone(),
536 blacklist_patterns: self.config.blacklist_patterns.clone(),
537 #[cfg(feature = "adblock")]
538 adblock_filter_rules: self.config.adblock_filter_rules.clone(),
539 page_wake: self.page_wake.clone(),
540 },
541 browser_ctx,
542 );
543
544 let tid = target.target_id().clone();
545 self.target_ids.push(tid.clone());
546 self.targets.insert(tid, target);
547 }
548
549 fn on_attached_to_target(&mut self, event: Box<EventAttachedToTarget>) {
551 let session = Session::new(event.session_id.clone(), event.target_info.target_id);
552 if let Some(target) = self.targets.get_mut(session.target_id()) {
553 target.set_session_id(session.session_id().clone())
554 }
555 self.sessions.insert(event.session_id, session);
556 }
557
558 fn on_detached_from_target(&mut self, event: EventDetachedFromTarget) {
562 if let Some(session) = self.sessions.remove(&event.session_id) {
564 if let Some(target) = self.targets.get_mut(session.target_id()) {
565 target.session_id_mut().take();
566 }
567 }
568 }
569
570 fn on_target_destroyed(&mut self, event: EventTargetDestroyed) {
572 self.attached_targets.remove(&event.target_id);
573
574 if let Some(target) = self.targets.remove(&event.target_id) {
575 if let Some(session) = target.session_id() {
577 self.sessions.remove(session);
578 }
579 }
580 }
581
582 fn on_target_crashed(&mut self, event: EventTargetCrashed) {
605 let crashed_id = event.target_id.clone();
606 let status = event.status.clone();
607 let error_code = event.error_code;
608
609 let to_cancel: Vec<CallId> = self
615 .pending_commands
616 .iter()
617 .filter_map(|(&call_id, (req, _, _))| match req {
618 PendingRequest::ExternalCommand {
619 target_id: Some(tid),
620 ..
621 } if *tid == crashed_id => Some(call_id),
622 PendingRequest::InternalCommand(tid) if *tid == crashed_id => Some(call_id),
623 _ => None,
624 })
625 .collect();
626
627 for call_id in to_cancel {
628 if let Some((req, _, _)) = self.pending_commands.remove(&call_id) {
629 match req {
630 PendingRequest::ExternalCommand { tx, .. } => {
631 let _ = tx.send(Err(CdpError::msg(format!(
632 "target {:?} crashed: {} (errorCode={})",
633 crashed_id, status, error_code
634 ))));
635 }
636 PendingRequest::InternalCommand(_) => {
637 }
640 _ => {}
641 }
642 }
643 }
644
645 self.attached_targets.remove(&crashed_id);
647 if let Some(target) = self.targets.remove(&crashed_id) {
648 if let Some(session) = target.session_id() {
649 self.sessions.remove(session);
650 }
651 }
652 }
653
654 fn evict_timed_out_commands(&mut self, now: Instant) {
659 let deadline = match now.checked_sub(self.config.request_timeout) {
660 Some(d) => d,
661 None => return,
662 };
663
664 let timed_out: Vec<_> = self
665 .pending_commands
666 .iter()
667 .filter(|(_, (_, _, timestamp))| *timestamp < deadline)
668 .map(|(k, _)| *k)
669 .collect();
670
671 for call in timed_out {
672 if let Some((req, _, _)) = self.pending_commands.remove(&call) {
673 match req {
674 PendingRequest::CreateTarget(tx) => {
675 let _ = tx.send(Err(CdpError::Timeout));
676 }
677 PendingRequest::GetTargets(tx) => {
678 let _ = tx.send(Err(CdpError::Timeout));
679 }
680 PendingRequest::Navigate(nav) => {
681 if let Some(nav) = self.navigations.remove(&nav) {
682 match nav {
683 NavigationRequest::Navigate(nav) => {
684 let _ = nav.tx.send(Err(CdpError::Timeout));
685 }
686 }
687 }
688 }
689 PendingRequest::ExternalCommand { tx, .. } => {
690 let _ = tx.send(Err(CdpError::Timeout));
691 }
692 PendingRequest::InternalCommand(_) => {}
693 PendingRequest::CloseBrowser(tx) => {
694 let _ = tx.send(Err(CdpError::Timeout));
695 }
696 }
697 }
698 }
699 }
700
701 pub fn event_listeners_mut(&mut self) -> &mut EventListeners {
702 &mut self.event_listeners
703 }
704
705 pub async fn run(mut self) -> Result<()> {
735 use chromiumoxide_types::Message;
736 use tokio::time::MissedTickBehavior;
737 use tokio_tungstenite::tungstenite::{self, error::ProtocolError};
738
739 let page_wake = Arc::new(Notify::new());
741 self.page_wake = Some(page_wake.clone());
742
743 let conn = self
745 .conn
746 .take()
747 .ok_or_else(|| CdpError::msg("Handler::run() called with no connection"))?;
748 let async_conn = conn.into_async();
749 let mut ws_reader = async_conn.reader;
750 let ws_tx = async_conn.cmd_tx;
751 let mut writer_handle = async_conn.writer_handle;
752 let mut next_call_id = async_conn.next_id;
753
754 let mut alloc_call_id = || {
756 let id = chromiumoxide_types::CallId::new(next_call_id);
757 next_call_id = next_call_id.wrapping_add(1);
758 id
759 };
760
761 let mut evict_timer = tokio::time::interval_at(
763 tokio::time::Instant::now() + self.config.request_timeout,
764 self.config.request_timeout,
765 );
766 evict_timer.set_missed_tick_behavior(MissedTickBehavior::Delay);
767
768 macro_rules! ws_submit {
770 ($method:expr, $session_id:expr, $params:expr) => {{
771 let id = alloc_call_id();
772 let call = chromiumoxide_types::MethodCall {
773 id,
774 method: $method,
775 session_id: $session_id,
776 params: $params,
777 };
778 match ws_tx.try_send(call) {
779 Ok(()) => Ok::<_, CdpError>(id),
780 Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => {
781 tracing::warn!("WS command channel full — dropping command");
782 Err(CdpError::msg("WS command channel full"))
783 }
784 Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {
785 Err(CdpError::msg("WS writer closed"))
786 }
787 }
788 }};
789 }
790
791 loop {
793 let now = std::time::Instant::now();
794
795 const PER_TARGET_DRAIN_BUDGET: usize = 128;
801
802 for n in (0..self.target_ids.len()).rev() {
803 let target_id = self.target_ids.swap_remove(n);
804
805 if let Some((id, mut target)) = self.targets.remove_entry(&target_id) {
806 {
808 let mut msgs = Vec::new();
809 if let Some(handle) = target.page_mut() {
810 while msgs.len() < PER_TARGET_DRAIN_BUDGET {
811 match handle.rx.try_recv() {
812 Ok(msg) => msgs.push(msg),
813 Err(_) => break,
814 }
815 }
816 }
817 for msg in msgs {
818 target.on_page_message(msg);
819 }
820 }
821
822 while let Some(event) = target.advance(now) {
824 match event {
825 TargetEvent::Request(req) => {
826 if let Ok(call_id) =
827 ws_submit!(req.method.clone(), req.session_id, req.params)
828 {
829 self.pending_commands.insert(
830 call_id,
831 (
832 PendingRequest::InternalCommand(
833 target.target_id().clone(),
834 ),
835 req.method,
836 now,
837 ),
838 );
839 }
840 }
841 TargetEvent::Command(msg) => {
842 if msg.is_navigation() {
843 let (req, tx) = msg.split();
844 let nav_id = self.next_navigation_id();
845 target.goto(FrameRequestedNavigation::new(
846 nav_id,
847 req.clone(),
848 self.config.request_timeout,
849 ));
850 if let Ok(call_id) =
851 ws_submit!(req.method.clone(), req.session_id, req.params)
852 {
853 self.pending_commands.insert(
854 call_id,
855 (PendingRequest::Navigate(nav_id), req.method, now),
856 );
857 }
858 self.navigations.insert(
859 nav_id,
860 NavigationRequest::Navigate(NavigationInProgress::new(tx)),
861 );
862 } else if let Ok(call_id) = ws_submit!(
863 msg.method.clone(),
864 msg.session_id.map(Into::into),
865 msg.params
866 ) {
867 let target_id = Some(target.target_id().clone());
871 self.pending_commands.insert(
872 call_id,
873 (
874 PendingRequest::ExternalCommand {
875 tx: msg.sender,
876 target_id,
877 },
878 msg.method,
879 now,
880 ),
881 );
882 }
883 }
884 TargetEvent::NavigationRequest(nav_id, req) => {
885 if let Ok(call_id) =
886 ws_submit!(req.method.clone(), req.session_id, req.params)
887 {
888 self.pending_commands.insert(
889 call_id,
890 (PendingRequest::Navigate(nav_id), req.method, now),
891 );
892 }
893 }
894 TargetEvent::NavigationResult(res) => {
895 self.on_navigation_lifecycle_completed(res);
896 }
897 TargetEvent::BytesConsumed(n) => {
898 if let Some(rem) = self.remaining_bytes.as_mut() {
899 *rem = rem.saturating_sub(n);
900 if *rem == 0 {
901 self.budget_exhausted = true;
902 }
903 }
904 }
905 }
906 }
907
908 target.event_listeners_mut().flush();
910
911 self.targets.insert(id, target);
912 self.target_ids.push(target_id);
913 }
914 }
915
916 self.event_listeners.flush();
918
919 if self.budget_exhausted {
920 for t in self.targets.values_mut() {
921 t.network_manager.set_block_all(true);
922 }
923 }
924
925 if self.closing {
926 break;
927 }
928
929 tokio::select! {
931 msg = self.from_browser.recv() => {
932 match msg {
933 Some(msg) => {
934 match msg {
935 HandlerMessage::Command(cmd) => {
936 let target_id = cmd
939 .session_id
940 .as_ref()
941 .and_then(|sid| self.sessions.get(sid.as_ref()))
942 .map(|s| s.target_id().clone());
943 if let Ok(call_id) = ws_submit!(
944 cmd.method.clone(),
945 cmd.session_id.map(Into::into),
946 cmd.params
947 ) {
948 self.pending_commands.insert(
949 call_id,
950 (
951 PendingRequest::ExternalCommand {
952 tx: cmd.sender,
953 target_id,
954 },
955 cmd.method,
956 now,
957 ),
958 );
959 }
960 }
961 HandlerMessage::FetchTargets(tx) => {
962 let msg = TARGET_PARAMS_ID.clone();
963 if let Ok(call_id) = ws_submit!(msg.0.clone(), None, msg.1) {
964 self.pending_commands.insert(
965 call_id,
966 (PendingRequest::GetTargets(tx), msg.0, now),
967 );
968 }
969 }
970 HandlerMessage::CloseBrowser(tx) => {
971 let close_msg = CLOSE_PARAMS_ID.clone();
972 if let Ok(call_id) = ws_submit!(close_msg.0.clone(), None, close_msg.1) {
973 self.pending_commands.insert(
974 call_id,
975 (PendingRequest::CloseBrowser(tx), close_msg.0, now),
976 );
977 }
978 }
979 HandlerMessage::CreatePage(params, tx) => {
980 if let Some(ref id) = params.browser_context_id {
981 self.browser_contexts.insert(BrowserContext::from(id.clone()));
982 }
983 self.create_page_async(params, tx, &mut alloc_call_id, &ws_tx, now);
984 }
985 HandlerMessage::GetPages(tx) => {
986 let pages: Vec<_> = self.targets.values_mut()
987 .filter(|p| p.is_page())
988 .filter_map(|target| target.get_or_create_page())
989 .map(|page| Page::from(page.clone()))
990 .collect();
991 let _ = tx.send(pages);
992 }
993 HandlerMessage::InsertContext(ctx) => {
994 if self.default_browser_context.id().is_none() {
995 self.default_browser_context = ctx.clone();
996 }
997 self.browser_contexts.insert(ctx);
998 }
999 HandlerMessage::DisposeContext(ctx) => {
1000 self.browser_contexts.remove(&ctx);
1001 self.attached_targets.retain(|tid| {
1002 self.targets.get(tid)
1003 .and_then(|t| t.browser_context_id())
1004 .map(|id| Some(id) != ctx.id())
1005 .unwrap_or(true)
1006 });
1007 self.closing = true;
1008 }
1009 HandlerMessage::GetPage(target_id, tx) => {
1010 let page = self.targets.get_mut(&target_id)
1011 .and_then(|target| target.get_or_create_page())
1012 .map(|page| Page::from(page.clone()));
1013 let _ = tx.send(page);
1014 }
1015 HandlerMessage::AddEventListener(req) => {
1016 self.event_listeners.add_listener(req);
1017 }
1018 }
1019 }
1020 None => break, }
1022 }
1023
1024 frame = ws_reader.next_message() => {
1025 match frame {
1026 Some(Ok(boxed_msg)) => match *boxed_msg {
1027 Message::Response(resp) => {
1028 self.on_response(resp);
1029 }
1030 Message::Event(ev) => {
1031 self.on_event(ev);
1032 }
1033 },
1034 Some(Err(err)) => {
1035 tracing::error!("WS Connection error: {:?}", err);
1036 if let CdpError::Ws(ref ws_error) = err {
1037 match ws_error {
1038 tungstenite::Error::AlreadyClosed => break,
1039 tungstenite::Error::Protocol(detail)
1040 if detail == &ProtocolError::ResetWithoutClosingHandshake =>
1041 {
1042 break;
1043 }
1044 _ => return Err(err),
1045 }
1046 } else {
1047 return Err(err);
1048 }
1049 }
1050 None => break, }
1052 }
1053
1054 _ = page_wake.notified() => {
1055 }
1057
1058 _ = evict_timer.tick() => {
1059 self.evict_timed_out_commands(now);
1060 for t in self.targets.values_mut() {
1061 t.network_manager.evict_stale_entries(now);
1062 t.frame_manager_mut().evict_stale_context_ids();
1063 }
1064 }
1065
1066 result = &mut writer_handle => {
1067 match result {
1069 Ok(Ok(())) => break,
1070 Ok(Err(e)) => return Err(e),
1071 Err(e) => return Err(CdpError::msg(format!("WS writer panicked: {e}"))),
1072 }
1073 }
1074 }
1075 }
1076
1077 writer_handle.abort();
1078 Ok(())
1079 }
1080
1081 fn create_page_async(
1083 &mut self,
1084 params: CreateTargetParams,
1085 tx: OneshotSender<Result<Page>>,
1086 alloc_call_id: &mut impl FnMut() -> chromiumoxide_types::CallId,
1087 ws_tx: &tokio::sync::mpsc::Sender<chromiumoxide_types::MethodCall>,
1088 now: std::time::Instant,
1089 ) {
1090 let about_blank = params.url == "about:blank";
1091 let http_check =
1092 !about_blank && params.url.starts_with("http") || params.url.starts_with("file://");
1093
1094 if about_blank || http_check {
1095 let method = params.identifier();
1096 match serde_json::to_value(params) {
1097 Ok(params) => {
1098 let id = alloc_call_id();
1099 let call = chromiumoxide_types::MethodCall {
1100 id,
1101 method: method.clone(),
1102 session_id: None,
1103 params,
1104 };
1105 match ws_tx.try_send(call) {
1106 Ok(()) => {
1107 self.pending_commands
1108 .insert(id, (PendingRequest::CreateTarget(tx), method, now));
1109 }
1110 Err(_) => {
1111 let _ = tx
1112 .send(Err(CdpError::msg("WS command channel full or closed")))
1113 .ok();
1114 }
1115 }
1116 }
1117 Err(err) => {
1118 let _ = tx.send(Err(err.into())).ok();
1119 }
1120 }
1121 } else {
1122 let _ = tx.send(Err(CdpError::NotFound)).ok();
1123 }
1124 }
1125}
1126
1127impl Stream for Handler {
1128 type Item = Result<()>;
1129
1130 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
1131 let pin = self.get_mut();
1132
1133 let mut dispose = false;
1134
1135 let now = Instant::now();
1136
1137 loop {
1138 while let Poll::Ready(Some(msg)) = pin.from_browser.poll_recv(cx) {
1142 match msg {
1143 HandlerMessage::Command(cmd) => {
1144 pin.submit_external_command(cmd, now)?;
1145 }
1146 HandlerMessage::FetchTargets(tx) => {
1147 pin.submit_fetch_targets(tx, now);
1148 }
1149 HandlerMessage::CloseBrowser(tx) => {
1150 pin.submit_close(tx, now);
1151 }
1152 HandlerMessage::CreatePage(params, tx) => {
1153 if let Some(ref id) = params.browser_context_id {
1154 pin.browser_contexts
1155 .insert(BrowserContext::from(id.clone()));
1156 }
1157 pin.create_page(params, tx);
1158 }
1159 HandlerMessage::GetPages(tx) => {
1160 let pages: Vec<_> = pin
1161 .targets
1162 .values_mut()
1163 .filter(|p: &&mut Target| p.is_page())
1164 .filter_map(|target| target.get_or_create_page())
1165 .map(|page| Page::from(page.clone()))
1166 .collect();
1167 let _ = tx.send(pages);
1168 }
1169 HandlerMessage::InsertContext(ctx) => {
1170 if pin.default_browser_context.id().is_none() {
1171 pin.default_browser_context = ctx.clone();
1172 }
1173 pin.browser_contexts.insert(ctx);
1174 }
1175 HandlerMessage::DisposeContext(ctx) => {
1176 pin.browser_contexts.remove(&ctx);
1177 pin.attached_targets.retain(|tid| {
1178 pin.targets
1179 .get(tid)
1180 .and_then(|t| t.browser_context_id()) .map(|id| Some(id) != ctx.id())
1182 .unwrap_or(true)
1183 });
1184 pin.closing = true;
1185 dispose = true;
1186 }
1187 HandlerMessage::GetPage(target_id, tx) => {
1188 let page = pin
1189 .targets
1190 .get_mut(&target_id)
1191 .and_then(|target| target.get_or_create_page())
1192 .map(|page| Page::from(page.clone()));
1193 let _ = tx.send(page);
1194 }
1195 HandlerMessage::AddEventListener(req) => {
1196 pin.event_listeners.add_listener(req);
1197 }
1198 }
1199 }
1200
1201 for n in (0..pin.target_ids.len()).rev() {
1202 let target_id = pin.target_ids.swap_remove(n);
1203
1204 if let Some((id, mut target)) = pin.targets.remove_entry(&target_id) {
1205 while let Some(event) = target.poll(cx, now) {
1206 match event {
1207 TargetEvent::Request(req) => {
1208 let _ = pin.submit_internal_command(
1209 target.target_id().clone(),
1210 req,
1211 now,
1212 );
1213 }
1214 TargetEvent::Command(msg) => {
1215 pin.on_target_message(&mut target, msg, now);
1216 }
1217 TargetEvent::NavigationRequest(id, req) => {
1218 pin.submit_navigation(id, req, now);
1219 }
1220 TargetEvent::NavigationResult(res) => {
1221 pin.on_navigation_lifecycle_completed(res)
1222 }
1223 TargetEvent::BytesConsumed(n) => {
1224 if let Some(rem) = pin.remaining_bytes.as_mut() {
1225 *rem = rem.saturating_sub(n);
1226 if *rem == 0 {
1227 pin.budget_exhausted = true;
1228 }
1229 }
1230 }
1231 }
1232 }
1233
1234 target.event_listeners_mut().poll(cx);
1236
1237 pin.targets.insert(id, target);
1238 pin.target_ids.push(target_id);
1239 }
1240 }
1241
1242 pin.event_listeners_mut().poll(cx);
1245
1246 let mut done = true;
1247
1248 let mut ws_msgs = Vec::new();
1251 let mut ws_err = None;
1252 {
1253 let Some(conn) = pin.conn.as_mut() else {
1254 return Poll::Ready(Some(Err(CdpError::msg(
1255 "connection consumed by Handler::run()",
1256 ))));
1257 };
1258 while let Poll::Ready(Some(ev)) = Pin::new(&mut *conn).poll_next(cx) {
1259 match ev {
1260 Ok(msg) => ws_msgs.push(msg),
1261 Err(err) => {
1262 ws_err = Some(err);
1263 break;
1264 }
1265 }
1266 }
1267 }
1268
1269 for boxed_msg in ws_msgs {
1270 match *boxed_msg {
1271 Message::Response(resp) => {
1272 pin.on_response(resp);
1273 if pin.closing {
1274 return Poll::Ready(None);
1275 }
1276 }
1277 Message::Event(ev) => {
1278 pin.on_event(ev);
1279 }
1280 }
1281 done = false;
1282 }
1283
1284 if let Some(err) = ws_err {
1285 tracing::error!("WS Connection error: {:?}", err);
1286 if let CdpError::Ws(ref ws_error) = err {
1287 match ws_error {
1288 Error::AlreadyClosed => {
1289 pin.closing = true;
1290 dispose = true;
1291 }
1292 Error::Protocol(detail)
1293 if detail == &ProtocolError::ResetWithoutClosingHandshake =>
1294 {
1295 pin.closing = true;
1296 dispose = true;
1297 }
1298 _ => return Poll::Ready(Some(Err(err))),
1299 }
1300 } else {
1301 return Poll::Ready(Some(Err(err)));
1302 }
1303 }
1304
1305 if pin.evict_command_timeout.poll_ready(cx) {
1306 pin.evict_timed_out_commands(now);
1308 for t in pin.targets.values_mut() {
1311 t.network_manager.evict_stale_entries(now);
1312 t.frame_manager_mut().evict_stale_context_ids();
1313 }
1314 }
1315
1316 if pin.budget_exhausted {
1317 for t in pin.targets.values_mut() {
1318 t.network_manager.set_block_all(true);
1319 }
1320 }
1321
1322 if dispose {
1323 return Poll::Ready(None);
1324 }
1325
1326 if done {
1327 return Poll::Pending;
1329 }
1330 }
1331 }
1332}
1333
1334#[derive(Debug, Clone)]
1336pub struct HandlerConfig {
1337 pub ignore_https_errors: bool,
1339 pub viewport: Option<Viewport>,
1341 pub context_ids: Vec<BrowserContextId>,
1343 pub request_timeout: Duration,
1345 pub request_intercept: bool,
1347 pub cache_enabled: bool,
1349 pub service_worker_enabled: bool,
1351 pub ignore_visuals: bool,
1353 pub ignore_stylesheets: bool,
1355 pub ignore_javascript: bool,
1357 pub ignore_analytics: bool,
1359 pub ignore_prefetch: bool,
1361 pub ignore_ads: bool,
1363 pub extra_headers: Option<std::collections::HashMap<String, String>>,
1365 pub only_html: bool,
1367 pub created_first_target: bool,
1369 pub intercept_manager: NetworkInterceptManager,
1371 pub max_bytes_allowed: Option<u64>,
1373 pub max_redirects: Option<usize>,
1380 pub max_main_frame_navigations: Option<u32>,
1384 pub whitelist_patterns: Option<Vec<String>>,
1386 pub blacklist_patterns: Option<Vec<String>>,
1388 #[cfg(feature = "adblock")]
1390 pub adblock_filter_rules: Option<Vec<String>>,
1391 pub channel_capacity: usize,
1394 pub connection_retries: u32,
1397}
1398
1399impl Default for HandlerConfig {
1400 fn default() -> Self {
1401 Self {
1402 ignore_https_errors: true,
1403 viewport: Default::default(),
1404 context_ids: Vec::new(),
1405 request_timeout: Duration::from_millis(REQUEST_TIMEOUT),
1406 request_intercept: false,
1407 cache_enabled: true,
1408 service_worker_enabled: true,
1409 ignore_visuals: false,
1410 ignore_stylesheets: false,
1411 ignore_ads: false,
1412 ignore_javascript: false,
1413 ignore_analytics: true,
1414 ignore_prefetch: true,
1415 only_html: false,
1416 extra_headers: Default::default(),
1417 created_first_target: false,
1418 intercept_manager: NetworkInterceptManager::Unknown,
1419 max_bytes_allowed: None,
1420 max_redirects: None,
1421 max_main_frame_navigations: None,
1422 whitelist_patterns: None,
1423 blacklist_patterns: None,
1424 #[cfg(feature = "adblock")]
1425 adblock_filter_rules: None,
1426 channel_capacity: 4096,
1427 connection_retries: crate::conn::DEFAULT_CONNECTION_RETRIES,
1428 }
1429 }
1430}
1431
1432#[derive(Debug)]
1434pub struct NavigationInProgress<T> {
1435 navigated: bool,
1437 response: Option<Response>,
1439 tx: OneshotSender<T>,
1441}
1442
1443impl<T> NavigationInProgress<T> {
1444 fn new(tx: OneshotSender<T>) -> Self {
1445 Self {
1446 navigated: false,
1447 response: None,
1448 tx,
1449 }
1450 }
1451
1452 fn set_response(&mut self, resp: Response) {
1454 self.response = Some(resp);
1455 }
1456
1457 fn set_navigated(&mut self) {
1459 self.navigated = true;
1460 }
1461}
1462
1463#[derive(Debug)]
1465enum NavigationRequest {
1466 Navigate(NavigationInProgress<Result<Response>>),
1468 }
1470
1471#[derive(Debug)]
1474enum PendingRequest {
1475 CreateTarget(OneshotSender<Result<Page>>),
1478 GetTargets(OneshotSender<Result<Vec<TargetInfo>>>),
1480 Navigate(NavigationId),
1487 ExternalCommand {
1496 tx: OneshotSender<Result<Response>>,
1497 target_id: Option<TargetId>,
1498 },
1499 InternalCommand(TargetId),
1502 CloseBrowser(OneshotSender<Result<CloseReturns>>),
1504}
1505
1506#[derive(Debug)]
1510pub(crate) enum HandlerMessage {
1511 CreatePage(CreateTargetParams, OneshotSender<Result<Page>>),
1512 FetchTargets(OneshotSender<Result<Vec<TargetInfo>>>),
1513 InsertContext(BrowserContext),
1514 DisposeContext(BrowserContext),
1515 GetPages(OneshotSender<Vec<Page>>),
1516 Command(CommandMessage),
1517 GetPage(TargetId, OneshotSender<Option<Page>>),
1518 AddEventListener(EventListenerRequest),
1519 CloseBrowser(OneshotSender<Result<CloseReturns>>),
1520}
1521
1522#[cfg(test)]
1523mod tests {
1524 use super::*;
1525 use chromiumoxide_cdp::cdp::browser_protocol::target::{AttachToTargetReturns, TargetInfo};
1526
1527 #[test]
1528 fn attach_to_target_response_sets_session_id_before_event_arrives() {
1529 let info = TargetInfo::builder()
1530 .target_id("target-1".to_string())
1531 .r#type("page")
1532 .title("")
1533 .url("about:blank")
1534 .attached(false)
1535 .can_access_opener(false)
1536 .build()
1537 .expect("target info");
1538 let mut target = Target::new(info, TargetConfig::default(), BrowserContext::default());
1539 let method: MethodId = AttachToTargetParams::IDENTIFIER.into();
1540 let result = serde_json::to_value(AttachToTargetReturns::new("session-1".to_string()))
1541 .expect("attach result");
1542 let resp = Response {
1543 id: CallId::new(1),
1544 result: Some(result),
1545 error: None,
1546 };
1547
1548 maybe_store_attach_session_id(&mut target, &method, &resp);
1549
1550 assert_eq!(
1551 target.session_id().map(AsRef::as_ref),
1552 Some("session-1"),
1553 "attach response should seed the flat session id even before Target.attachedToTarget"
1554 );
1555 }
1556}