1use crate::listeners::{EventListenerRequest, EventListeners};
2use chromiumoxide_cdp::cdp::browser_protocol::browser::*;
3use chromiumoxide_cdp::cdp::browser_protocol::target::*;
4use chromiumoxide_cdp::cdp::events::CdpEvent;
5use chromiumoxide_cdp::cdp::events::CdpEventMessage;
6use chromiumoxide_types::{CallId, Message, Method, Response};
7use chromiumoxide_types::{MethodId, Request as CdpRequest};
8use fnv::FnvHashMap;
9use futures_util::Stream;
10use hashbrown::{HashMap, HashSet};
11use spider_network_blocker::intercept_manager::NetworkInterceptManager;
12use std::pin::Pin;
13use std::task::{Context, Poll};
14use std::time::{Duration, Instant};
15use tokio::sync::mpsc::Receiver;
16use tokio::sync::oneshot::Sender as OneshotSender;
17use tokio_tungstenite::tungstenite::error::ProtocolError;
18use tokio_tungstenite::tungstenite::Error;
19
20use std::sync::Arc;
21use tokio::sync::Notify;
22
23use crate::cmd::{to_command_response, CommandMessage};
24use crate::conn::Connection;
25use crate::error::{CdpError, Result};
26use crate::handler::browser::BrowserContext;
27use crate::handler::frame::FrameRequestedNavigation;
28use crate::handler::frame::{NavigationError, NavigationId, NavigationOk};
29use crate::handler::job::PeriodicJob;
30use crate::handler::session::Session;
31use crate::handler::target::TargetEvent;
32use crate::handler::target::{Target, TargetConfig};
33use crate::handler::viewport::Viewport;
34use crate::page::Page;
35pub(crate) use page::PageInner;
36
37pub const REQUEST_TIMEOUT: u64 = 30_000;
39
40pub mod blockers;
41pub mod browser;
42pub mod commandfuture;
43pub mod domworld;
44pub mod emulation;
45pub mod frame;
46pub mod http;
47pub mod httpfuture;
48mod job;
49pub mod network;
50pub mod network_utils;
51pub mod page;
52pub mod sender;
53mod session;
54pub mod target;
55pub mod target_message_future;
56pub mod viewport;
57
58#[must_use = "streams do nothing unless polled"]
61#[derive(Debug)]
62pub struct Handler {
63 pub default_browser_context: BrowserContext,
64 pub browser_contexts: HashSet<BrowserContext>,
65 pending_commands: FnvHashMap<CallId, (PendingRequest, MethodId, Instant)>,
69 from_browser: Receiver<HandlerMessage>,
71 target_ids: Vec<TargetId>,
73 targets: HashMap<TargetId, Target>,
75 navigations: FnvHashMap<NavigationId, NavigationRequest>,
77 sessions: HashMap<SessionId, Session>,
81 conn: Option<Connection<CdpEventMessage>>,
84 evict_command_timeout: PeriodicJob,
86 next_navigation_id: usize,
88 config: HandlerConfig,
90 event_listeners: EventListeners,
92 closing: bool,
94 remaining_bytes: Option<u64>,
96 budget_exhausted: bool,
98 attached_targets: HashSet<TargetId>,
100 page_wake: Option<Arc<Notify>>,
103}
104
105lazy_static::lazy_static! {
106 static ref DISCOVER_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
108 let discover = SetDiscoverTargetsParams::new(true);
109 (discover.identifier(), serde_json::to_value(discover).expect("valid discover target params"))
110 };
111 static ref TARGET_PARAMS_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
113 let msg = GetTargetsParams { filter: None };
114 (msg.identifier(), serde_json::to_value(msg).expect("valid paramtarget"))
115 };
116 static ref CLOSE_PARAMS_ID: (std::borrow::Cow<'static, str>, serde_json::Value) = {
118 let close_msg = CloseParams::default();
119 (close_msg.identifier(), serde_json::to_value(close_msg).expect("valid close params"))
120 };
121}
122
123fn maybe_store_attach_session_id(target: &mut Target, method: &MethodId, resp: &Response) {
124 if method.as_ref() != AttachToTargetParams::IDENTIFIER {
125 return;
126 }
127
128 if let Ok(resp) = to_command_response::<AttachToTargetParams>(resp.clone(), method.clone()) {
129 target.set_session_id(resp.result.session_id);
130 }
131}
132
133impl Handler {
134 pub(crate) fn new(
137 mut conn: Connection<CdpEventMessage>,
138 rx: Receiver<HandlerMessage>,
139 config: HandlerConfig,
140 ) -> Self {
141 let discover = DISCOVER_ID.clone();
142 let _ = conn.submit_command(discover.0, None, discover.1);
143 let conn = Some(conn);
144
145 let browser_contexts = config
146 .context_ids
147 .iter()
148 .map(|id| BrowserContext::from(id.clone()))
149 .collect();
150
151 Self {
152 pending_commands: Default::default(),
153 from_browser: rx,
154 default_browser_context: Default::default(),
155 browser_contexts,
156 target_ids: Default::default(),
157 targets: Default::default(),
158 navigations: Default::default(),
159 sessions: Default::default(),
160 conn,
161 evict_command_timeout: PeriodicJob::new(config.request_timeout),
162 next_navigation_id: 0,
163 config,
164 event_listeners: Default::default(),
165 closing: false,
166 remaining_bytes: None,
167 budget_exhausted: false,
168 attached_targets: Default::default(),
169 page_wake: None,
170 }
171 }
172
173 #[inline]
176 fn conn(&mut self) -> Result<&mut Connection<CdpEventMessage>> {
177 self.conn
178 .as_mut()
179 .ok_or_else(|| CdpError::msg("connection consumed by Handler::run()"))
180 }
181
182 pub fn get_target(&self, target_id: &TargetId) -> Option<&Target> {
184 self.targets.get(target_id)
185 }
186
187 pub fn targets(&self) -> impl Iterator<Item = &Target> + '_ {
189 self.targets.values()
190 }
191
192 pub fn default_browser_context(&self) -> &BrowserContext {
194 &self.default_browser_context
195 }
196
197 pub fn browser_contexts(&self) -> impl Iterator<Item = &BrowserContext> + '_ {
199 self.browser_contexts.iter()
200 }
201
202 fn on_navigation_response(&mut self, id: NavigationId, resp: Response) {
204 if let Some(nav) = self.navigations.remove(&id) {
205 match nav {
206 NavigationRequest::Navigate(mut nav) => {
207 if nav.navigated {
208 let _ = nav.tx.send(Ok(resp));
209 } else {
210 nav.set_response(resp);
211 self.navigations
212 .insert(id, NavigationRequest::Navigate(nav));
213 }
214 }
215 }
216 }
217 }
218
219 fn on_navigation_lifecycle_completed(&mut self, res: Result<NavigationOk, NavigationError>) {
221 match res {
222 Ok(ok) => {
223 let id = *ok.navigation_id();
224 if let Some(nav) = self.navigations.remove(&id) {
225 match nav {
226 NavigationRequest::Navigate(mut nav) => {
227 if let Some(resp) = nav.response.take() {
228 let _ = nav.tx.send(Ok(resp));
229 } else {
230 nav.set_navigated();
231 self.navigations
232 .insert(id, NavigationRequest::Navigate(nav));
233 }
234 }
235 }
236 }
237 }
238 Err(err) => {
239 if let Some(nav) = self.navigations.remove(err.navigation_id()) {
240 match nav {
241 NavigationRequest::Navigate(nav) => {
242 let _ = nav.tx.send(Err(err.into()));
243 }
244 }
245 }
246 }
247 }
248 }
249
250 fn on_response(&mut self, resp: Response) {
252 if let Some((req, method, _)) = self.pending_commands.remove(&resp.id) {
253 match req {
254 PendingRequest::CreateTarget(tx) => {
255 match to_command_response::<CreateTargetParams>(resp, method) {
256 Ok(resp) => {
257 if let Some(target) = self.targets.get_mut(&resp.target_id) {
258 target.set_initiator(tx);
259 } else {
260 let _ = tx.send(Err(CdpError::NotFound)).ok();
261 }
262 }
263 Err(err) => {
264 let _ = tx.send(Err(err)).ok();
265 }
266 }
267 }
268 PendingRequest::GetTargets(tx) => {
269 match to_command_response::<GetTargetsParams>(resp, method) {
270 Ok(resp) => {
271 let targets = resp.result.target_infos;
272 let results = targets.clone();
273
274 for target_info in targets {
275 let event: EventTargetCreated = EventTargetCreated { target_info };
276 self.on_target_created(event);
277 }
278
279 let _ = tx.send(Ok(results)).ok();
280 }
281 Err(err) => {
282 let _ = tx.send(Err(err)).ok();
283 }
284 }
285 }
286 PendingRequest::Navigate(id) => {
287 self.on_navigation_response(id, resp);
288 if self.config.only_html && !self.config.created_first_target {
289 self.config.created_first_target = true;
290 }
291 }
292 PendingRequest::ExternalCommand { tx, .. } => {
293 let _ = tx.send(Ok(resp)).ok();
294 }
295 PendingRequest::InternalCommand(target_id) => {
296 if let Some(target) = self.targets.get_mut(&target_id) {
297 maybe_store_attach_session_id(target, &method, &resp);
298 target.on_response(resp, method.as_ref());
299 }
300 }
301 PendingRequest::CloseBrowser(tx) => {
302 self.closing = true;
303 let _ = tx.send(Ok(CloseReturns {})).ok();
304 }
305 }
306 }
307 }
308
309 pub(crate) fn submit_external_command(
311 &mut self,
312 msg: CommandMessage,
313 now: Instant,
314 ) -> Result<()> {
315 let target_id = msg
320 .session_id
321 .as_ref()
322 .and_then(|sid| self.sessions.get(sid.as_ref()))
323 .map(|s| s.target_id().clone());
324 let call_id =
325 self.conn()?
326 .submit_command(msg.method.clone(), msg.session_id, msg.params)?;
327 self.pending_commands.insert(
328 call_id,
329 (
330 PendingRequest::ExternalCommand {
331 tx: msg.sender,
332 target_id,
333 },
334 msg.method,
335 now,
336 ),
337 );
338 Ok(())
339 }
340
341 pub(crate) fn submit_internal_command(
342 &mut self,
343 target_id: TargetId,
344 req: CdpRequest,
345 now: Instant,
346 ) -> Result<()> {
347 let call_id = self.conn()?.submit_command(
348 req.method.clone(),
349 req.session_id.map(Into::into),
350 req.params,
351 )?;
352 self.pending_commands.insert(
353 call_id,
354 (PendingRequest::InternalCommand(target_id), req.method, now),
355 );
356 Ok(())
357 }
358
359 fn submit_fetch_targets(&mut self, tx: OneshotSender<Result<Vec<TargetInfo>>>, now: Instant) {
360 let msg = TARGET_PARAMS_ID.clone();
361
362 if let Some(conn) = self.conn.as_mut() {
363 if let Ok(call_id) = conn.submit_command(msg.0.clone(), None, msg.1) {
364 self.pending_commands
365 .insert(call_id, (PendingRequest::GetTargets(tx), msg.0, now));
366 }
367 }
368 }
369
370 fn submit_navigation(&mut self, id: NavigationId, req: CdpRequest, now: Instant) {
373 if let Some(conn) = self.conn.as_mut() {
374 if let Ok(call_id) = conn.submit_command(
375 req.method.clone(),
376 req.session_id.map(Into::into),
377 req.params,
378 ) {
379 self.pending_commands
380 .insert(call_id, (PendingRequest::Navigate(id), req.method, now));
381 }
382 }
383 }
384
385 fn submit_close(&mut self, tx: OneshotSender<Result<CloseReturns>>, now: Instant) {
386 let close_msg = CLOSE_PARAMS_ID.clone();
387
388 if let Some(conn) = self.conn.as_mut() {
389 if let Ok(call_id) = conn.submit_command(close_msg.0.clone(), None, close_msg.1) {
390 self.pending_commands.insert(
391 call_id,
392 (PendingRequest::CloseBrowser(tx), close_msg.0, now),
393 );
394 }
395 }
396 }
397
398 fn on_target_message(&mut self, target: &mut Target, msg: CommandMessage, now: Instant) {
400 if msg.is_navigation() {
401 let (req, tx) = msg.split();
402 let id = self.next_navigation_id();
403
404 target.goto(FrameRequestedNavigation::new(
405 id,
406 req,
407 self.config.request_timeout,
408 ));
409
410 self.navigations.insert(
411 id,
412 NavigationRequest::Navigate(NavigationInProgress::new(tx)),
413 );
414 } else {
415 let _ = self.submit_external_command(msg, now);
416 }
417 }
418
419 fn next_navigation_id(&mut self) -> NavigationId {
421 let id = NavigationId(self.next_navigation_id);
422 self.next_navigation_id = self.next_navigation_id.wrapping_add(1);
423 id
424 }
425
426 fn create_page(&mut self, params: CreateTargetParams, tx: OneshotSender<Result<Page>>) {
437 let about_blank = params.url == "about:blank";
438 let http_check =
439 !about_blank && params.url.starts_with("http") || params.url.starts_with("file://");
440
441 if about_blank || http_check {
442 let method = params.identifier();
443
444 let Some(conn) = self.conn.as_mut() else {
445 let _ = tx.send(Err(CdpError::msg("connection consumed"))).ok();
446 return;
447 };
448 match serde_json::to_value(params) {
449 Ok(params) => match conn.submit_command(method.clone(), None, params) {
450 Ok(call_id) => {
451 self.pending_commands.insert(
452 call_id,
453 (PendingRequest::CreateTarget(tx), method, Instant::now()),
454 );
455 }
456 Err(err) => {
457 let _ = tx.send(Err(err.into())).ok();
458 }
459 },
460 Err(err) => {
461 let _ = tx.send(Err(err.into())).ok();
462 }
463 }
464 } else {
465 let _ = tx.send(Err(CdpError::NotFound)).ok();
466 }
467 }
468
469 fn on_event(&mut self, event: CdpEventMessage) {
471 if let Some(session_id) = &event.session_id {
472 if let Some(session) = self.sessions.get(session_id.as_str()) {
473 if let Some(target) = self.targets.get_mut(session.target_id()) {
474 return target.on_event(event);
475 }
476 }
477 }
478 let CdpEventMessage { params, method, .. } = event;
479
480 match params {
481 CdpEvent::TargetTargetCreated(ref ev) => self.on_target_created((**ev).clone()),
482 CdpEvent::TargetAttachedToTarget(ref ev) => self.on_attached_to_target(ev.clone()),
483 CdpEvent::TargetTargetDestroyed(ref ev) => self.on_target_destroyed(ev.clone()),
484 CdpEvent::TargetTargetCrashed(ref ev) => self.on_target_crashed(ev.clone()),
485 CdpEvent::TargetDetachedFromTarget(ref ev) => self.on_detached_from_target(ev.clone()),
486 _ => {}
487 }
488
489 chromiumoxide_cdp::consume_event!(match params {
490 |ev| self.event_listeners.start_send(ev),
491 |json| { let _ = self.event_listeners.try_send_custom(&method, json);}
492 });
493 }
494
495 fn on_target_created(&mut self, event: EventTargetCreated) {
499 if !self.browser_contexts.is_empty() {
500 if let Some(ref context_id) = event.target_info.browser_context_id {
501 let bc = BrowserContext {
502 id: Some(context_id.clone()),
503 };
504 if !self.browser_contexts.contains(&bc) {
505 return;
506 }
507 }
508 }
509 let browser_ctx = event
510 .target_info
511 .browser_context_id
512 .clone()
513 .map(BrowserContext::from)
514 .unwrap_or_else(|| self.default_browser_context.clone());
515 let target = Target::new(
516 event.target_info,
517 TargetConfig {
518 ignore_https_errors: self.config.ignore_https_errors,
519 request_timeout: self.config.request_timeout,
520 viewport: self.config.viewport.clone(),
521 request_intercept: self.config.request_intercept,
522 cache_enabled: self.config.cache_enabled,
523 service_worker_enabled: self.config.service_worker_enabled,
524 ignore_visuals: self.config.ignore_visuals,
525 ignore_stylesheets: self.config.ignore_stylesheets,
526 ignore_javascript: self.config.ignore_javascript,
527 ignore_analytics: self.config.ignore_analytics,
528 ignore_prefetch: self.config.ignore_prefetch,
529 extra_headers: self.config.extra_headers.clone(),
530 only_html: self.config.only_html && self.config.created_first_target,
531 intercept_manager: self.config.intercept_manager,
532 max_bytes_allowed: self.config.max_bytes_allowed,
533 max_redirects: self.config.max_redirects,
534 max_main_frame_navigations: self.config.max_main_frame_navigations,
535 whitelist_patterns: self.config.whitelist_patterns.clone(),
536 blacklist_patterns: self.config.blacklist_patterns.clone(),
537 #[cfg(feature = "adblock")]
538 adblock_filter_rules: self.config.adblock_filter_rules.clone(),
539 page_wake: self.page_wake.clone(),
540 },
541 browser_ctx,
542 );
543
544 let tid = target.target_id().clone();
545 self.target_ids.push(tid.clone());
546 self.targets.insert(tid, target);
547 }
548
549 fn on_attached_to_target(&mut self, event: Box<EventAttachedToTarget>) {
551 let session = Session::new(event.session_id.clone(), event.target_info.target_id);
552 if let Some(target) = self.targets.get_mut(session.target_id()) {
553 target.set_session_id(session.session_id().clone())
554 }
555 self.sessions.insert(event.session_id, session);
556 }
557
558 fn on_detached_from_target(&mut self, event: EventDetachedFromTarget) {
562 if let Some(session) = self.sessions.remove(&event.session_id) {
564 if let Some(target) = self.targets.get_mut(session.target_id()) {
565 target.session_id_mut().take();
566 }
567 }
568 }
569
570 fn on_target_destroyed(&mut self, event: EventTargetDestroyed) {
572 self.attached_targets.remove(&event.target_id);
573
574 if let Some(target) = self.targets.remove(&event.target_id) {
575 if let Some(session) = target.session_id() {
577 self.sessions.remove(session);
578 }
579 }
580 }
581
582 fn on_target_crashed(&mut self, event: EventTargetCrashed) {
605 let crashed_id = event.target_id.clone();
606 let status = event.status.clone();
607 let error_code = event.error_code;
608
609 let to_cancel: Vec<CallId> = self
615 .pending_commands
616 .iter()
617 .filter_map(|(&call_id, (req, _, _))| match req {
618 PendingRequest::ExternalCommand {
619 target_id: Some(tid),
620 ..
621 } if *tid == crashed_id => Some(call_id),
622 PendingRequest::InternalCommand(tid) if *tid == crashed_id => Some(call_id),
623 _ => None,
624 })
625 .collect();
626
627 for call_id in to_cancel {
628 if let Some((req, _, _)) = self.pending_commands.remove(&call_id) {
629 match req {
630 PendingRequest::ExternalCommand { tx, .. } => {
631 let _ = tx.send(Err(CdpError::msg(format!(
632 "target {:?} crashed: {} (errorCode={})",
633 crashed_id, status, error_code
634 ))));
635 }
636 PendingRequest::InternalCommand(_) => {
637 }
640 _ => {}
641 }
642 }
643 }
644
645 self.attached_targets.remove(&crashed_id);
647 if let Some(target) = self.targets.remove(&crashed_id) {
648 if let Some(session) = target.session_id() {
649 self.sessions.remove(session);
650 }
651 }
652 }
653
654 fn evict_timed_out_commands(&mut self, now: Instant) {
659 let deadline = match now.checked_sub(self.config.request_timeout) {
660 Some(d) => d,
661 None => return,
662 };
663
664 let timed_out: Vec<_> = self
665 .pending_commands
666 .iter()
667 .filter(|(_, (_, _, timestamp))| *timestamp < deadline)
668 .map(|(k, _)| *k)
669 .collect();
670
671 for call in timed_out {
672 if let Some((req, _, _)) = self.pending_commands.remove(&call) {
673 match req {
674 PendingRequest::CreateTarget(tx) => {
675 let _ = tx.send(Err(CdpError::Timeout));
676 }
677 PendingRequest::GetTargets(tx) => {
678 let _ = tx.send(Err(CdpError::Timeout));
679 }
680 PendingRequest::Navigate(nav) => {
681 if let Some(nav) = self.navigations.remove(&nav) {
682 match nav {
683 NavigationRequest::Navigate(nav) => {
684 let _ = nav.tx.send(Err(CdpError::Timeout));
685 }
686 }
687 }
688 }
689 PendingRequest::ExternalCommand { tx, .. } => {
690 let _ = tx.send(Err(CdpError::Timeout));
691 }
692 PendingRequest::InternalCommand(_) => {}
693 PendingRequest::CloseBrowser(tx) => {
694 let _ = tx.send(Err(CdpError::Timeout));
695 }
696 }
697 }
698 }
699 }
700
701 pub fn event_listeners_mut(&mut self) -> &mut EventListeners {
702 &mut self.event_listeners
703 }
704
705 pub async fn run(mut self) -> Result<()> {
735 use chromiumoxide_types::Message;
736 use tokio::time::MissedTickBehavior;
737 use tokio_tungstenite::tungstenite::{self, error::ProtocolError};
738
739 let page_wake = Arc::new(Notify::new());
741 self.page_wake = Some(page_wake.clone());
742
743 let conn = self
745 .conn
746 .take()
747 .ok_or_else(|| CdpError::msg("Handler::run() called with no connection"))?;
748 let async_conn = conn.into_async();
749 let mut ws_reader = async_conn.reader;
750 let ws_tx = async_conn.cmd_tx;
751 let mut writer_handle = async_conn.writer_handle;
752 let mut next_call_id = async_conn.next_id;
753
754 let mut alloc_call_id = || {
756 let id = chromiumoxide_types::CallId::new(next_call_id);
757 next_call_id = next_call_id.wrapping_add(1);
758 id
759 };
760
761 let mut evict_timer = tokio::time::interval_at(
763 tokio::time::Instant::now() + self.config.request_timeout,
764 self.config.request_timeout,
765 );
766 evict_timer.set_missed_tick_behavior(MissedTickBehavior::Delay);
767
768 macro_rules! ws_submit {
770 ($method:expr, $session_id:expr, $params:expr) => {{
771 let id = alloc_call_id();
772 let call = chromiumoxide_types::MethodCall {
773 id,
774 method: $method,
775 session_id: $session_id,
776 params: $params,
777 };
778 match ws_tx.try_send(call) {
779 Ok(()) => Ok::<_, CdpError>(id),
780 Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => {
781 tracing::warn!("WS command channel full — dropping command");
782 Err(CdpError::msg("WS command channel full"))
783 }
784 Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {
785 Err(CdpError::msg("WS writer closed"))
786 }
787 }
788 }};
789 }
790
791 loop {
793 let now = std::time::Instant::now();
794
795 const PER_TARGET_DRAIN_BUDGET: usize = 128;
801
802 for n in (0..self.target_ids.len()).rev() {
803 let target_id = self.target_ids.swap_remove(n);
804
805 if let Some((id, mut target)) = self.targets.remove_entry(&target_id) {
806 {
808 let mut msgs = Vec::new();
809 if let Some(handle) = target.page_mut() {
810 while msgs.len() < PER_TARGET_DRAIN_BUDGET {
811 match handle.rx.try_recv() {
812 Ok(msg) => msgs.push(msg),
813 Err(_) => break,
814 }
815 }
816 }
817 for msg in msgs {
818 target.on_page_message(msg);
819 }
820 }
821
822 while let Some(event) = target.advance(now) {
824 match event {
825 TargetEvent::Request(req) => {
826 if let Ok(call_id) =
827 ws_submit!(req.method.clone(), req.session_id, req.params)
828 {
829 self.pending_commands.insert(
830 call_id,
831 (
832 PendingRequest::InternalCommand(
833 target.target_id().clone(),
834 ),
835 req.method,
836 now,
837 ),
838 );
839 }
840 }
841 TargetEvent::Command(msg) => {
842 if msg.is_navigation() {
843 let (req, tx) = msg.split();
844 let nav_id = self.next_navigation_id();
845 target.goto(FrameRequestedNavigation::new(
846 nav_id,
847 req.clone(),
848 self.config.request_timeout,
849 ));
850 if let Ok(call_id) =
851 ws_submit!(req.method.clone(), req.session_id, req.params)
852 {
853 self.pending_commands.insert(
854 call_id,
855 (PendingRequest::Navigate(nav_id), req.method, now),
856 );
857 }
858 self.navigations.insert(
859 nav_id,
860 NavigationRequest::Navigate(NavigationInProgress::new(tx)),
861 );
862 } else if let Ok(call_id) = ws_submit!(
863 msg.method.clone(),
864 msg.session_id.map(Into::into),
865 msg.params
866 ) {
867 let target_id = Some(target.target_id().clone());
871 self.pending_commands.insert(
872 call_id,
873 (
874 PendingRequest::ExternalCommand {
875 tx: msg.sender,
876 target_id,
877 },
878 msg.method,
879 now,
880 ),
881 );
882 }
883 }
884 TargetEvent::NavigationRequest(nav_id, req) => {
885 if let Ok(call_id) =
886 ws_submit!(req.method.clone(), req.session_id, req.params)
887 {
888 self.pending_commands.insert(
889 call_id,
890 (PendingRequest::Navigate(nav_id), req.method, now),
891 );
892 }
893 }
894 TargetEvent::NavigationResult(res) => {
895 self.on_navigation_lifecycle_completed(res);
896 }
897 TargetEvent::BytesConsumed(n) => {
898 if let Some(rem) = self.remaining_bytes.as_mut() {
899 *rem = rem.saturating_sub(n);
900 if *rem == 0 {
901 self.budget_exhausted = true;
902 }
903 }
904 }
905 }
906 }
907
908 target.event_listeners_mut().flush();
910
911 self.targets.insert(id, target);
912 self.target_ids.push(target_id);
913 }
914 }
915
916 self.event_listeners.flush();
918
919 if self.budget_exhausted {
920 for t in self.targets.values_mut() {
921 t.network_manager.set_block_all(true);
922 }
923 }
924
925 if self.closing {
926 break;
927 }
928
929 tokio::select! {
931 msg = self.from_browser.recv() => {
932 match msg {
933 Some(msg) => {
934 match msg {
935 HandlerMessage::Command(cmd) => {
936 let target_id = cmd
939 .session_id
940 .as_ref()
941 .and_then(|sid| self.sessions.get(sid.as_ref()))
942 .map(|s| s.target_id().clone());
943 if let Ok(call_id) = ws_submit!(
944 cmd.method.clone(),
945 cmd.session_id.map(Into::into),
946 cmd.params
947 ) {
948 self.pending_commands.insert(
949 call_id,
950 (
951 PendingRequest::ExternalCommand {
952 tx: cmd.sender,
953 target_id,
954 },
955 cmd.method,
956 now,
957 ),
958 );
959 }
960 }
961 HandlerMessage::FetchTargets(tx) => {
962 let msg = TARGET_PARAMS_ID.clone();
963 if let Ok(call_id) = ws_submit!(msg.0.clone(), None, msg.1) {
964 self.pending_commands.insert(
965 call_id,
966 (PendingRequest::GetTargets(tx), msg.0, now),
967 );
968 }
969 }
970 HandlerMessage::CloseBrowser(tx) => {
971 let close_msg = CLOSE_PARAMS_ID.clone();
972 if let Ok(call_id) = ws_submit!(close_msg.0.clone(), None, close_msg.1) {
973 self.pending_commands.insert(
974 call_id,
975 (PendingRequest::CloseBrowser(tx), close_msg.0, now),
976 );
977 }
978 }
979 HandlerMessage::CreatePage(params, tx) => {
980 if let Some(ref id) = params.browser_context_id {
981 self.browser_contexts.insert(BrowserContext::from(id.clone()));
982 }
983 self.create_page_async(params, tx, &mut alloc_call_id, &ws_tx, now);
984 }
985 HandlerMessage::GetPages(tx) => {
986 let pages: Vec<_> = self.targets.values_mut()
987 .filter(|p| p.is_page())
988 .filter_map(|target| target.get_or_create_page())
989 .map(|page| Page::from(page.clone()))
990 .collect();
991 let _ = tx.send(pages);
992 }
993 HandlerMessage::InsertContext(ctx) => {
994 if self.default_browser_context.id().is_none() {
995 self.default_browser_context = ctx.clone();
996 }
997 self.browser_contexts.insert(ctx);
998 }
999 HandlerMessage::DisposeContext(ctx) => {
1000 self.browser_contexts.remove(&ctx);
1001 self.attached_targets.retain(|tid| {
1002 self.targets.get(tid)
1003 .and_then(|t| t.browser_context_id())
1004 .map(|id| Some(id) != ctx.id())
1005 .unwrap_or(true)
1006 });
1007 self.closing = true;
1008 }
1009 HandlerMessage::GetPage(target_id, tx) => {
1010 let page = self.targets.get_mut(&target_id)
1011 .and_then(|target| target.get_or_create_page())
1012 .map(|page| Page::from(page.clone()));
1013 let _ = tx.send(page);
1014 }
1015 HandlerMessage::AddEventListener(req) => {
1016 self.event_listeners.add_listener(req);
1017 }
1018 }
1019 }
1020 None => break, }
1022 }
1023
1024 frame = ws_reader.next_message() => {
1025 match frame {
1026 Some(Ok(boxed_msg)) => match *boxed_msg {
1027 Message::Response(resp) => {
1028 self.on_response(resp);
1029 }
1030 Message::Event(ev) => {
1031 self.on_event(ev);
1032 }
1033 },
1034 Some(Err(err)) => {
1035 tracing::error!("WS Connection error: {:?}", err);
1036 if let CdpError::Ws(ref ws_error) = err {
1037 match ws_error {
1038 tungstenite::Error::AlreadyClosed => break,
1039 tungstenite::Error::Protocol(detail)
1040 if detail == &ProtocolError::ResetWithoutClosingHandshake =>
1041 {
1042 break;
1043 }
1044 _ => return Err(err),
1045 }
1046 } else {
1047 return Err(err);
1048 }
1049 }
1050 None => break, }
1052 }
1053
1054 _ = page_wake.notified() => {
1055 }
1057
1058 _ = evict_timer.tick() => {
1059 self.evict_timed_out_commands(now);
1060 for t in self.targets.values_mut() {
1061 t.network_manager.evict_stale_entries(now);
1062 t.frame_manager_mut().evict_stale_context_ids();
1063 }
1064 }
1065
1066 result = &mut writer_handle => {
1067 match result {
1069 Ok(Ok(())) => break,
1070 Ok(Err(e)) => return Err(e),
1071 Err(e) => return Err(CdpError::msg(format!("WS writer panicked: {e}"))),
1072 }
1073 }
1074 }
1075 }
1076
1077 writer_handle.abort();
1078 Ok(())
1079 }
1080
1081 fn create_page_async(
1083 &mut self,
1084 params: CreateTargetParams,
1085 tx: OneshotSender<Result<Page>>,
1086 alloc_call_id: &mut impl FnMut() -> chromiumoxide_types::CallId,
1087 ws_tx: &tokio::sync::mpsc::Sender<chromiumoxide_types::MethodCall>,
1088 now: std::time::Instant,
1089 ) {
1090 let about_blank = params.url == "about:blank";
1091 let http_check =
1092 !about_blank && params.url.starts_with("http") || params.url.starts_with("file://");
1093
1094 if about_blank || http_check {
1095 let method = params.identifier();
1096 match serde_json::to_value(params) {
1097 Ok(params) => {
1098 let id = alloc_call_id();
1099 let call = chromiumoxide_types::MethodCall {
1100 id,
1101 method: method.clone(),
1102 session_id: None,
1103 params,
1104 };
1105 match ws_tx.try_send(call) {
1106 Ok(()) => {
1107 self.pending_commands
1108 .insert(id, (PendingRequest::CreateTarget(tx), method, now));
1109 }
1110 Err(_) => {
1111 let _ = tx
1112 .send(Err(CdpError::msg("WS command channel full or closed")))
1113 .ok();
1114 }
1115 }
1116 }
1117 Err(err) => {
1118 let _ = tx.send(Err(err.into())).ok();
1119 }
1120 }
1121 } else {
1122 let _ = tx.send(Err(CdpError::NotFound)).ok();
1123 }
1124 }
1125}
1126
1127impl Stream for Handler {
1128 type Item = Result<()>;
1129
1130 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
1131 const BROWSER_MSG_BUDGET: usize = 128;
1137 const PER_TARGET_DRAIN_BUDGET: usize = 128;
1138 const WS_MSG_BUDGET: usize = 512;
1139
1140 let pin = self.get_mut();
1141
1142 let mut dispose = false;
1143 let mut budget_hit = false;
1144
1145 let now = Instant::now();
1146
1147 loop {
1148 let mut browser_msgs = 0usize;
1152 while let Poll::Ready(Some(msg)) = pin.from_browser.poll_recv(cx) {
1153 match msg {
1154 HandlerMessage::Command(cmd) => {
1155 pin.submit_external_command(cmd, now)?;
1156 }
1157 HandlerMessage::FetchTargets(tx) => {
1158 pin.submit_fetch_targets(tx, now);
1159 }
1160 HandlerMessage::CloseBrowser(tx) => {
1161 pin.submit_close(tx, now);
1162 }
1163 HandlerMessage::CreatePage(params, tx) => {
1164 if let Some(ref id) = params.browser_context_id {
1165 pin.browser_contexts
1166 .insert(BrowserContext::from(id.clone()));
1167 }
1168 pin.create_page(params, tx);
1169 }
1170 HandlerMessage::GetPages(tx) => {
1171 let pages: Vec<_> = pin
1172 .targets
1173 .values_mut()
1174 .filter(|p: &&mut Target| p.is_page())
1175 .filter_map(|target| target.get_or_create_page())
1176 .map(|page| Page::from(page.clone()))
1177 .collect();
1178 let _ = tx.send(pages);
1179 }
1180 HandlerMessage::InsertContext(ctx) => {
1181 if pin.default_browser_context.id().is_none() {
1182 pin.default_browser_context = ctx.clone();
1183 }
1184 pin.browser_contexts.insert(ctx);
1185 }
1186 HandlerMessage::DisposeContext(ctx) => {
1187 pin.browser_contexts.remove(&ctx);
1188 pin.attached_targets.retain(|tid| {
1189 pin.targets
1190 .get(tid)
1191 .and_then(|t| t.browser_context_id()) .map(|id| Some(id) != ctx.id())
1193 .unwrap_or(true)
1194 });
1195 pin.closing = true;
1196 dispose = true;
1197 }
1198 HandlerMessage::GetPage(target_id, tx) => {
1199 let page = pin
1200 .targets
1201 .get_mut(&target_id)
1202 .and_then(|target| target.get_or_create_page())
1203 .map(|page| Page::from(page.clone()));
1204 let _ = tx.send(page);
1205 }
1206 HandlerMessage::AddEventListener(req) => {
1207 pin.event_listeners.add_listener(req);
1208 }
1209 }
1210 browser_msgs += 1;
1211 if browser_msgs >= BROWSER_MSG_BUDGET {
1212 budget_hit = true;
1213 break;
1214 }
1215 }
1216
1217 for n in (0..pin.target_ids.len()).rev() {
1218 let target_id = pin.target_ids.swap_remove(n);
1219
1220 if let Some((id, mut target)) = pin.targets.remove_entry(&target_id) {
1221 let mut drained = 0usize;
1222 while let Some(event) = target.poll(cx, now) {
1223 match event {
1224 TargetEvent::Request(req) => {
1225 let _ = pin.submit_internal_command(
1226 target.target_id().clone(),
1227 req,
1228 now,
1229 );
1230 }
1231 TargetEvent::Command(msg) => {
1232 pin.on_target_message(&mut target, msg, now);
1233 }
1234 TargetEvent::NavigationRequest(id, req) => {
1235 pin.submit_navigation(id, req, now);
1236 }
1237 TargetEvent::NavigationResult(res) => {
1238 pin.on_navigation_lifecycle_completed(res)
1239 }
1240 TargetEvent::BytesConsumed(n) => {
1241 if let Some(rem) = pin.remaining_bytes.as_mut() {
1242 *rem = rem.saturating_sub(n);
1243 if *rem == 0 {
1244 pin.budget_exhausted = true;
1245 }
1246 }
1247 }
1248 }
1249 drained += 1;
1250 if drained >= PER_TARGET_DRAIN_BUDGET {
1251 budget_hit = true;
1252 break;
1253 }
1254 }
1255
1256 target.event_listeners_mut().poll(cx);
1258
1259 pin.targets.insert(id, target);
1260 pin.target_ids.push(target_id);
1261 }
1262 }
1263
1264 pin.event_listeners_mut().poll(cx);
1267
1268 let mut done = true;
1269
1270 let mut ws_msgs = Vec::new();
1273 let mut ws_err = None;
1274 {
1275 let Some(conn) = pin.conn.as_mut() else {
1276 return Poll::Ready(Some(Err(CdpError::msg(
1277 "connection consumed by Handler::run()",
1278 ))));
1279 };
1280 while let Poll::Ready(Some(ev)) = Pin::new(&mut *conn).poll_next(cx) {
1281 match ev {
1282 Ok(msg) => ws_msgs.push(msg),
1283 Err(err) => {
1284 ws_err = Some(err);
1285 break;
1286 }
1287 }
1288 if ws_msgs.len() >= WS_MSG_BUDGET {
1289 budget_hit = true;
1290 break;
1291 }
1292 }
1293 }
1294
1295 for boxed_msg in ws_msgs {
1296 match *boxed_msg {
1297 Message::Response(resp) => {
1298 pin.on_response(resp);
1299 if pin.closing {
1300 return Poll::Ready(None);
1301 }
1302 }
1303 Message::Event(ev) => {
1304 pin.on_event(ev);
1305 }
1306 }
1307 done = false;
1308 }
1309
1310 if let Some(err) = ws_err {
1311 tracing::error!("WS Connection error: {:?}", err);
1312 if let CdpError::Ws(ref ws_error) = err {
1313 match ws_error {
1314 Error::AlreadyClosed => {
1315 pin.closing = true;
1316 dispose = true;
1317 }
1318 Error::Protocol(detail)
1319 if detail == &ProtocolError::ResetWithoutClosingHandshake =>
1320 {
1321 pin.closing = true;
1322 dispose = true;
1323 }
1324 _ => return Poll::Ready(Some(Err(err))),
1325 }
1326 } else {
1327 return Poll::Ready(Some(Err(err)));
1328 }
1329 }
1330
1331 if pin.evict_command_timeout.poll_ready(cx) {
1332 pin.evict_timed_out_commands(now);
1334 for t in pin.targets.values_mut() {
1337 t.network_manager.evict_stale_entries(now);
1338 t.frame_manager_mut().evict_stale_context_ids();
1339 }
1340 }
1341
1342 if pin.budget_exhausted {
1343 for t in pin.targets.values_mut() {
1344 t.network_manager.set_block_all(true);
1345 }
1346 }
1347
1348 if dispose {
1349 return Poll::Ready(None);
1350 }
1351
1352 if budget_hit {
1353 cx.waker().wake_by_ref();
1357 return Poll::Pending;
1358 }
1359
1360 if done {
1361 return Poll::Pending;
1363 }
1364 }
1365 }
1366}
1367
1368#[derive(Debug, Clone)]
1370pub struct HandlerConfig {
1371 pub ignore_https_errors: bool,
1373 pub viewport: Option<Viewport>,
1375 pub context_ids: Vec<BrowserContextId>,
1377 pub request_timeout: Duration,
1379 pub request_intercept: bool,
1381 pub cache_enabled: bool,
1383 pub service_worker_enabled: bool,
1385 pub ignore_visuals: bool,
1387 pub ignore_stylesheets: bool,
1389 pub ignore_javascript: bool,
1391 pub ignore_analytics: bool,
1393 pub ignore_prefetch: bool,
1395 pub ignore_ads: bool,
1397 pub extra_headers: Option<std::collections::HashMap<String, String>>,
1399 pub only_html: bool,
1401 pub created_first_target: bool,
1403 pub intercept_manager: NetworkInterceptManager,
1405 pub max_bytes_allowed: Option<u64>,
1407 pub max_redirects: Option<usize>,
1414 pub max_main_frame_navigations: Option<u32>,
1418 pub whitelist_patterns: Option<Vec<String>>,
1420 pub blacklist_patterns: Option<Vec<String>>,
1422 #[cfg(feature = "adblock")]
1424 pub adblock_filter_rules: Option<Vec<String>>,
1425 pub channel_capacity: usize,
1428 pub connection_retries: u32,
1431}
1432
1433impl Default for HandlerConfig {
1434 fn default() -> Self {
1435 Self {
1436 ignore_https_errors: true,
1437 viewport: Default::default(),
1438 context_ids: Vec::new(),
1439 request_timeout: Duration::from_millis(REQUEST_TIMEOUT),
1440 request_intercept: false,
1441 cache_enabled: true,
1442 service_worker_enabled: true,
1443 ignore_visuals: false,
1444 ignore_stylesheets: false,
1445 ignore_ads: false,
1446 ignore_javascript: false,
1447 ignore_analytics: true,
1448 ignore_prefetch: true,
1449 only_html: false,
1450 extra_headers: Default::default(),
1451 created_first_target: false,
1452 intercept_manager: NetworkInterceptManager::Unknown,
1453 max_bytes_allowed: None,
1454 max_redirects: None,
1455 max_main_frame_navigations: None,
1456 whitelist_patterns: None,
1457 blacklist_patterns: None,
1458 #[cfg(feature = "adblock")]
1459 adblock_filter_rules: None,
1460 channel_capacity: 4096,
1461 connection_retries: crate::conn::DEFAULT_CONNECTION_RETRIES,
1462 }
1463 }
1464}
1465
1466#[derive(Debug)]
1468pub struct NavigationInProgress<T> {
1469 navigated: bool,
1471 response: Option<Response>,
1473 tx: OneshotSender<T>,
1475}
1476
1477impl<T> NavigationInProgress<T> {
1478 fn new(tx: OneshotSender<T>) -> Self {
1479 Self {
1480 navigated: false,
1481 response: None,
1482 tx,
1483 }
1484 }
1485
1486 fn set_response(&mut self, resp: Response) {
1488 self.response = Some(resp);
1489 }
1490
1491 fn set_navigated(&mut self) {
1493 self.navigated = true;
1494 }
1495}
1496
1497#[derive(Debug)]
1499enum NavigationRequest {
1500 Navigate(NavigationInProgress<Result<Response>>),
1502 }
1504
1505#[derive(Debug)]
1508enum PendingRequest {
1509 CreateTarget(OneshotSender<Result<Page>>),
1512 GetTargets(OneshotSender<Result<Vec<TargetInfo>>>),
1514 Navigate(NavigationId),
1521 ExternalCommand {
1530 tx: OneshotSender<Result<Response>>,
1531 target_id: Option<TargetId>,
1532 },
1533 InternalCommand(TargetId),
1536 CloseBrowser(OneshotSender<Result<CloseReturns>>),
1538}
1539
1540#[derive(Debug)]
1544pub(crate) enum HandlerMessage {
1545 CreatePage(CreateTargetParams, OneshotSender<Result<Page>>),
1546 FetchTargets(OneshotSender<Result<Vec<TargetInfo>>>),
1547 InsertContext(BrowserContext),
1548 DisposeContext(BrowserContext),
1549 GetPages(OneshotSender<Vec<Page>>),
1550 Command(CommandMessage),
1551 GetPage(TargetId, OneshotSender<Option<Page>>),
1552 AddEventListener(EventListenerRequest),
1553 CloseBrowser(OneshotSender<Result<CloseReturns>>),
1554}
1555
1556#[cfg(test)]
1557mod tests {
1558 use super::*;
1559 use chromiumoxide_cdp::cdp::browser_protocol::target::{AttachToTargetReturns, TargetInfo};
1560
1561 #[test]
1562 fn attach_to_target_response_sets_session_id_before_event_arrives() {
1563 let info = TargetInfo::builder()
1564 .target_id("target-1".to_string())
1565 .r#type("page")
1566 .title("")
1567 .url("about:blank")
1568 .attached(false)
1569 .can_access_opener(false)
1570 .build()
1571 .expect("target info");
1572 let mut target = Target::new(info, TargetConfig::default(), BrowserContext::default());
1573 let method: MethodId = AttachToTargetParams::IDENTIFIER.into();
1574 let result = serde_json::to_value(AttachToTargetReturns::new("session-1".to_string()))
1575 .expect("attach result");
1576 let resp = Response {
1577 id: CallId::new(1),
1578 result: Some(result),
1579 error: None,
1580 };
1581
1582 maybe_store_attach_session_id(&mut target, &method, &resp);
1583
1584 assert_eq!(
1585 target.session_id().map(AsRef::as_ref),
1586 Some("session-1"),
1587 "attach response should seed the flat session id even before Target.attachedToTarget"
1588 );
1589 }
1590}