chromiumoxide/handler/
target.rs

1use std::collections::VecDeque;
2use std::pin::Pin;
3use std::sync::Arc;
4use std::time::Instant;
5
6use chromiumoxide_cdp::cdp::browser_protocol::target::DetachFromTargetParams;
7use futures::channel::oneshot::Sender;
8use futures::stream::Stream;
9use futures::task::{Context, Poll};
10
11use crate::auth::Credentials;
12use crate::cdp::browser_protocol::target::CloseTargetParams;
13use crate::cmd::CommandChain;
14use crate::cmd::CommandMessage;
15use crate::error::{CdpError, Result};
16use crate::handler::browser::BrowserContext;
17use crate::handler::domworld::DOMWorldKind;
18use crate::handler::emulation::EmulationManager;
19use crate::handler::frame::FrameRequestedNavigation;
20use crate::handler::frame::{
21    FrameEvent, FrameManager, NavigationError, NavigationId, NavigationOk,
22};
23use crate::handler::network::{NetworkEvent, NetworkManager};
24use crate::handler::page::PageHandle;
25use crate::handler::viewport::Viewport;
26use crate::handler::{PageInner, REQUEST_TIMEOUT};
27use crate::listeners::{EventListenerRequest, EventListeners};
28use crate::{page::Page, ArcHttpRequest};
29use chromiumoxide_cdp::cdp::browser_protocol::{
30    browser::BrowserContextId,
31    log as cdplog,
32    page::{FrameId, GetFrameTreeParams},
33    target::{AttachToTargetParams, SessionId, SetAutoAttachParams, TargetId, TargetInfo},
34};
35use chromiumoxide_cdp::cdp::events::CdpEvent;
36use chromiumoxide_cdp::cdp::js_protocol::runtime::{
37    ExecutionContextId, RunIfWaitingForDebuggerParams,
38};
39use chromiumoxide_cdp::cdp::CdpEventMessage;
40use chromiumoxide_types::{Command, Method, Request, Response};
41use spider_network_blocker::intercept_manager::NetworkInterceptManager;
42use std::time::Duration;
43
44macro_rules! advance_state {
45    ($s:ident, $cx:ident, $now:ident, $cmds: ident, $next_state:expr ) => {{
46        if let Poll::Ready(poll) = $cmds.poll($now) {
47            return match poll {
48                None => {
49                    $s.init_state = $next_state;
50                    $s.poll($cx, $now)
51                }
52                Some(Ok((method, params))) => Some(TargetEvent::Request(Request {
53                    method,
54                    session_id: $s.session_id.clone().map(Into::into),
55                    params,
56                })),
57                Some(Err(_)) => Some($s.on_initialization_failed()),
58            };
59        } else {
60            return None;
61        }
62    }};
63}
64
65lazy_static::lazy_static! {
66    /// Initial start command params.
67    static ref INIT_COMMANDS_PARAMS: Vec<(chromiumoxide_types::MethodId, serde_json::Value)> = {
68        if let Ok(attach) = SetAutoAttachParams::builder()
69            .flatten(true)
70            .auto_attach(true)
71            .wait_for_debugger_on_start(true)
72            .build() {
73                let disable_log = cdplog::DisableParams::default();
74
75                let mut cmds =  vec![
76                    (
77                        attach.identifier(),
78                        serde_json::to_value(attach).unwrap_or_default(),
79                    ),
80                    (
81                        disable_log.identifier(),
82                        serde_json::to_value(disable_log).unwrap_or_default(),
83                    )
84                ];
85
86                // enable performance on pages.
87                if cfg!(feature = "collect_metrics") {
88                    let enable_performance = chromiumoxide_cdp::cdp::browser_protocol::performance::EnableParams::default();
89                    cmds.push((
90                        enable_performance.identifier(),
91                        serde_json::to_value(enable_performance).unwrap_or_default(),
92                    ));
93                }
94
95                cmds
96            } else {
97                vec![]
98            }
99    };
100
101    /// Attach to target commands
102    static ref ATTACH_TARGET: (chromiumoxide_types::MethodId, serde_json::Value) = {
103        let runtime_cmd = RunIfWaitingForDebuggerParams::default();
104
105        (runtime_cmd.identifier(), serde_json::to_value(runtime_cmd).unwrap_or_default())
106    };
107}
108
109#[derive(Debug)]
110pub struct Target {
111    /// Info about this target as returned from the chromium instance
112    info: TargetInfo,
113    /// The type of this target
114    r#type: TargetType,
115    /// Configs for this target
116    config: TargetConfig,
117    /// The context this target is running in
118    browser_context: BrowserContext,
119    /// The frame manager that maintains the state of all frames and handles
120    /// navigations of frames
121    frame_manager: FrameManager,
122    /// Handles all the https
123    pub(crate) network_manager: NetworkManager,
124    emulation_manager: EmulationManager,
125    /// The identifier of the session this target is attached to
126    session_id: Option<SessionId>,
127    /// The handle of the browser page of this target
128    page: Option<PageHandle>,
129    /// Drives this target towards initialization
130    pub(crate) init_state: TargetInit,
131    /// Currently queued events to report to the `Handler`
132    queued_events: VecDeque<TargetEvent>,
133    /// All registered event subscriptions
134    event_listeners: EventListeners,
135    /// Senders that need to be notified once the main frame has loaded
136    wait_for_frame_navigation: Vec<Sender<ArcHttpRequest>>,
137    /// Senders that need to be notified once the main frame reaches `networkIdle`.
138    wait_for_network_idle: Vec<Sender<ArcHttpRequest>>,
139    /// (Optional) for `networkAlmostIdle` if you want it as well.
140    wait_for_network_almost_idle: Vec<Sender<ArcHttpRequest>>,
141    /// The sender who requested the page.
142    initiator: Option<Sender<Result<Page>>>,
143}
144
145impl Target {
146    /// Create a new target instance with `TargetInfo` after a
147    /// `CreateTargetParams` request.
148    pub fn new(info: TargetInfo, config: TargetConfig, browser_context: BrowserContext) -> Self {
149        let ty = TargetType::new(&info.r#type);
150        let request_timeout: Duration = config.request_timeout;
151        let mut network_manager = NetworkManager::new(config.ignore_https_errors, request_timeout);
152
153        if !config.cache_enabled {
154            network_manager.set_cache_enabled(false);
155        }
156
157        if !config.service_worker_enabled {
158            network_manager.set_service_worker_enabled(true);
159        }
160
161        network_manager.set_request_interception(config.request_intercept);
162        network_manager.max_bytes_allowed = config.max_bytes_allowed;
163
164        if let Some(headers) = &config.extra_headers {
165            network_manager.set_extra_headers(headers.clone());
166        }
167
168        if let Some(whitelist) = &config.whitelist_patterns {
169            network_manager.set_whitelist_patterns(whitelist.clone());
170        }
171
172        if let Some(blacklist) = &config.blacklist_patterns {
173            network_manager.set_blacklist_patterns(blacklist);
174        }
175
176        network_manager.ignore_visuals = config.ignore_visuals;
177        network_manager.block_javascript = config.ignore_javascript;
178        network_manager.block_analytics = config.ignore_analytics;
179        network_manager.block_prefetch = config.ignore_prefetch;
180
181        network_manager.block_stylesheets = config.ignore_stylesheets;
182        network_manager.only_html = config.only_html;
183        network_manager.intercept_manager = config.intercept_manager;
184
185        Self {
186            info,
187            r#type: ty,
188            config,
189            frame_manager: FrameManager::new(request_timeout),
190            network_manager,
191            emulation_manager: EmulationManager::new(request_timeout),
192            session_id: None,
193            page: None,
194            init_state: TargetInit::AttachToTarget,
195            wait_for_frame_navigation: Default::default(),
196            wait_for_network_idle: Default::default(),
197            wait_for_network_almost_idle: Default::default(),
198            queued_events: Default::default(),
199            event_listeners: Default::default(),
200            initiator: None,
201            browser_context,
202        }
203    }
204
205    /// Set the session id.
206    pub fn set_session_id(&mut self, id: SessionId) {
207        self.session_id = Some(id)
208    }
209
210    /// Get the session id.
211    pub fn session_id(&self) -> Option<&SessionId> {
212        self.session_id.as_ref()
213    }
214
215    /// Get the session id mut.
216    pub fn session_id_mut(&mut self) -> &mut Option<SessionId> {
217        &mut self.session_id
218    }
219
220    /// Get the browser context.
221    pub fn browser_context(&self) -> &BrowserContext {
222        &self.browser_context
223    }
224
225    /// The identifier for this target
226    pub fn target_id(&self) -> &TargetId {
227        &self.info.target_id
228    }
229
230    /// The type of this target
231    pub fn r#type(&self) -> &TargetType {
232        &self.r#type
233    }
234
235    /// Whether this target is already initialized
236    pub fn is_initialized(&self) -> bool {
237        matches!(self.init_state, TargetInit::Initialized)
238    }
239
240    /// Navigate a frame
241    pub fn goto(&mut self, req: FrameRequestedNavigation) {
242        if self.network_manager.has_target_domain() {
243            self.network_manager.clear_target_domain();
244            let goto_url = req
245                .req
246                .params
247                .as_object()
248                .and_then(|o| o.get("url"))
249                .and_then(|v| v.as_str());
250
251            if let Some(url) = goto_url {
252                self.network_manager.set_page_url(url.into());
253            }
254        }
255        self.frame_manager.goto(req);
256    }
257
258    /// Create a new page from the session.
259    fn create_page(&mut self) {
260        if self.page.is_none() {
261            if let Some(session) = self.session_id.clone() {
262                let handle = PageHandle::new(
263                    self.target_id().clone(),
264                    session,
265                    self.opener_id().cloned(),
266                    self.config.request_timeout,
267                );
268                self.page = Some(handle);
269            }
270        }
271    }
272
273    /// Tries to create the `PageInner` if this target is already initialized
274    pub(crate) fn get_or_create_page(&mut self) -> Option<&Arc<PageInner>> {
275        self.create_page();
276        self.page.as_ref().map(|p| p.inner())
277    }
278
279    /// Is the target a page?
280    pub fn is_page(&self) -> bool {
281        self.r#type().is_page()
282    }
283
284    /// The browser context ID.
285    pub fn browser_context_id(&self) -> Option<&BrowserContextId> {
286        self.info.browser_context_id.as_ref()
287    }
288
289    /// The target connection info.
290    pub fn info(&self) -> &TargetInfo {
291        &self.info
292    }
293
294    /// Get the target that opened this target. Top-level targets return `None`.
295    pub fn opener_id(&self) -> Option<&TargetId> {
296        self.info.opener_id.as_ref()
297    }
298
299    pub fn frame_manager(&self) -> &FrameManager {
300        &self.frame_manager
301    }
302
303    /// The frame manager.
304    pub fn frame_manager_mut(&mut self) -> &mut FrameManager {
305        &mut self.frame_manager
306    }
307
308    /// Get event listeners mutably.
309    pub fn event_listeners_mut(&mut self) -> &mut EventListeners {
310        &mut self.event_listeners
311    }
312
313    /// Received a response to a command issued by this target
314    pub fn on_response(&mut self, resp: Response, method: &str) {
315        if let Some(cmds) = self.init_state.commands_mut() {
316            cmds.received_response(method);
317        }
318
319        if let GetFrameTreeParams::IDENTIFIER = method {
320            if let Some(resp) = resp
321                .result
322                .and_then(|val| GetFrameTreeParams::response_from_value(val).ok())
323            {
324                self.frame_manager.on_frame_tree(resp.frame_tree);
325            }
326        }
327        // requests originated from the network manager all return an empty response, hence they
328        // can be ignored here
329    }
330
331    /// On CDP Event message.
332    pub fn on_event(&mut self, event: CdpEventMessage) {
333        let CdpEventMessage {
334            params,
335            method,
336            session_id,
337            ..
338        } = event;
339
340        let is_session_scoped = matches!(
341            params,
342            CdpEvent::FetchRequestPaused(_)
343                | CdpEvent::FetchAuthRequired(_)
344                | CdpEvent::NetworkRequestWillBeSent(_)
345                | CdpEvent::NetworkResponseReceived(_)
346                | CdpEvent::NetworkLoadingFinished(_)
347                | CdpEvent::NetworkLoadingFailed(_)
348                | CdpEvent::PageFrameAttached(_)
349                | CdpEvent::PageFrameDetached(_)
350                | CdpEvent::PageFrameNavigated(_)
351                | CdpEvent::PageNavigatedWithinDocument(_)
352                | CdpEvent::PageLifecycleEvent(_)
353                | CdpEvent::PageFrameStartedLoading(_)
354                | CdpEvent::PageFrameStoppedLoading(_)
355                | CdpEvent::RuntimeExecutionContextCreated(_)
356                | CdpEvent::RuntimeExecutionContextDestroyed(_)
357                | CdpEvent::RuntimeExecutionContextsCleared(_)
358                | CdpEvent::RuntimeBindingCalled(_)
359        );
360
361        if is_session_scoped {
362            let ev_sid: &str = match session_id.as_deref() {
363                Some(s) => s,
364                None => return,
365            };
366
367            let self_sid: &str = match self.session_id.as_ref() {
368                Some(sid) => sid.as_ref(),
369                None => return,
370            };
371
372            if self_sid != ev_sid {
373                return;
374            }
375        }
376
377        match &params {
378            // `FrameManager` events
379            CdpEvent::PageFrameAttached(ev) => self
380                .frame_manager
381                .on_frame_attached(ev.frame_id.clone(), Some(ev.parent_frame_id.clone())),
382            CdpEvent::PageFrameDetached(ev) => self.frame_manager.on_frame_detached(ev),
383            CdpEvent::PageFrameNavigated(ev) => {
384                self.frame_manager.on_frame_navigated(&ev.frame);
385            }
386            CdpEvent::PageNavigatedWithinDocument(ev) => {
387                self.frame_manager.on_frame_navigated_within_document(ev)
388            }
389            CdpEvent::RuntimeExecutionContextCreated(ev) => {
390                self.frame_manager.on_frame_execution_context_created(ev)
391            }
392            CdpEvent::RuntimeExecutionContextDestroyed(ev) => {
393                self.frame_manager.on_frame_execution_context_destroyed(ev)
394            }
395            CdpEvent::RuntimeExecutionContextsCleared(_) => {
396                self.frame_manager.on_execution_contexts_cleared()
397            }
398            CdpEvent::RuntimeBindingCalled(ev) => {
399                // TODO check if binding registered and payload is json
400                self.frame_manager.on_runtime_binding_called(ev)
401            }
402            CdpEvent::PageLifecycleEvent(ev) => self.frame_manager.on_page_lifecycle_event(ev),
403            CdpEvent::PageFrameStartedLoading(ev) => {
404                self.frame_manager.on_frame_started_loading(ev);
405            }
406            CdpEvent::PageFrameStoppedLoading(ev) => {
407                self.frame_manager.on_frame_stopped_loading(ev);
408            }
409            // `Target` events
410            CdpEvent::TargetAttachedToTarget(ev) => {
411                if ev.waiting_for_debugger {
412                    let runtime_cmd = ATTACH_TARGET.clone();
413
414                    self.queued_events.push_back(TargetEvent::Request(Request {
415                        method: runtime_cmd.0,
416                        session_id: Some(ev.session_id.clone().into()),
417                        params: runtime_cmd.1,
418                    }));
419                }
420
421                if "service_worker" == &ev.target_info.r#type {
422                    let detach_command = DetachFromTargetParams::builder()
423                        .session_id(ev.session_id.clone())
424                        .build();
425
426                    let method = detach_command.identifier();
427
428                    if let Ok(params) = serde_json::to_value(detach_command) {
429                        self.queued_events.push_back(TargetEvent::Request(Request {
430                            method,
431                            session_id: self.session_id.clone().map(Into::into),
432                            params,
433                        }));
434                    }
435                }
436            }
437            // `NetworkManager` events
438            CdpEvent::FetchRequestPaused(ev) => self.network_manager.on_fetch_request_paused(ev),
439            CdpEvent::FetchAuthRequired(ev) => self.network_manager.on_fetch_auth_required(ev),
440            CdpEvent::NetworkRequestWillBeSent(ev) => {
441                self.network_manager.on_request_will_be_sent(ev)
442            }
443            CdpEvent::NetworkRequestServedFromCache(ev) => {
444                self.network_manager.on_request_served_from_cache(ev)
445            }
446            CdpEvent::NetworkResponseReceived(ev) => self.network_manager.on_response_received(ev),
447            CdpEvent::NetworkLoadingFinished(ev) => {
448                self.network_manager.on_network_loading_finished(ev)
449            }
450            CdpEvent::NetworkLoadingFailed(ev) => {
451                self.network_manager.on_network_loading_failed(ev)
452            }
453            _ => (),
454        }
455        chromiumoxide_cdp::consume_event!(match params {
456           |ev| self.event_listeners.start_send(ev),
457           |json| { let _ = self.event_listeners.try_send_custom(&method, json);}
458        });
459    }
460
461    /// Called when a init command timed out
462    fn on_initialization_failed(&mut self) -> TargetEvent {
463        if let Some(initiator) = self.initiator.take() {
464            let _ = initiator.send(Err(CdpError::Timeout));
465        }
466        self.init_state = TargetInit::Closing;
467        let close_target = CloseTargetParams::new(self.info.target_id.clone());
468
469        TargetEvent::Request(Request {
470            method: close_target.identifier(),
471            session_id: self.session_id.clone().map(Into::into),
472            params: serde_json::to_value(close_target).unwrap_or_default(),
473        })
474    }
475
476    /// Advance that target's state
477    pub(crate) fn poll(&mut self, cx: &mut Context<'_>, now: Instant) -> Option<TargetEvent> {
478        if !self.is_page() {
479            // can only poll pages
480            return None;
481        }
482
483        match &mut self.init_state {
484            TargetInit::AttachToTarget => {
485                self.init_state = TargetInit::InitializingFrame(FrameManager::init_commands(
486                    self.config.request_timeout,
487                ));
488
489                if let Ok(params) = AttachToTargetParams::builder()
490                    .target_id(self.target_id().clone())
491                    .flatten(true)
492                    .build()
493                {
494                    return Some(TargetEvent::Request(Request::new(
495                        params.identifier(),
496                        serde_json::to_value(params).unwrap_or_default(),
497                    )));
498                } else {
499                    return None;
500                }
501            }
502            TargetInit::InitializingFrame(cmds) => {
503                self.session_id.as_ref()?;
504                if let Poll::Ready(poll) = cmds.poll(now) {
505                    return match poll {
506                        None => {
507                            if let Some(world_name) = self.frame_manager.get_isolated_world_name() {
508                                let world_name = world_name.clone();
509
510                                if let Some(isolated_world_cmds) =
511                                    self.frame_manager.ensure_isolated_world(&world_name)
512                                {
513                                    *cmds = isolated_world_cmds;
514                                } else {
515                                    self.init_state = TargetInit::InitializingNetwork(
516                                        self.network_manager.init_commands(),
517                                    );
518                                }
519                            } else {
520                                self.init_state = TargetInit::InitializingNetwork(
521                                    self.network_manager.init_commands(),
522                                );
523                            }
524                            self.poll(cx, now)
525                        }
526                        Some(Ok((method, params))) => Some(TargetEvent::Request(Request {
527                            method,
528                            session_id: self.session_id.clone().map(Into::into),
529                            params,
530                        })),
531                        Some(Err(_)) => Some(self.on_initialization_failed()),
532                    };
533                } else {
534                    return None;
535                }
536            }
537            TargetInit::InitializingNetwork(cmds) => {
538                advance_state!(
539                    self,
540                    cx,
541                    now,
542                    cmds,
543                    TargetInit::InitializingPage(Self::page_init_commands(
544                        self.config.request_timeout
545                    ))
546                );
547            }
548            TargetInit::InitializingPage(cmds) => {
549                advance_state!(
550                    self,
551                    cx,
552                    now,
553                    cmds,
554                    match self.config.viewport.as_ref() {
555                        Some(viewport) => TargetInit::InitializingEmulation(
556                            self.emulation_manager.init_commands(viewport)
557                        ),
558                        None => TargetInit::Initialized,
559                    }
560                );
561            }
562            TargetInit::InitializingEmulation(cmds) => {
563                advance_state!(self, cx, now, cmds, TargetInit::Initialized);
564            }
565            TargetInit::Initialized => {
566                if let Some(initiator) = self.initiator.take() {
567                    // make sure that the main frame of the page has finished loading
568                    if self
569                        .frame_manager
570                        .main_frame()
571                        .map(|frame| frame.is_loaded())
572                        .unwrap_or_default()
573                    {
574                        if let Some(page) = self.get_or_create_page() {
575                            let _ = initiator.send(Ok(page.clone().into()));
576                        } else {
577                            self.initiator = Some(initiator);
578                        }
579                    } else {
580                        self.initiator = Some(initiator);
581                    }
582                }
583            }
584            TargetInit::Closing => return None,
585        };
586
587        loop {
588            if self.init_state == TargetInit::Closing {
589                break None;
590            }
591
592            if let Some(frame) = self.frame_manager.main_frame() {
593                if frame.is_loaded() {
594                    while let Some(tx) = self.wait_for_frame_navigation.pop() {
595                        let _ = tx.send(frame.http_request().cloned());
596                    }
597                }
598
599                if frame.is_network_idle() {
600                    while let Some(tx) = self.wait_for_network_idle.pop() {
601                        let _ = tx.send(frame.http_request().cloned());
602                    }
603                }
604
605                if frame.is_network_almost_idle() {
606                    while let Some(tx) = self.wait_for_network_almost_idle.pop() {
607                        let _ = tx.send(frame.http_request().cloned());
608                    }
609                }
610            }
611
612            // Drain queued messages first.
613            if let Some(ev) = self.queued_events.pop_front() {
614                return Some(ev);
615            }
616
617            if let Some(handle) = self.page.as_mut() {
618                while let Poll::Ready(Some(msg)) = Pin::new(&mut handle.rx).poll_next(cx) {
619                    if self.init_state == TargetInit::Closing {
620                        break;
621                    }
622
623                    match msg {
624                        TargetMessage::Command(cmd) => {
625                            if cmd.method == "Network.setBlockedURLs" {
626                                if let Some(arr) = cmd.params.get("urls").and_then(|v| v.as_array())
627                                {
628                                    let mut unblock_all = false;
629                                    let mut block_all = false;
630
631                                    for s in arr.iter().filter_map(|v| v.as_str()) {
632                                        if s == "!*" {
633                                            unblock_all = true;
634                                            break; // "!*" overrides any block rules
635                                        }
636                                        if s.contains('*') {
637                                            block_all = true;
638                                        }
639                                    }
640
641                                    if unblock_all {
642                                        self.network_manager.set_block_all(false);
643                                    } else if block_all {
644                                        self.network_manager.set_block_all(true);
645                                    }
646                                }
647                            }
648                            self.queued_events.push_back(TargetEvent::Command(cmd));
649                        }
650                        TargetMessage::MainFrame(tx) => {
651                            let _ =
652                                tx.send(self.frame_manager.main_frame().map(|f| f.id().clone()));
653                        }
654                        TargetMessage::AllFrames(tx) => {
655                            let _ = tx.send(
656                                self.frame_manager
657                                    .frames()
658                                    .map(|f| f.id().clone())
659                                    .collect(),
660                            );
661                        }
662                        #[cfg(feature = "_cache")]
663                        TargetMessage::CacheKey((cache_key, cache_policy)) => {
664                            self.network_manager.set_cache_site_key(cache_key);
665                            self.network_manager.set_cache_policy(cache_policy);
666                        }
667                        TargetMessage::Url(req) => {
668                            let GetUrl { frame_id, tx } = req;
669                            let frame = if let Some(frame_id) = frame_id {
670                                self.frame_manager.frame(&frame_id)
671                            } else {
672                                self.frame_manager.main_frame()
673                            };
674                            let _ = tx.send(frame.and_then(|f| f.url().map(str::to_string)));
675                        }
676                        TargetMessage::Name(req) => {
677                            let GetName { frame_id, tx } = req;
678                            let frame = if let Some(frame_id) = frame_id {
679                                self.frame_manager.frame(&frame_id)
680                            } else {
681                                self.frame_manager.main_frame()
682                            };
683                            let _ = tx.send(frame.and_then(|f| f.name().map(str::to_string)));
684                        }
685                        TargetMessage::Parent(req) => {
686                            let GetParent { frame_id, tx } = req;
687                            let frame = self.frame_manager.frame(&frame_id);
688                            let _ = tx.send(frame.and_then(|f| f.parent_id().cloned()));
689                        }
690                        TargetMessage::WaitForNavigation(tx) => {
691                            if let Some(frame) = self.frame_manager.main_frame() {
692                                if frame.is_loaded() {
693                                    let _ = tx.send(frame.http_request().cloned());
694                                } else {
695                                    self.wait_for_frame_navigation.push(tx);
696                                }
697                            } else {
698                                self.wait_for_frame_navigation.push(tx);
699                            }
700                        }
701                        TargetMessage::WaitForNetworkIdle(tx) => {
702                            if let Some(frame) = self.frame_manager.main_frame() {
703                                if frame.is_network_idle() {
704                                    let _ = tx.send(frame.http_request().cloned());
705                                } else {
706                                    self.wait_for_network_idle.push(tx);
707                                }
708                            } else {
709                                self.wait_for_network_idle.push(tx);
710                            }
711                        }
712                        TargetMessage::WaitForNetworkAlmostIdle(tx) => {
713                            if let Some(frame) = self.frame_manager.main_frame() {
714                                if frame.is_network_almost_idle() {
715                                    let _ = tx.send(frame.http_request().cloned());
716                                } else {
717                                    self.wait_for_network_almost_idle.push(tx);
718                                }
719                            } else {
720                                self.wait_for_network_almost_idle.push(tx);
721                            }
722                        }
723                        TargetMessage::AddEventListener(req) => {
724                            if req.method == "Fetch.requestPaused" {
725                                self.network_manager.enable_request_intercept();
726                            }
727                            // register a new listener
728                            self.event_listeners.add_listener(req);
729                        }
730                        TargetMessage::GetExecutionContext(ctx) => {
731                            let GetExecutionContext {
732                                dom_world,
733                                frame_id,
734                                tx,
735                            } = ctx;
736                            let frame = if let Some(frame_id) = frame_id {
737                                self.frame_manager.frame(&frame_id)
738                            } else {
739                                self.frame_manager.main_frame()
740                            };
741
742                            if let Some(frame) = frame {
743                                match dom_world {
744                                    DOMWorldKind::Main => {
745                                        let _ = tx.send(frame.main_world().execution_context());
746                                    }
747                                    DOMWorldKind::Secondary => {
748                                        let _ =
749                                            tx.send(frame.secondary_world().execution_context());
750                                    }
751                                }
752                            } else {
753                                let _ = tx.send(None);
754                            }
755                        }
756                        TargetMessage::Authenticate(credentials) => {
757                            self.network_manager.authenticate(credentials);
758                        }
759                        TargetMessage::BlockNetwork(blocked) => {
760                            self.network_manager.set_block_all(blocked);
761                        }
762                        TargetMessage::EnableInterception(enabled) => {
763                            // if interception is enabled disable the user facing handling.
764                            self.network_manager.user_request_interception_enabled = !enabled;
765                        }
766                    }
767                }
768            }
769
770            while let Some(event) = self.network_manager.poll() {
771                if self.init_state == TargetInit::Closing {
772                    break;
773                }
774                match event {
775                    NetworkEvent::SendCdpRequest((method, params)) => {
776                        // send a message to the browser
777                        self.queued_events.push_back(TargetEvent::Request(Request {
778                            method,
779                            session_id: self.session_id.clone().map(Into::into),
780                            params,
781                        }))
782                    }
783                    NetworkEvent::Request(_) => {}
784                    NetworkEvent::Response(_) => {}
785                    NetworkEvent::RequestFailed(request) => {
786                        self.frame_manager.on_http_request_finished(request);
787                    }
788                    NetworkEvent::RequestFinished(request) => {
789                        self.frame_manager.on_http_request_finished(request);
790                    }
791                    NetworkEvent::BytesConsumed(n) => {
792                        self.queued_events.push_back(TargetEvent::BytesConsumed(n));
793                    }
794                }
795            }
796
797            while let Some(event) = self.frame_manager.poll(now) {
798                if self.init_state == TargetInit::Closing {
799                    break;
800                }
801                match event {
802                    FrameEvent::NavigationResult(res) => {
803                        self.queued_events
804                            .push_back(TargetEvent::NavigationResult(res));
805                    }
806                    FrameEvent::NavigationRequest(id, req) => {
807                        self.queued_events
808                            .push_back(TargetEvent::NavigationRequest(id, req));
809                    }
810                }
811            }
812
813            if self.queued_events.is_empty() {
814                return None;
815            }
816        }
817    }
818
819    /// Set the sender half of the channel who requested the creation of this
820    /// target
821    pub fn set_initiator(&mut self, tx: Sender<Result<Page>>) {
822        self.initiator = Some(tx);
823    }
824
825    pub(crate) fn page_init_commands(timeout: Duration) -> CommandChain {
826        CommandChain::new(INIT_COMMANDS_PARAMS.clone(), timeout)
827    }
828}
829
830/// Configuration for how a single target/page should be fetched and processed.
831#[derive(Debug, Clone)]
832pub struct TargetConfig {
833    /// Whether to ignore TLS/HTTPS certificate errors (e.g. self-signed or expired certs).
834    /// When `true`, connections will proceed even if certificate validation fails.
835    pub ignore_https_errors: bool,
836    /// Request timeout to use for the main navigation / resource fetch.
837    /// This is the total time allowed before a request is considered failed.
838    pub request_timeout: Duration,
839    /// Optional browser viewport to use for this target.
840    /// When `None`, the default viewport (or headless browser default) is used.
841    pub viewport: Option<Viewport>,
842    /// Enable request interception for this target.
843    /// When `true`, all network requests will pass through the intercept manager.
844    pub request_intercept: bool,
845    /// Enable caching for this target.
846    /// When `true`, responses may be read from and written to the cache layer.
847    pub cache_enabled: bool,
848    /// If `true`, skip visual/asset resources that are not required for HTML content
849    /// (e.g. images, fonts, media). Useful for performance-oriented crawls.
850    pub ignore_visuals: bool,
851    /// If `true`, block JavaScript execution (or avoid loading JS resources)
852    /// for this target. This is useful for purely static HTML crawls.
853    pub ignore_javascript: bool,
854    /// If `true`, block analytics / tracking requests (e.g. Google Analytics,
855    /// common tracker domains, etc.).
856    pub ignore_analytics: bool,
857    /// Ignore prefetching.
858    pub ignore_prefetch: bool,
859    /// If `true`, block stylesheets and related CSS resources for this target.
860    /// This can reduce bandwidth when only raw HTML is needed.
861    pub ignore_stylesheets: bool,
862    /// If `true`, only HTML documents will be fetched/kept.
863    /// Non-HTML subresources may be skipped entirely.
864    pub only_html: bool,
865    /// Whether service workers are allowed for this target.
866    /// When `true`, service workers may register and intercept requests.
867    pub service_worker_enabled: bool,
868    /// Extra HTTP headers to send with each request for this target.
869    /// Keys should be header names, values their corresponding header values.
870    pub extra_headers: Option<std::collections::HashMap<String, String>>,
871    /// Network intercept manager used to make allow/deny/modify decisions
872    /// for requests when `request_intercept` is enabled.
873    pub intercept_manager: NetworkInterceptManager,
874    /// The maximum number of response bytes allowed for this target.
875    /// When set, responses larger than this limit may be truncated or aborted.
876    pub max_bytes_allowed: Option<u64>,
877    /// Whitelist patterns to allow through the network.
878    pub whitelist_patterns: Option<Vec<String>>,
879    /// Blacklist patterns to black through the network.
880    pub blacklist_patterns: Option<Vec<String>>,
881}
882
883impl Default for TargetConfig {
884    fn default() -> Self {
885        Self {
886            ignore_https_errors: true,
887            request_timeout: Duration::from_millis(REQUEST_TIMEOUT),
888            viewport: Default::default(),
889            request_intercept: false,
890            cache_enabled: true,
891            service_worker_enabled: true,
892            ignore_javascript: false,
893            ignore_visuals: false,
894            ignore_stylesheets: false,
895            ignore_analytics: true,
896            ignore_prefetch: true,
897            only_html: false,
898            extra_headers: Default::default(),
899            intercept_manager: NetworkInterceptManager::Unknown,
900            max_bytes_allowed: None,
901            whitelist_patterns: None,
902            blacklist_patterns: None,
903        }
904    }
905}
906
907#[derive(Debug, Clone, Eq, PartialEq)]
908pub enum TargetType {
909    Page,
910    BackgroundPage,
911    ServiceWorker,
912    SharedWorker,
913    Other,
914    Browser,
915    Webview,
916    Unknown(String),
917}
918
919impl TargetType {
920    pub fn new(ty: &str) -> Self {
921        match ty {
922            "page" => TargetType::Page,
923            "background_page" => TargetType::BackgroundPage,
924            "service_worker" => TargetType::ServiceWorker,
925            "shared_worker" => TargetType::SharedWorker,
926            "other" => TargetType::Other,
927            "browser" => TargetType::Browser,
928            "webview" => TargetType::Webview,
929            s => TargetType::Unknown(s.to_string()),
930        }
931    }
932
933    pub fn is_page(&self) -> bool {
934        matches!(self, TargetType::Page)
935    }
936
937    pub fn is_background_page(&self) -> bool {
938        matches!(self, TargetType::BackgroundPage)
939    }
940
941    pub fn is_service_worker(&self) -> bool {
942        matches!(self, TargetType::ServiceWorker)
943    }
944
945    pub fn is_shared_worker(&self) -> bool {
946        matches!(self, TargetType::SharedWorker)
947    }
948
949    pub fn is_other(&self) -> bool {
950        matches!(self, TargetType::Other)
951    }
952
953    pub fn is_browser(&self) -> bool {
954        matches!(self, TargetType::Browser)
955    }
956
957    pub fn is_webview(&self) -> bool {
958        matches!(self, TargetType::Webview)
959    }
960}
961
962#[derive(Debug)]
963pub(crate) enum TargetEvent {
964    /// An internal request
965    Request(Request),
966    /// An internal navigation request
967    NavigationRequest(NavigationId, Request),
968    /// Indicates that a previous requested navigation has finished
969    NavigationResult(Result<NavigationOk, NavigationError>),
970    /// A new command arrived via a channel
971    Command(CommandMessage),
972    /// The bytes consumed by the network.
973    BytesConsumed(u64),
974}
975
976// TODO this can be moved into the classes?
977#[derive(Debug, PartialEq)]
978pub enum TargetInit {
979    InitializingFrame(CommandChain),
980    InitializingNetwork(CommandChain),
981    InitializingPage(CommandChain),
982    InitializingEmulation(CommandChain),
983    AttachToTarget,
984    Initialized,
985    Closing,
986}
987
988impl TargetInit {
989    fn commands_mut(&mut self) -> Option<&mut CommandChain> {
990        match self {
991            TargetInit::InitializingFrame(cmd) => Some(cmd),
992            TargetInit::InitializingNetwork(cmd) => Some(cmd),
993            TargetInit::InitializingPage(cmd) => Some(cmd),
994            TargetInit::InitializingEmulation(cmd) => Some(cmd),
995            TargetInit::AttachToTarget => None,
996            TargetInit::Initialized => None,
997            TargetInit::Closing => None,
998        }
999    }
1000}
1001
1002#[derive(Debug)]
1003pub struct GetExecutionContext {
1004    /// For which world the execution context was requested
1005    pub dom_world: DOMWorldKind,
1006    /// The if of the frame to get the `ExecutionContext` for
1007    pub frame_id: Option<FrameId>,
1008    /// Sender half of the channel to send the response back
1009    pub tx: Sender<Option<ExecutionContextId>>,
1010}
1011
1012impl GetExecutionContext {
1013    pub fn new(tx: Sender<Option<ExecutionContextId>>) -> Self {
1014        Self {
1015            dom_world: DOMWorldKind::Main,
1016            frame_id: None,
1017            tx,
1018        }
1019    }
1020}
1021
1022#[derive(Debug)]
1023pub struct GetUrl {
1024    /// The id of the frame to get the url for (None = main frame)
1025    pub frame_id: Option<FrameId>,
1026    /// Sender half of the channel to send the response back
1027    pub tx: Sender<Option<String>>,
1028}
1029
1030impl GetUrl {
1031    pub fn new(tx: Sender<Option<String>>) -> Self {
1032        Self { frame_id: None, tx }
1033    }
1034}
1035
1036#[derive(Debug)]
1037pub struct GetName {
1038    /// The id of the frame to get the name for (None = main frame)
1039    pub frame_id: Option<FrameId>,
1040    /// Sender half of the channel to send the response back
1041    pub tx: Sender<Option<String>>,
1042}
1043
1044#[derive(Debug)]
1045pub struct GetParent {
1046    /// The id of the frame to get the parent for (None = main frame)
1047    pub frame_id: FrameId,
1048    /// Sender half of the channel to send the response back
1049    pub tx: Sender<Option<FrameId>>,
1050}
1051
1052#[derive(Debug)]
1053pub enum TargetMessage {
1054    /// Execute a command within the session of this target
1055    Command(CommandMessage),
1056    /// Return the main frame of this target's page
1057    MainFrame(Sender<Option<FrameId>>),
1058    /// Return all the frames of this target's page
1059    AllFrames(Sender<Vec<FrameId>>),
1060    #[cfg(feature = "_cache")]
1061    /// Set the cache key and policy for the target page.
1062    CacheKey((Option<String>, Option<crate::cache::BasicCachePolicy>)),
1063    /// Return the url if available
1064    Url(GetUrl),
1065    /// Return the name if available
1066    Name(GetName),
1067    /// Return the parent id of a frame
1068    Parent(GetParent),
1069    /// A Message that resolves when the frame finished loading a new url
1070    WaitForNavigation(Sender<ArcHttpRequest>),
1071    /// A Message that resolves when the frame network is idle
1072    WaitForNetworkIdle(Sender<ArcHttpRequest>),
1073    /// A Message that resolves when the frame network is almost idle
1074    WaitForNetworkAlmostIdle(Sender<ArcHttpRequest>),
1075    /// A request to submit a new listener that gets notified with every
1076    /// received event
1077    AddEventListener(EventListenerRequest),
1078    /// Get the `ExecutionContext` if available
1079    GetExecutionContext(GetExecutionContext),
1080    Authenticate(Credentials),
1081    /// Set block/unblocked networking
1082    BlockNetwork(bool),
1083    /// Enable/Disable internal request paused interception
1084    EnableInterception(bool),
1085}
chromiumoxide/handler/target.rs

chromiumoxide/handler/
target.rs
