<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/result/crawler.rs`."><title>crawler.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2" crossorigin href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-46132b98.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="scrapfly_sdk" data-themes="" data-resource-suffix="" data-rustdoc-version="1.85.0 (4d91de4e4 2025-02-17)" data-channel="1.85.0" data-search-js="search-75f5ac3e.js" data-settings-js="settings-0f613d39.js" ><script src="../../../static.files/storage-59e33391.js"></script><script defer src="../../../static.files/src-script-56102188.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5f194d8c.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-893ab5e7.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-6580c154.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer"></div><main><rustdoc-search></rustdoc-search><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">scrapfly_sdk/result/</div>crawler.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap"><div data-nosnippet><pre class="src-line-numbers">
<a href="#1" id="1">1</a>
<a href="#2" id="2">2</a>
<a href="#3" id="3">3</a>
<a href="#4" id="4">4</a>
<a href="#5" id="5">5</a>
<a href="#6" id="6">6</a>
<a href="#7" id="7">7</a>
<a href="#8" id="8">8</a>
<a href="#9" id="9">9</a>
<a href="#10" id="10">10</a>
<a href="#11" id="11">11</a>
<a href="#12" id="12">12</a>
<a href="#13" id="13">13</a>
<a href="#14" id="14">14</a>
<a href="#15" id="15">15</a>
<a href="#16" id="16">16</a>
<a href="#17" id="17">17</a>
<a href="#18" id="18">18</a>
<a href="#19" id="19">19</a>
<a href="#20" id="20">20</a>
<a href="#21" id="21">21</a>
<a href="#22" id="22">22</a>
<a href="#23" id="23">23</a>
<a href="#24" id="24">24</a>
<a href="#25" id="25">25</a>
<a href="#26" id="26">26</a>
<a href="#27" id="27">27</a>
<a href="#28" id="28">28</a>
<a href="#29" id="29">29</a>
<a href="#30" id="30">30</a>
<a href="#31" id="31">31</a>
<a href="#32" id="32">32</a>
<a href="#33" id="33">33</a>
<a href="#34" id="34">34</a>
<a href="#35" id="35">35</a>
<a href="#36" id="36">36</a>
<a href="#37" id="37">37</a>
<a href="#38" id="38">38</a>
<a href="#39" id="39">39</a>
<a href="#40" id="40">40</a>
<a href="#41" id="41">41</a>
<a href="#42" id="42">42</a>
<a href="#43" id="43">43</a>
<a href="#44" id="44">44</a>
<a href="#45" id="45">45</a>
<a href="#46" id="46">46</a>
<a href="#47" id="47">47</a>
<a href="#48" id="48">48</a>
<a href="#49" id="49">49</a>
<a href="#50" id="50">50</a>
<a href="#51" id="51">51</a>
<a href="#52" id="52">52</a>
<a href="#53" id="53">53</a>
<a href="#54" id="54">54</a>
<a href="#55" id="55">55</a>
<a href="#56" id="56">56</a>
<a href="#57" id="57">57</a>
<a href="#58" id="58">58</a>
<a href="#59" id="59">59</a>
<a href="#60" id="60">60</a>
<a href="#61" id="61">61</a>
<a href="#62" id="62">62</a>
<a href="#63" id="63">63</a>
<a href="#64" id="64">64</a>
<a href="#65" id="65">65</a>
<a href="#66" id="66">66</a>
<a href="#67" id="67">67</a>
<a href="#68" id="68">68</a>
<a href="#69" id="69">69</a>
<a href="#70" id="70">70</a>
<a href="#71" id="71">71</a>
<a href="#72" id="72">72</a>
<a href="#73" id="73">73</a>
<a href="#74" id="74">74</a>
<a href="#75" id="75">75</a>
<a href="#76" id="76">76</a>
<a href="#77" id="77">77</a>
<a href="#78" id="78">78</a>
<a href="#79" id="79">79</a>
<a href="#80" id="80">80</a>
<a href="#81" id="81">81</a>
<a href="#82" id="82">82</a>
<a href="#83" id="83">83</a>
<a href="#84" id="84">84</a>
<a href="#85" id="85">85</a>
<a href="#86" id="86">86</a>
<a href="#87" id="87">87</a>
<a href="#88" id="88">88</a>
<a href="#89" id="89">89</a>
<a href="#90" id="90">90</a>
<a href="#91" id="91">91</a>
<a href="#92" id="92">92</a>
<a href="#93" id="93">93</a>
<a href="#94" id="94">94</a>
<a href="#95" id="95">95</a>
<a href="#96" id="96">96</a>
<a href="#97" id="97">97</a>
<a href="#98" id="98">98</a>
<a href="#99" id="99">99</a>
<a href="#100" id="100">100</a>
<a href="#101" id="101">101</a>
<a href="#102" id="102">102</a>
<a href="#103" id="103">103</a>
<a href="#104" id="104">104</a>
<a href="#105" id="105">105</a>
<a href="#106" id="106">106</a>
<a href="#107" id="107">107</a>
<a href="#108" id="108">108</a>
<a href="#109" id="109">109</a>
<a href="#110" id="110">110</a>
<a href="#111" id="111">111</a>
<a href="#112" id="112">112</a>
<a href="#113" id="113">113</a>
<a href="#114" id="114">114</a>
<a href="#115" id="115">115</a>
<a href="#116" id="116">116</a>
<a href="#117" id="117">117</a>
<a href="#118" id="118">118</a>
<a href="#119" id="119">119</a>
<a href="#120" id="120">120</a>
<a href="#121" id="121">121</a>
<a href="#122" id="122">122</a>
<a href="#123" id="123">123</a>
<a href="#124" id="124">124</a>
<a href="#125" id="125">125</a>
<a href="#126" id="126">126</a>
<a href="#127" id="127">127</a>
<a href="#128" id="128">128</a>
<a href="#129" id="129">129</a>
<a href="#130" id="130">130</a>
<a href="#131" id="131">131</a>
<a href="#132" id="132">132</a>
<a href="#133" id="133">133</a>
<a href="#134" id="134">134</a>
<a href="#135" id="135">135</a>
<a href="#136" id="136">136</a>
<a href="#137" id="137">137</a>
<a href="#138" id="138">138</a>
<a href="#139" id="139">139</a>
<a href="#140" id="140">140</a>
<a href="#141" id="141">141</a>
<a href="#142" id="142">142</a>
<a href="#143" id="143">143</a>
<a href="#144" id="144">144</a>
<a href="#145" id="145">145</a>
<a href="#146" id="146">146</a>
<a href="#147" id="147">147</a>
<a href="#148" id="148">148</a>
<a href="#149" id="149">149</a>
<a href="#150" id="150">150</a>
<a href="#151" id="151">151</a>
<a href="#152" id="152">152</a>
<a href="#153" id="153">153</a>
<a href="#154" id="154">154</a>
<a href="#155" id="155">155</a>
<a href="#156" id="156">156</a>
<a href="#157" id="157">157</a>
<a href="#158" id="158">158</a>
<a href="#159" id="159">159</a>
<a href="#160" id="160">160</a>
<a href="#161" id="161">161</a>
<a href="#162" id="162">162</a>
<a href="#163" id="163">163</a>
<a href="#164" id="164">164</a>
<a href="#165" id="165">165</a>
<a href="#166" id="166">166</a>
<a href="#167" id="167">167</a>
<a href="#168" id="168">168</a>
<a href="#169" id="169">169</a>
<a href="#170" id="170">170</a>
<a href="#171" id="171">171</a>
<a href="#172" id="172">172</a>
<a href="#173" id="173">173</a>
<a href="#174" id="174">174</a>
<a href="#175" id="175">175</a>
<a href="#176" id="176">176</a>
<a href="#177" id="177">177</a>
<a href="#178" id="178">178</a>
<a href="#179" id="179">179</a>
<a href="#180" id="180">180</a>
<a href="#181" id="181">181</a>
<a href="#182" id="182">182</a>
<a href="#183" id="183">183</a>
<a href="#184" id="184">184</a>
<a href="#185" id="185">185</a>
<a href="#186" id="186">186</a>
<a href="#187" id="187">187</a>
<a href="#188" id="188">188</a>
<a href="#189" id="189">189</a>
<a href="#190" id="190">190</a>
<a href="#191" id="191">191</a>
<a href="#192" id="192">192</a>
<a href="#193" id="193">193</a>
<a href="#194" id="194">194</a>
<a href="#195" id="195">195</a>
<a href="#196" id="196">196</a>
<a href="#197" id="197">197</a>
<a href="#198" id="198">198</a>
<a href="#199" id="199">199</a>
<a href="#200" id="200">200</a>
<a href="#201" id="201">201</a>
<a href="#202" id="202">202</a>
<a href="#203" id="203">203</a>
<a href="#204" id="204">204</a>
<a href="#205" id="205">205</a>
<a href="#206" id="206">206</a>
<a href="#207" id="207">207</a>
<a href="#208" id="208">208</a>
<a href="#209" id="209">209</a>
<a href="#210" id="210">210</a>
<a href="#211" id="211">211</a>
<a href="#212" id="212">212</a>
<a href="#213" id="213">213</a>
<a href="#214" id="214">214</a>
<a href="#215" id="215">215</a>
<a href="#216" id="216">216</a>
<a href="#217" id="217">217</a>
<a href="#218" id="218">218</a>
<a href="#219" id="219">219</a>
<a href="#220" id="220">220</a>
<a href="#221" id="221">221</a>
<a href="#222" id="222">222</a>
<a href="#223" id="223">223</a>
<a href="#224" id="224">224</a>
<a href="#225" id="225">225</a>
<a href="#226" id="226">226</a>
<a href="#227" id="227">227</a>
<a href="#228" id="228">228</a>
<a href="#229" id="229">229</a>
<a href="#230" id="230">230</a>
<a href="#231" id="231">231</a>
<a href="#232" id="232">232</a>
<a href="#233" id="233">233</a>
<a href="#234" id="234">234</a>
<a href="#235" id="235">235</a>
<a href="#236" id="236">236</a>
<a href="#237" id="237">237</a>
<a href="#238" id="238">238</a>
<a href="#239" id="239">239</a>
<a href="#240" id="240">240</a>
<a href="#241" id="241">241</a>
<a href="#242" id="242">242</a>
<a href="#243" id="243">243</a>
<a href="#244" id="244">244</a>
<a href="#245" id="245">245</a>
<a href="#246" id="246">246</a>
<a href="#247" id="247">247</a>
<a href="#248" id="248">248</a>
<a href="#249" id="249">249</a>
<a href="#250" id="250">250</a>
<a href="#251" id="251">251</a>
<a href="#252" id="252">252</a>
<a href="#253" id="253">253</a>
<a href="#254" id="254">254</a>
<a href="#255" id="255">255</a>
<a href="#256" id="256">256</a>
<a href="#257" id="257">257</a>
<a href="#258" id="258">258</a>
<a href="#259" id="259">259</a>
<a href="#260" id="260">260</a>
<a href="#261" id="261">261</a>
<a href="#262" id="262">262</a>
<a href="#263" id="263">263</a>
<a href="#264" id="264">264</a>
<a href="#265" id="265">265</a>
<a href="#266" id="266">266</a>
<a href="#267" id="267">267</a>
<a href="#268" id="268">268</a>
<a href="#269" id="269">269</a>
<a href="#270" id="270">270</a>
<a href="#271" id="271">271</a>
<a href="#272" id="272">272</a>
<a href="#273" id="273">273</a>
<a href="#274" id="274">274</a>
<a href="#275" id="275">275</a>
<a href="#276" id="276">276</a>
<a href="#277" id="277">277</a>
<a href="#278" id="278">278</a>
<a href="#279" id="279">279</a>
<a href="#280" id="280">280</a>
<a href="#281" id="281">281</a>
<a href="#282" id="282">282</a>
<a href="#283" id="283">283</a>
<a href="#284" id="284">284</a>
<a href="#285" id="285">285</a>
<a href="#286" id="286">286</a></pre></div><pre class="rust"><code><span class="doccomment">//! Crawler result objects — port of `sdk/go/result_crawler.go`.
</span><span class="kw">use </span>std::collections::BTreeMap;
<span class="kw">use </span>bytes::Bytes;
<span class="kw">use </span>serde::Deserialize;
<span class="doccomment">/// Crawler status constants.
</span><span class="kw">pub mod </span>status {
<span class="doccomment">/// Pending — not yet picked up.
</span><span class="kw">pub const </span>PENDING: <span class="kw-2">&</span>str = <span class="string">"PENDING"</span>;
<span class="doccomment">/// Running.
</span><span class="kw">pub const </span>RUNNING: <span class="kw-2">&</span>str = <span class="string">"RUNNING"</span>;
<span class="doccomment">/// Done (check `is_success` for success/failure).
</span><span class="kw">pub const </span>DONE: <span class="kw-2">&</span>str = <span class="string">"DONE"</span>;
<span class="doccomment">/// Cancelled by the user.
</span><span class="kw">pub const </span>CANCELLED: <span class="kw-2">&</span>str = <span class="string">"CANCELLED"</span>;
}
<span class="doccomment">/// Response from `POST /crawl`.
</span><span class="attr">#[derive(Debug, Clone, Deserialize, Default)]
</span><span class="kw">pub struct </span>CrawlerStartResponse {
<span class="doccomment">/// Crawler job UUID.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>crawler_uuid: String,
<span class="doccomment">/// Initial status.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>status: String,
}
<span class="doccomment">/// Inner `state` block of [`CrawlerStatus`].
</span><span class="attr">#[derive(Debug, Clone, Deserialize, Default)]
</span><span class="kw">pub struct </span>CrawlerState {
<span class="doccomment">/// URLs visited.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>urls_visited: u64,
<span class="doccomment">/// URLs extracted.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>urls_extracted: u64,
<span class="doccomment">/// URLs failed.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>urls_failed: u64,
<span class="doccomment">/// URLs skipped.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>urls_skipped: u64,
<span class="doccomment">/// URLs queued.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>urls_to_crawl: u64,
<span class="doccomment">/// API credit used.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>api_credit_used: u64,
<span class="doccomment">/// Duration (seconds).
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>duration: u64,
<span class="doccomment">/// Start time (Unix seconds, null while PENDING).
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>start_time: <span class="prelude-ty">Option</span><i64>,
<span class="doccomment">/// Stop time (Unix seconds, null until terminal).
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>stop_time: <span class="prelude-ty">Option</span><i64>,
<span class="doccomment">/// Documented stop reason (null while running).
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>stop_reason: <span class="prelude-ty">Option</span><String>,
}
<span class="doccomment">/// Response from `GET /crawl/{uuid}/status`.
</span><span class="attr">#[derive(Debug, Clone, Deserialize, Default)]
</span><span class="kw">pub struct </span>CrawlerStatus {
<span class="doccomment">/// Crawler UUID.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>crawler_uuid: String,
<span class="doccomment">/// Status enum (`PENDING`, `RUNNING`, `DONE`, `CANCELLED`).
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>status: String,
<span class="doccomment">/// Whether the crawler reached a terminal state.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>is_finished: bool,
<span class="doccomment">/// Success marker (nullable while running).
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>is_success: <span class="prelude-ty">Option</span><bool>,
<span class="doccomment">/// Per-job metrics.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>state: CrawlerState,
}
<span class="kw">impl </span>CrawlerStatus {
<span class="doccomment">/// True while still pending or running.
</span><span class="kw">pub fn </span>is_running(<span class="kw-2">&</span><span class="self">self</span>) -> bool {
<span class="self">self</span>.status == status::PENDING || <span class="self">self</span>.status == status::RUNNING
}
<span class="doccomment">/// True when terminated successfully.
</span><span class="kw">pub fn </span>is_complete(<span class="kw-2">&</span><span class="self">self</span>) -> bool {
<span class="self">self</span>.status == status::DONE && <span class="self">self</span>.is_success == <span class="prelude-val">Some</span>(<span class="bool-val">true</span>)
}
<span class="doccomment">/// True when terminated with failure.
</span><span class="kw">pub fn </span>is_failed(<span class="kw-2">&</span><span class="self">self</span>) -> bool {
<span class="self">self</span>.status == status::DONE && <span class="self">self</span>.is_success == <span class="prelude-val">Some</span>(<span class="bool-val">false</span>)
}
<span class="doccomment">/// True when cancelled by the user.
</span><span class="kw">pub fn </span>is_cancelled(<span class="kw-2">&</span><span class="self">self</span>) -> bool {
<span class="self">self</span>.status == status::CANCELLED
}
}
<span class="doccomment">/// One entry in the streaming `urls` list.
</span><span class="attr">#[derive(Debug, Clone)]
</span><span class="kw">pub struct </span>CrawlerUrlEntry {
<span class="doccomment">/// URL.
</span><span class="kw">pub </span>url: String,
<span class="doccomment">/// Status (visited/pending/failed/skipped) — echoed from the request.
</span><span class="kw">pub </span>status: String,
<span class="doccomment">/// Reason for failure/skip (only set for `failed`/`skipped`).
</span><span class="kw">pub </span>reason: String,
}
<span class="doccomment">/// Streaming response from `GET /crawl/{uuid}/urls`.
</span><span class="attr">#[derive(Debug, Clone, Default)]
</span><span class="kw">pub struct </span>CrawlerUrls {
<span class="doccomment">/// URL entries on this page.
</span><span class="kw">pub </span>urls: Vec<CrawlerUrlEntry>,
<span class="doccomment">/// Page number.
</span><span class="kw">pub </span>page: u32,
<span class="doccomment">/// Page size.
</span><span class="kw">pub </span>per_page: u32,
}
<span class="kw">impl </span>CrawlerUrls {
<span class="doccomment">/// Parse a `text/plain` body into a [`CrawlerUrls`]. Mirrors
/// `sdk/go/result_crawler.go::parseCrawlerURLs`.
</span><span class="kw">pub fn </span>from_text(body: <span class="kw-2">&</span>str, status_hint: <span class="kw-2">&</span>str, page: u32, per_page: u32) -> <span class="self">Self </span>{
<span class="kw">let </span><span class="kw-2">mut </span>urls = Vec::new();
<span class="kw">for </span>raw_line <span class="kw">in </span>body.split(<span class="string">'\n'</span>) {
<span class="kw">let </span>line = raw_line.trim();
<span class="kw">if </span>line.is_empty() {
<span class="kw">continue</span>;
}
<span class="kw">if </span>status_hint == <span class="string">"visited" </span>|| status_hint == <span class="string">"pending" </span>{
urls.push(CrawlerUrlEntry {
url: line.to_string(),
status: status_hint.to_string(),
reason: String::new(),
});
<span class="kw">continue</span>;
}
<span class="kw">if let </span><span class="prelude-val">Some</span>(idx) = line.find(<span class="string">','</span>) {
urls.push(CrawlerUrlEntry {
url: line[..idx].to_string(),
status: status_hint.to_string(),
reason: line[idx + <span class="number">1</span>..].to_string(),
});
} <span class="kw">else </span>{
urls.push(CrawlerUrlEntry {
url: line.to_string(),
status: status_hint.to_string(),
reason: String::new(),
});
}
}
<span class="self">Self </span>{
urls,
page,
per_page,
}
}
}
<span class="doccomment">/// `GET /crawl/{uuid}/contents` bulk-JSON envelope.
</span><span class="attr">#[derive(Debug, Clone, Deserialize, Default)]
</span><span class="kw">pub struct </span>CrawlerContents {
<span class="doccomment">/// `url → format → content`. The API can emit `null` for a format that
/// couldn't be produced for a given URL (e.g. `extracted_data` on a page
/// that no template matched); the SDK flattens `null → ""` so consumers
/// always get a string and can check emptiness. Mirrors Go's map[string]string
/// zero-value semantics.
</span><span class="attr">#[serde(default, deserialize_with = <span class="string">"deserialize_contents_map"</span>)]
</span><span class="kw">pub </span>contents: BTreeMap<String, BTreeMap<String, String>>,
<span class="doccomment">/// Pagination links.
</span><span class="attr">#[serde(default)]
</span><span class="kw">pub </span>links: CrawlerContentsLinks,
}
<span class="doccomment">/// Deserialize `{url: {format: string|null}}` tolerating `null` inner values
/// by mapping them to the empty string.
</span><span class="kw">fn </span>deserialize_contents_map<<span class="lifetime">'de</span>, D>(
deserializer: D,
) -> <span class="prelude-ty">Result</span><BTreeMap<String, BTreeMap<String, String>>, D::Error>
<span class="kw">where
</span>D: serde::Deserializer<<span class="lifetime">'de</span>>,
{
<span class="kw">let </span>raw: BTreeMap<String, BTreeMap<String, <span class="prelude-ty">Option</span><String>>> =
BTreeMap::deserialize(deserializer)<span class="question-mark">?</span>;
<span class="prelude-val">Ok</span>(raw
.into_iter()
.map(|(url, by_format)| {
(
url,
by_format
.into_iter()
.map(|(fmt, body)| (fmt, body.unwrap_or_default()))
.collect(),
)
})
.collect())
}
<span class="doccomment">/// Pagination links returned with bulk contents.
///
/// `next`/`prev` arrive as JSON `null` when there is no adjacent page, which
/// would reject under a plain `String` field; [`null_as_empty_string`] maps
/// both null and absent to the empty string so the public API stays typed.
</span><span class="attr">#[derive(Debug, Clone, Deserialize, Default)]
</span><span class="kw">pub struct </span>CrawlerContentsLinks {
<span class="doccomment">/// Crawled URLs link.
</span><span class="attr">#[serde(default, deserialize_with = <span class="string">"null_as_empty_string"</span>)]
</span><span class="kw">pub </span>crawled_urls: String,
<span class="doccomment">/// Next-page link (empty when on the last page).
</span><span class="attr">#[serde(default, deserialize_with = <span class="string">"null_as_empty_string"</span>)]
</span><span class="kw">pub </span>next: String,
<span class="doccomment">/// Previous-page link (empty when on the first page).
</span><span class="attr">#[serde(default, deserialize_with = <span class="string">"null_as_empty_string"</span>)]
</span><span class="kw">pub </span>prev: String,
}
<span class="doccomment">/// Coerce JSON `null | absent | string` into a plain `String`, where null
/// and absent both collapse to the empty string. Mirrors Go's `string`
/// zero-value behavior under `encoding/json`.
</span><span class="kw">fn </span>null_as_empty_string<<span class="lifetime">'de</span>, D>(deserializer: D) -> <span class="prelude-ty">Result</span><String, D::Error>
<span class="kw">where
</span>D: serde::Deserializer<<span class="lifetime">'de</span>>,
{
<span class="prelude-val">Ok</span>(<span class="prelude-ty">Option</span>::<String>::deserialize(deserializer)<span class="question-mark">?</span>.unwrap_or_default())
}
<span class="doccomment">/// Typed content wrapper for a single crawled URL (`Crawl::read`).
</span><span class="attr">#[derive(Debug, Clone, Default)]
</span><span class="kw">pub struct </span>CrawlContent {
<span class="doccomment">/// URL.
</span><span class="kw">pub </span>url: String,
<span class="doccomment">/// Content in the requested format.
</span><span class="kw">pub </span>content: String,
<span class="doccomment">/// Parent crawler UUID.
</span><span class="kw">pub </span>crawl_uuid: String,
}
<span class="doccomment">/// Artifact type — `warc` or `har`.
</span><span class="attr">#[derive(Debug, Clone, Copy, PartialEq, Eq)]
</span><span class="kw">pub enum </span>CrawlerArtifactType {
<span class="doccomment">/// WARC artifact.
</span>Warc,
<span class="doccomment">/// HAR artifact.
</span>Har,
}
<span class="kw">impl </span>CrawlerArtifactType {
<span class="doccomment">/// Wire-format string.
</span><span class="kw">pub fn </span>as_str(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="kw-2">&</span><span class="lifetime">'static </span>str {
<span class="kw">match </span><span class="self">self </span>{
<span class="self">Self</span>::Warc => <span class="string">"warc"</span>,
<span class="self">Self</span>::Har => <span class="string">"har"</span>,
}
}
}
<span class="doccomment">/// WARC or HAR artifact downloaded from the crawler endpoint.
</span><span class="attr">#[derive(Debug, Clone)]
</span><span class="kw">pub struct </span>CrawlerArtifact {
<span class="doccomment">/// Artifact type.
</span><span class="kw">pub </span>artifact_type: CrawlerArtifactType,
<span class="doccomment">/// Raw bytes.
</span><span class="kw">pub </span>data: Bytes,
}
<span class="kw">impl </span>CrawlerArtifact {
<span class="doccomment">/// Write the artifact to disk.
</span><span class="kw">pub fn </span>save(<span class="kw-2">&</span><span class="self">self</span>, path: <span class="kw-2">&</span>std::path::Path) -> std::io::Result<()> {
std::fs::write(path, <span class="kw-2">&</span><span class="self">self</span>.data)
}
<span class="doccomment">/// Byte length of the artifact.
</span><span class="kw">pub fn </span>len(<span class="kw-2">&</span><span class="self">self</span>) -> usize {
<span class="self">self</span>.data.len()
}
<span class="doccomment">/// True when empty.
</span><span class="kw">pub fn </span>is_empty(<span class="kw-2">&</span><span class="self">self</span>) -> bool {
<span class="self">self</span>.data.is_empty()
}
}
</code></pre></div></section></main></body></html>