<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/client.rs`."><title>client.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2" crossorigin href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-46132b98.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="scrapfly_sdk" data-themes="" data-resource-suffix="" data-rustdoc-version="1.85.0 (4d91de4e4 2025-02-17)" data-channel="1.85.0" data-search-js="search-75f5ac3e.js" data-settings-js="settings-0f613d39.js" ><script src="../../static.files/storage-59e33391.js"></script><script defer src="../../static.files/src-script-56102188.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5f194d8c.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-893ab5e7.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-6580c154.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer"></div><main><rustdoc-search></rustdoc-search><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">scrapfly_sdk/</div>client.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap"><div data-nosnippet><pre class="src-line-numbers">
<a href="#1" id="1">1</a>
<a href="#2" id="2">2</a>
<a href="#3" id="3">3</a>
<a href="#4" id="4">4</a>
<a href="#5" id="5">5</a>
<a href="#6" id="6">6</a>
<a href="#7" id="7">7</a>
<a href="#8" id="8">8</a>
<a href="#9" id="9">9</a>
<a href="#10" id="10">10</a>
<a href="#11" id="11">11</a>
<a href="#12" id="12">12</a>
<a href="#13" id="13">13</a>
<a href="#14" id="14">14</a>
<a href="#15" id="15">15</a>
<a href="#16" id="16">16</a>
<a href="#17" id="17">17</a>
<a href="#18" id="18">18</a>
<a href="#19" id="19">19</a>
<a href="#20" id="20">20</a>
<a href="#21" id="21">21</a>
<a href="#22" id="22">22</a>
<a href="#23" id="23">23</a>
<a href="#24" id="24">24</a>
<a href="#25" id="25">25</a>
<a href="#26" id="26">26</a>
<a href="#27" id="27">27</a>
<a href="#28" id="28">28</a>
<a href="#29" id="29">29</a>
<a href="#30" id="30">30</a>
<a href="#31" id="31">31</a>
<a href="#32" id="32">32</a>
<a href="#33" id="33">33</a>
<a href="#34" id="34">34</a>
<a href="#35" id="35">35</a>
<a href="#36" id="36">36</a>
<a href="#37" id="37">37</a>
<a href="#38" id="38">38</a>
<a href="#39" id="39">39</a>
<a href="#40" id="40">40</a>
<a href="#41" id="41">41</a>
<a href="#42" id="42">42</a>
<a href="#43" id="43">43</a>
<a href="#44" id="44">44</a>
<a href="#45" id="45">45</a>
<a href="#46" id="46">46</a>
<a href="#47" id="47">47</a>
<a href="#48" id="48">48</a>
<a href="#49" id="49">49</a>
<a href="#50" id="50">50</a>
<a href="#51" id="51">51</a>
<a href="#52" id="52">52</a>
<a href="#53" id="53">53</a>
<a href="#54" id="54">54</a>
<a href="#55" id="55">55</a>
<a href="#56" id="56">56</a>
<a href="#57" id="57">57</a>
<a href="#58" id="58">58</a>
<a href="#59" id="59">59</a>
<a href="#60" id="60">60</a>
<a href="#61" id="61">61</a>
<a href="#62" id="62">62</a>
<a href="#63" id="63">63</a>
<a href="#64" id="64">64</a>
<a href="#65" id="65">65</a>
<a href="#66" id="66">66</a>
<a href="#67" id="67">67</a>
<a href="#68" id="68">68</a>
<a href="#69" id="69">69</a>
<a href="#70" id="70">70</a>
<a href="#71" id="71">71</a>
<a href="#72" id="72">72</a>
<a href="#73" id="73">73</a>
<a href="#74" id="74">74</a>
<a href="#75" id="75">75</a>
<a href="#76" id="76">76</a>
<a href="#77" id="77">77</a>
<a href="#78" id="78">78</a>
<a href="#79" id="79">79</a>
<a href="#80" id="80">80</a>
<a href="#81" id="81">81</a>
<a href="#82" id="82">82</a>
<a href="#83" id="83">83</a>
<a href="#84" id="84">84</a>
<a href="#85" id="85">85</a>
<a href="#86" id="86">86</a>
<a href="#87" id="87">87</a>
<a href="#88" id="88">88</a>
<a href="#89" id="89">89</a>
<a href="#90" id="90">90</a>
<a href="#91" id="91">91</a>
<a href="#92" id="92">92</a>
<a href="#93" id="93">93</a>
<a href="#94" id="94">94</a>
<a href="#95" id="95">95</a>
<a href="#96" id="96">96</a>
<a href="#97" id="97">97</a>
<a href="#98" id="98">98</a>
<a href="#99" id="99">99</a>
<a href="#100" id="100">100</a>
<a href="#101" id="101">101</a>
<a href="#102" id="102">102</a>
<a href="#103" id="103">103</a>
<a href="#104" id="104">104</a>
<a href="#105" id="105">105</a>
<a href="#106" id="106">106</a>
<a href="#107" id="107">107</a>
<a href="#108" id="108">108</a>
<a href="#109" id="109">109</a>
<a href="#110" id="110">110</a>
<a href="#111" id="111">111</a>
<a href="#112" id="112">112</a>
<a href="#113" id="113">113</a>
<a href="#114" id="114">114</a>
<a href="#115" id="115">115</a>
<a href="#116" id="116">116</a>
<a href="#117" id="117">117</a>
<a href="#118" id="118">118</a>
<a href="#119" id="119">119</a>
<a href="#120" id="120">120</a>
<a href="#121" id="121">121</a>
<a href="#122" id="122">122</a>
<a href="#123" id="123">123</a>
<a href="#124" id="124">124</a>
<a href="#125" id="125">125</a>
<a href="#126" id="126">126</a>
<a href="#127" id="127">127</a>
<a href="#128" id="128">128</a>
<a href="#129" id="129">129</a>
<a href="#130" id="130">130</a>
<a href="#131" id="131">131</a>
<a href="#132" id="132">132</a>
<a href="#133" id="133">133</a>
<a href="#134" id="134">134</a>
<a href="#135" id="135">135</a>
<a href="#136" id="136">136</a>
<a href="#137" id="137">137</a>
<a href="#138" id="138">138</a>
<a href="#139" id="139">139</a>
<a href="#140" id="140">140</a>
<a href="#141" id="141">141</a>
<a href="#142" id="142">142</a>
<a href="#143" id="143">143</a>
<a href="#144" id="144">144</a>
<a href="#145" id="145">145</a>
<a href="#146" id="146">146</a>
<a href="#147" id="147">147</a>
<a href="#148" id="148">148</a>
<a href="#149" id="149">149</a>
<a href="#150" id="150">150</a>
<a href="#151" id="151">151</a>
<a href="#152" id="152">152</a>
<a href="#153" id="153">153</a>
<a href="#154" id="154">154</a>
<a href="#155" id="155">155</a>
<a href="#156" id="156">156</a>
<a href="#157" id="157">157</a>
<a href="#158" id="158">158</a>
<a href="#159" id="159">159</a>
<a href="#160" id="160">160</a>
<a href="#161" id="161">161</a>
<a href="#162" id="162">162</a>
<a href="#163" id="163">163</a>
<a href="#164" id="164">164</a>
<a href="#165" id="165">165</a>
<a href="#166" id="166">166</a>
<a href="#167" id="167">167</a>
<a href="#168" id="168">168</a>
<a href="#169" id="169">169</a>
<a href="#170" id="170">170</a>
<a href="#171" id="171">171</a>
<a href="#172" id="172">172</a>
<a href="#173" id="173">173</a>
<a href="#174" id="174">174</a>
<a href="#175" id="175">175</a>
<a href="#176" id="176">176</a>
<a href="#177" id="177">177</a>
<a href="#178" id="178">178</a>
<a href="#179" id="179">179</a>
<a href="#180" id="180">180</a>
<a href="#181" id="181">181</a>
<a href="#182" id="182">182</a>
<a href="#183" id="183">183</a>
<a href="#184" id="184">184</a>
<a href="#185" id="185">185</a>
<a href="#186" id="186">186</a>
<a href="#187" id="187">187</a>
<a href="#188" id="188">188</a>
<a href="#189" id="189">189</a>
<a href="#190" id="190">190</a>
<a href="#191" id="191">191</a>
<a href="#192" id="192">192</a>
<a href="#193" id="193">193</a>
<a href="#194" id="194">194</a>
<a href="#195" id="195">195</a>
<a href="#196" id="196">196</a>
<a href="#197" id="197">197</a>
<a href="#198" id="198">198</a>
<a href="#199" id="199">199</a>
<a href="#200" id="200">200</a>
<a href="#201" id="201">201</a>
<a href="#202" id="202">202</a>
<a href="#203" id="203">203</a>
<a href="#204" id="204">204</a>
<a href="#205" id="205">205</a>
<a href="#206" id="206">206</a>
<a href="#207" id="207">207</a>
<a href="#208" id="208">208</a>
<a href="#209" id="209">209</a>
<a href="#210" id="210">210</a>
<a href="#211" id="211">211</a>
<a href="#212" id="212">212</a>
<a href="#213" id="213">213</a>
<a href="#214" id="214">214</a>
<a href="#215" id="215">215</a>
<a href="#216" id="216">216</a>
<a href="#217" id="217">217</a>
<a href="#218" id="218">218</a>
<a href="#219" id="219">219</a>
<a href="#220" id="220">220</a>
<a href="#221" id="221">221</a>
<a href="#222" id="222">222</a>
<a href="#223" id="223">223</a>
<a href="#224" id="224">224</a>
<a href="#225" id="225">225</a>
<a href="#226" id="226">226</a>
<a href="#227" id="227">227</a>
<a href="#228" id="228">228</a>
<a href="#229" id="229">229</a>
<a href="#230" id="230">230</a>
<a href="#231" id="231">231</a>
<a href="#232" id="232">232</a>
<a href="#233" id="233">233</a>
<a href="#234" id="234">234</a>
<a href="#235" id="235">235</a>
<a href="#236" id="236">236</a>
<a href="#237" id="237">237</a>
<a href="#238" id="238">238</a>
<a href="#239" id="239">239</a>
<a href="#240" id="240">240</a>
<a href="#241" id="241">241</a>
<a href="#242" id="242">242</a>
<a href="#243" id="243">243</a>
<a href="#244" id="244">244</a>
<a href="#245" id="245">245</a>
<a href="#246" id="246">246</a>
<a href="#247" id="247">247</a>
<a href="#248" id="248">248</a>
<a href="#249" id="249">249</a>
<a href="#250" id="250">250</a>
<a href="#251" id="251">251</a>
<a href="#252" id="252">252</a>
<a href="#253" id="253">253</a>
<a href="#254" id="254">254</a>
<a href="#255" id="255">255</a>
<a href="#256" id="256">256</a>
<a href="#257" id="257">257</a>
<a href="#258" id="258">258</a>
<a href="#259" id="259">259</a>
<a href="#260" id="260">260</a>
<a href="#261" id="261">261</a>
<a href="#262" id="262">262</a>
<a href="#263" id="263">263</a>
<a href="#264" id="264">264</a>
<a href="#265" id="265">265</a>
<a href="#266" id="266">266</a>
<a href="#267" id="267">267</a>
<a href="#268" id="268">268</a>
<a href="#269" id="269">269</a>
<a href="#270" id="270">270</a>
<a href="#271" id="271">271</a>
<a href="#272" id="272">272</a>
<a href="#273" id="273">273</a>
<a href="#274" id="274">274</a>
<a href="#275" id="275">275</a>
<a href="#276" id="276">276</a>
<a href="#277" id="277">277</a>
<a href="#278" id="278">278</a>
<a href="#279" id="279">279</a>
<a href="#280" id="280">280</a>
<a href="#281" id="281">281</a>
<a href="#282" id="282">282</a>
<a href="#283" id="283">283</a>
<a href="#284" id="284">284</a>
<a href="#285" id="285">285</a>
<a href="#286" id="286">286</a>
<a href="#287" id="287">287</a>
<a href="#288" id="288">288</a>
<a href="#289" id="289">289</a>
<a href="#290" id="290">290</a>
<a href="#291" id="291">291</a>
<a href="#292" id="292">292</a>
<a href="#293" id="293">293</a>
<a href="#294" id="294">294</a>
<a href="#295" id="295">295</a>
<a href="#296" id="296">296</a>
<a href="#297" id="297">297</a>
<a href="#298" id="298">298</a>
<a href="#299" id="299">299</a>
<a href="#300" id="300">300</a>
<a href="#301" id="301">301</a>
<a href="#302" id="302">302</a>
<a href="#303" id="303">303</a>
<a href="#304" id="304">304</a>
<a href="#305" id="305">305</a>
<a href="#306" id="306">306</a>
<a href="#307" id="307">307</a>
<a href="#308" id="308">308</a>
<a href="#309" id="309">309</a>
<a href="#310" id="310">310</a>
<a href="#311" id="311">311</a>
<a href="#312" id="312">312</a>
<a href="#313" id="313">313</a>
<a href="#314" id="314">314</a>
<a href="#315" id="315">315</a>
<a href="#316" id="316">316</a>
<a href="#317" id="317">317</a>
<a href="#318" id="318">318</a>
<a href="#319" id="319">319</a>
<a href="#320" id="320">320</a>
<a href="#321" id="321">321</a>
<a href="#322" id="322">322</a>
<a href="#323" id="323">323</a>
<a href="#324" id="324">324</a>
<a href="#325" id="325">325</a>
<a href="#326" id="326">326</a>
<a href="#327" id="327">327</a>
<a href="#328" id="328">328</a>
<a href="#329" id="329">329</a>
<a href="#330" id="330">330</a>
<a href="#331" id="331">331</a>
<a href="#332" id="332">332</a>
<a href="#333" id="333">333</a>
<a href="#334" id="334">334</a>
<a href="#335" id="335">335</a>
<a href="#336" id="336">336</a>
<a href="#337" id="337">337</a>
<a href="#338" id="338">338</a>
<a href="#339" id="339">339</a>
<a href="#340" id="340">340</a>
<a href="#341" id="341">341</a>
<a href="#342" id="342">342</a>
<a href="#343" id="343">343</a>
<a href="#344" id="344">344</a>
<a href="#345" id="345">345</a>
<a href="#346" id="346">346</a>
<a href="#347" id="347">347</a>
<a href="#348" id="348">348</a>
<a href="#349" id="349">349</a>
<a href="#350" id="350">350</a>
<a href="#351" id="351">351</a>
<a href="#352" id="352">352</a>
<a href="#353" id="353">353</a>
<a href="#354" id="354">354</a>
<a href="#355" id="355">355</a>
<a href="#356" id="356">356</a>
<a href="#357" id="357">357</a>
<a href="#358" id="358">358</a>
<a href="#359" id="359">359</a>
<a href="#360" id="360">360</a>
<a href="#361" id="361">361</a>
<a href="#362" id="362">362</a>
<a href="#363" id="363">363</a>
<a href="#364" id="364">364</a>
<a href="#365" id="365">365</a>
<a href="#366" id="366">366</a>
<a href="#367" id="367">367</a>
<a href="#368" id="368">368</a>
<a href="#369" id="369">369</a>
<a href="#370" id="370">370</a>
<a href="#371" id="371">371</a>
<a href="#372" id="372">372</a>
<a href="#373" id="373">373</a>
<a href="#374" id="374">374</a>
<a href="#375" id="375">375</a>
<a href="#376" id="376">376</a>
<a href="#377" id="377">377</a>
<a href="#378" id="378">378</a>
<a href="#379" id="379">379</a>
<a href="#380" id="380">380</a>
<a href="#381" id="381">381</a>
<a href="#382" id="382">382</a>
<a href="#383" id="383">383</a>
<a href="#384" id="384">384</a>
<a href="#385" id="385">385</a>
<a href="#386" id="386">386</a>
<a href="#387" id="387">387</a>
<a href="#388" id="388">388</a>
<a href="#389" id="389">389</a>
<a href="#390" id="390">390</a>
<a href="#391" id="391">391</a>
<a href="#392" id="392">392</a>
<a href="#393" id="393">393</a>
<a href="#394" id="394">394</a>
<a href="#395" id="395">395</a>
<a href="#396" id="396">396</a>
<a href="#397" id="397">397</a>
<a href="#398" id="398">398</a>
<a href="#399" id="399">399</a>
<a href="#400" id="400">400</a>
<a href="#401" id="401">401</a>
<a href="#402" id="402">402</a>
<a href="#403" id="403">403</a>
<a href="#404" id="404">404</a>
<a href="#405" id="405">405</a>
<a href="#406" id="406">406</a>
<a href="#407" id="407">407</a>
<a href="#408" id="408">408</a>
<a href="#409" id="409">409</a>
<a href="#410" id="410">410</a>
<a href="#411" id="411">411</a>
<a href="#412" id="412">412</a>
<a href="#413" id="413">413</a>
<a href="#414" id="414">414</a>
<a href="#415" id="415">415</a>
<a href="#416" id="416">416</a>
<a href="#417" id="417">417</a>
<a href="#418" id="418">418</a>
<a href="#419" id="419">419</a>
<a href="#420" id="420">420</a>
<a href="#421" id="421">421</a>
<a href="#422" id="422">422</a>
<a href="#423" id="423">423</a>
<a href="#424" id="424">424</a>
<a href="#425" id="425">425</a>
<a href="#426" id="426">426</a>
<a href="#427" id="427">427</a>
<a href="#428" id="428">428</a>
<a href="#429" id="429">429</a>
<a href="#430" id="430">430</a>
<a href="#431" id="431">431</a>
<a href="#432" id="432">432</a>
<a href="#433" id="433">433</a>
<a href="#434" id="434">434</a>
<a href="#435" id="435">435</a>
<a href="#436" id="436">436</a>
<a href="#437" id="437">437</a>
<a href="#438" id="438">438</a>
<a href="#439" id="439">439</a>
<a href="#440" id="440">440</a>
<a href="#441" id="441">441</a>
<a href="#442" id="442">442</a>
<a href="#443" id="443">443</a>
<a href="#444" id="444">444</a>
<a href="#445" id="445">445</a>
<a href="#446" id="446">446</a>
<a href="#447" id="447">447</a>
<a href="#448" id="448">448</a>
<a href="#449" id="449">449</a>
<a href="#450" id="450">450</a>
<a href="#451" id="451">451</a>
<a href="#452" id="452">452</a>
<a href="#453" id="453">453</a>
<a href="#454" id="454">454</a>
<a href="#455" id="455">455</a>
<a href="#456" id="456">456</a>
<a href="#457" id="457">457</a>
<a href="#458" id="458">458</a>
<a href="#459" id="459">459</a>
<a href="#460" id="460">460</a>
<a href="#461" id="461">461</a>
<a href="#462" id="462">462</a>
<a href="#463" id="463">463</a>
<a href="#464" id="464">464</a>
<a href="#465" id="465">465</a>
<a href="#466" id="466">466</a>
<a href="#467" id="467">467</a>
<a href="#468" id="468">468</a>
<a href="#469" id="469">469</a>
<a href="#470" id="470">470</a>
<a href="#471" id="471">471</a>
<a href="#472" id="472">472</a>
<a href="#473" id="473">473</a>
<a href="#474" id="474">474</a>
<a href="#475" id="475">475</a>
<a href="#476" id="476">476</a>
<a href="#477" id="477">477</a>
<a href="#478" id="478">478</a>
<a href="#479" id="479">479</a>
<a href="#480" id="480">480</a>
<a href="#481" id="481">481</a>
<a href="#482" id="482">482</a>
<a href="#483" id="483">483</a>
<a href="#484" id="484">484</a>
<a href="#485" id="485">485</a>
<a href="#486" id="486">486</a>
<a href="#487" id="487">487</a>
<a href="#488" id="488">488</a>
<a href="#489" id="489">489</a>
<a href="#490" id="490">490</a>
<a href="#491" id="491">491</a>
<a href="#492" id="492">492</a>
<a href="#493" id="493">493</a>
<a href="#494" id="494">494</a>
<a href="#495" id="495">495</a>
<a href="#496" id="496">496</a>
<a href="#497" id="497">497</a>
<a href="#498" id="498">498</a>
<a href="#499" id="499">499</a>
<a href="#500" id="500">500</a>
<a href="#501" id="501">501</a>
<a href="#502" id="502">502</a>
<a href="#503" id="503">503</a>
<a href="#504" id="504">504</a>
<a href="#505" id="505">505</a>
<a href="#506" id="506">506</a>
<a href="#507" id="507">507</a>
<a href="#508" id="508">508</a>
<a href="#509" id="509">509</a>
<a href="#510" id="510">510</a>
<a href="#511" id="511">511</a>
<a href="#512" id="512">512</a>
<a href="#513" id="513">513</a>
<a href="#514" id="514">514</a>
<a href="#515" id="515">515</a>
<a href="#516" id="516">516</a>
<a href="#517" id="517">517</a>
<a href="#518" id="518">518</a>
<a href="#519" id="519">519</a>
<a href="#520" id="520">520</a>
<a href="#521" id="521">521</a>
<a href="#522" id="522">522</a>
<a href="#523" id="523">523</a>
<a href="#524" id="524">524</a>
<a href="#525" id="525">525</a>
<a href="#526" id="526">526</a>
<a href="#527" id="527">527</a>
<a href="#528" id="528">528</a>
<a href="#529" id="529">529</a>
<a href="#530" id="530">530</a>
<a href="#531" id="531">531</a>
<a href="#532" id="532">532</a>
<a href="#533" id="533">533</a>
<a href="#534" id="534">534</a>
<a href="#535" id="535">535</a>
<a href="#536" id="536">536</a>
<a href="#537" id="537">537</a>
<a href="#538" id="538">538</a>
<a href="#539" id="539">539</a>
<a href="#540" id="540">540</a>
<a href="#541" id="541">541</a>
<a href="#542" id="542">542</a>
<a href="#543" id="543">543</a>
<a href="#544" id="544">544</a>
<a href="#545" id="545">545</a>
<a href="#546" id="546">546</a>
<a href="#547" id="547">547</a>
<a href="#548" id="548">548</a>
<a href="#549" id="549">549</a>
<a href="#550" id="550">550</a>
<a href="#551" id="551">551</a>
<a href="#552" id="552">552</a>
<a href="#553" id="553">553</a>
<a href="#554" id="554">554</a>
<a href="#555" id="555">555</a>
<a href="#556" id="556">556</a>
<a href="#557" id="557">557</a>
<a href="#558" id="558">558</a>
<a href="#559" id="559">559</a>
<a href="#560" id="560">560</a>
<a href="#561" id="561">561</a>
<a href="#562" id="562">562</a>
<a href="#563" id="563">563</a>
<a href="#564" id="564">564</a>
<a href="#565" id="565">565</a>
<a href="#566" id="566">566</a>
<a href="#567" id="567">567</a>
<a href="#568" id="568">568</a>
<a href="#569" id="569">569</a>
<a href="#570" id="570">570</a>
<a href="#571" id="571">571</a>
<a href="#572" id="572">572</a>
<a href="#573" id="573">573</a>
<a href="#574" id="574">574</a>
<a href="#575" id="575">575</a>
<a href="#576" id="576">576</a>
<a href="#577" id="577">577</a>
<a href="#578" id="578">578</a>
<a href="#579" id="579">579</a>
<a href="#580" id="580">580</a>
<a href="#581" id="581">581</a>
<a href="#582" id="582">582</a>
<a href="#583" id="583">583</a>
<a href="#584" id="584">584</a>
<a href="#585" id="585">585</a>
<a href="#586" id="586">586</a>
<a href="#587" id="587">587</a>
<a href="#588" id="588">588</a>
<a href="#589" id="589">589</a>
<a href="#590" id="590">590</a>
<a href="#591" id="591">591</a>
<a href="#592" id="592">592</a>
<a href="#593" id="593">593</a>
<a href="#594" id="594">594</a>
<a href="#595" id="595">595</a>
<a href="#596" id="596">596</a>
<a href="#597" id="597">597</a>
<a href="#598" id="598">598</a>
<a href="#599" id="599">599</a>
<a href="#600" id="600">600</a>
<a href="#601" id="601">601</a>
<a href="#602" id="602">602</a>
<a href="#603" id="603">603</a>
<a href="#604" id="604">604</a>
<a href="#605" id="605">605</a>
<a href="#606" id="606">606</a>
<a href="#607" id="607">607</a>
<a href="#608" id="608">608</a>
<a href="#609" id="609">609</a>
<a href="#610" id="610">610</a>
<a href="#611" id="611">611</a>
<a href="#612" id="612">612</a>
<a href="#613" id="613">613</a>
<a href="#614" id="614">614</a>
<a href="#615" id="615">615</a>
<a href="#616" id="616">616</a>
<a href="#617" id="617">617</a>
<a href="#618" id="618">618</a>
<a href="#619" id="619">619</a>
<a href="#620" id="620">620</a>
<a href="#621" id="621">621</a>
<a href="#622" id="622">622</a>
<a href="#623" id="623">623</a>
<a href="#624" id="624">624</a>
<a href="#625" id="625">625</a>
<a href="#626" id="626">626</a>
<a href="#627" id="627">627</a>
<a href="#628" id="628">628</a>
<a href="#629" id="629">629</a>
<a href="#630" id="630">630</a>
<a href="#631" id="631">631</a>
<a href="#632" id="632">632</a>
<a href="#633" id="633">633</a>
<a href="#634" id="634">634</a>
<a href="#635" id="635">635</a>
<a href="#636" id="636">636</a>
<a href="#637" id="637">637</a>
<a href="#638" id="638">638</a>
<a href="#639" id="639">639</a>
<a href="#640" id="640">640</a>
<a href="#641" id="641">641</a>
<a href="#642" id="642">642</a>
<a href="#643" id="643">643</a>
<a href="#644" id="644">644</a>
<a href="#645" id="645">645</a>
<a href="#646" id="646">646</a>
<a href="#647" id="647">647</a>
<a href="#648" id="648">648</a>
<a href="#649" id="649">649</a>
<a href="#650" id="650">650</a>
<a href="#651" id="651">651</a>
<a href="#652" id="652">652</a>
<a href="#653" id="653">653</a>
<a href="#654" id="654">654</a>
<a href="#655" id="655">655</a>
<a href="#656" id="656">656</a>
<a href="#657" id="657">657</a>
<a href="#658" id="658">658</a>
<a href="#659" id="659">659</a>
<a href="#660" id="660">660</a>
<a href="#661" id="661">661</a>
<a href="#662" id="662">662</a>
<a href="#663" id="663">663</a>
<a href="#664" id="664">664</a>
<a href="#665" id="665">665</a>
<a href="#666" id="666">666</a>
<a href="#667" id="667">667</a>
<a href="#668" id="668">668</a>
<a href="#669" id="669">669</a>
<a href="#670" id="670">670</a>
<a href="#671" id="671">671</a>
<a href="#672" id="672">672</a>
<a href="#673" id="673">673</a>
<a href="#674" id="674">674</a>
<a href="#675" id="675">675</a>
<a href="#676" id="676">676</a>
<a href="#677" id="677">677</a>
<a href="#678" id="678">678</a>
<a href="#679" id="679">679</a>
<a href="#680" id="680">680</a>
<a href="#681" id="681">681</a>
<a href="#682" id="682">682</a>
<a href="#683" id="683">683</a>
<a href="#684" id="684">684</a>
<a href="#685" id="685">685</a>
<a href="#686" id="686">686</a>
<a href="#687" id="687">687</a>
<a href="#688" id="688">688</a>
<a href="#689" id="689">689</a>
<a href="#690" id="690">690</a>
<a href="#691" id="691">691</a>
<a href="#692" id="692">692</a>
<a href="#693" id="693">693</a>
<a href="#694" id="694">694</a>
<a href="#695" id="695">695</a>
<a href="#696" id="696">696</a>
<a href="#697" id="697">697</a>
<a href="#698" id="698">698</a>
<a href="#699" id="699">699</a>
<a href="#700" id="700">700</a>
<a href="#701" id="701">701</a>
<a href="#702" id="702">702</a>
<a href="#703" id="703">703</a>
<a href="#704" id="704">704</a>
<a href="#705" id="705">705</a>
<a href="#706" id="706">706</a>
<a href="#707" id="707">707</a>
<a href="#708" id="708">708</a>
<a href="#709" id="709">709</a>
<a href="#710" id="710">710</a>
<a href="#711" id="711">711</a>
<a href="#712" id="712">712</a>
<a href="#713" id="713">713</a>
<a href="#714" id="714">714</a>
<a href="#715" id="715">715</a>
<a href="#716" id="716">716</a>
<a href="#717" id="717">717</a>
<a href="#718" id="718">718</a>
<a href="#719" id="719">719</a>
<a href="#720" id="720">720</a>
<a href="#721" id="721">721</a>
<a href="#722" id="722">722</a>
<a href="#723" id="723">723</a>
<a href="#724" id="724">724</a>
<a href="#725" id="725">725</a>
<a href="#726" id="726">726</a>
<a href="#727" id="727">727</a>
<a href="#728" id="728">728</a>
<a href="#729" id="729">729</a>
<a href="#730" id="730">730</a>
<a href="#731" id="731">731</a>
<a href="#732" id="732">732</a>
<a href="#733" id="733">733</a>
<a href="#734" id="734">734</a>
<a href="#735" id="735">735</a>
<a href="#736" id="736">736</a>
<a href="#737" id="737">737</a>
<a href="#738" id="738">738</a>
<a href="#739" id="739">739</a>
<a href="#740" id="740">740</a>
<a href="#741" id="741">741</a>
<a href="#742" id="742">742</a>
<a href="#743" id="743">743</a>
<a href="#744" id="744">744</a>
<a href="#745" id="745">745</a>
<a href="#746" id="746">746</a>
<a href="#747" id="747">747</a>
<a href="#748" id="748">748</a>
<a href="#749" id="749">749</a>
<a href="#750" id="750">750</a>
<a href="#751" id="751">751</a>
<a href="#752" id="752">752</a>
<a href="#753" id="753">753</a>
<a href="#754" id="754">754</a>
<a href="#755" id="755">755</a>
<a href="#756" id="756">756</a>
<a href="#757" id="757">757</a>
<a href="#758" id="758">758</a>
<a href="#759" id="759">759</a>
<a href="#760" id="760">760</a>
<a href="#761" id="761">761</a>
<a href="#762" id="762">762</a>
<a href="#763" id="763">763</a>
<a href="#764" id="764">764</a>
<a href="#765" id="765">765</a>
<a href="#766" id="766">766</a>
<a href="#767" id="767">767</a>
<a href="#768" id="768">768</a>
<a href="#769" id="769">769</a>
<a href="#770" id="770">770</a>
<a href="#771" id="771">771</a>
<a href="#772" id="772">772</a>
<a href="#773" id="773">773</a>
<a href="#774" id="774">774</a>
<a href="#775" id="775">775</a>
<a href="#776" id="776">776</a>
<a href="#777" id="777">777</a>
<a href="#778" id="778">778</a>
<a href="#779" id="779">779</a>
<a href="#780" id="780">780</a>
<a href="#781" id="781">781</a>
<a href="#782" id="782">782</a>
<a href="#783" id="783">783</a>
<a href="#784" id="784">784</a>
<a href="#785" id="785">785</a>
<a href="#786" id="786">786</a>
<a href="#787" id="787">787</a>
<a href="#788" id="788">788</a>
<a href="#789" id="789">789</a>
<a href="#790" id="790">790</a>
<a href="#791" id="791">791</a>
<a href="#792" id="792">792</a>
<a href="#793" id="793">793</a>
<a href="#794" id="794">794</a>
<a href="#795" id="795">795</a>
<a href="#796" id="796">796</a>
<a href="#797" id="797">797</a>
<a href="#798" id="798">798</a>
<a href="#799" id="799">799</a>
<a href="#800" id="800">800</a>
<a href="#801" id="801">801</a>
<a href="#802" id="802">802</a>
<a href="#803" id="803">803</a>
<a href="#804" id="804">804</a>
<a href="#805" id="805">805</a>
<a href="#806" id="806">806</a>
<a href="#807" id="807">807</a>
<a href="#808" id="808">808</a>
<a href="#809" id="809">809</a>
<a href="#810" id="810">810</a>
<a href="#811" id="811">811</a>
<a href="#812" id="812">812</a>
<a href="#813" id="813">813</a>
<a href="#814" id="814">814</a>
<a href="#815" id="815">815</a>
<a href="#816" id="816">816</a>
<a href="#817" id="817">817</a>
<a href="#818" id="818">818</a>
<a href="#819" id="819">819</a>
<a href="#820" id="820">820</a>
<a href="#821" id="821">821</a>
<a href="#822" id="822">822</a>
<a href="#823" id="823">823</a>
<a href="#824" id="824">824</a>
<a href="#825" id="825">825</a>
<a href="#826" id="826">826</a>
<a href="#827" id="827">827</a>
<a href="#828" id="828">828</a>
<a href="#829" id="829">829</a>
<a href="#830" id="830">830</a>
<a href="#831" id="831">831</a>
<a href="#832" id="832">832</a>
<a href="#833" id="833">833</a>
<a href="#834" id="834">834</a>
<a href="#835" id="835">835</a>
<a href="#836" id="836">836</a>
<a href="#837" id="837">837</a>
<a href="#838" id="838">838</a>
<a href="#839" id="839">839</a>
<a href="#840" id="840">840</a>
<a href="#841" id="841">841</a>
<a href="#842" id="842">842</a>
<a href="#843" id="843">843</a>
<a href="#844" id="844">844</a>
<a href="#845" id="845">845</a>
<a href="#846" id="846">846</a>
<a href="#847" id="847">847</a>
<a href="#848" id="848">848</a>
<a href="#849" id="849">849</a>
<a href="#850" id="850">850</a>
<a href="#851" id="851">851</a>
<a href="#852" id="852">852</a>
<a href="#853" id="853">853</a>
<a href="#854" id="854">854</a>
<a href="#855" id="855">855</a>
<a href="#856" id="856">856</a>
<a href="#857" id="857">857</a>
<a href="#858" id="858">858</a>
<a href="#859" id="859">859</a>
<a href="#860" id="860">860</a>
<a href="#861" id="861">861</a>
<a href="#862" id="862">862</a>
<a href="#863" id="863">863</a>
<a href="#864" id="864">864</a>
<a href="#865" id="865">865</a>
<a href="#866" id="866">866</a>
<a href="#867" id="867">867</a>
<a href="#868" id="868">868</a>
<a href="#869" id="869">869</a>
<a href="#870" id="870">870</a>
<a href="#871" id="871">871</a>
<a href="#872" id="872">872</a>
<a href="#873" id="873">873</a>
<a href="#874" id="874">874</a>
<a href="#875" id="875">875</a>
<a href="#876" id="876">876</a>
<a href="#877" id="877">877</a>
<a href="#878" id="878">878</a>
<a href="#879" id="879">879</a>
<a href="#880" id="880">880</a>
<a href="#881" id="881">881</a>
<a href="#882" id="882">882</a>
<a href="#883" id="883">883</a>
<a href="#884" id="884">884</a>
<a href="#885" id="885">885</a>
<a href="#886" id="886">886</a>
<a href="#887" id="887">887</a>
<a href="#888" id="888">888</a>
<a href="#889" id="889">889</a>
<a href="#890" id="890">890</a>
<a href="#891" id="891">891</a>
<a href="#892" id="892">892</a>
<a href="#893" id="893">893</a>
<a href="#894" id="894">894</a>
<a href="#895" id="895">895</a>
<a href="#896" id="896">896</a>
<a href="#897" id="897">897</a>
<a href="#898" id="898">898</a>
<a href="#899" id="899">899</a>
<a href="#900" id="900">900</a>
<a href="#901" id="901">901</a>
<a href="#902" id="902">902</a>
<a href="#903" id="903">903</a>
<a href="#904" id="904">904</a>
<a href="#905" id="905">905</a>
<a href="#906" id="906">906</a>
<a href="#907" id="907">907</a>
<a href="#908" id="908">908</a>
<a href="#909" id="909">909</a>
<a href="#910" id="910">910</a>
<a href="#911" id="911">911</a>
<a href="#912" id="912">912</a>
<a href="#913" id="913">913</a>
<a href="#914" id="914">914</a>
<a href="#915" id="915">915</a>
<a href="#916" id="916">916</a>
<a href="#917" id="917">917</a>
<a href="#918" id="918">918</a>
<a href="#919" id="919">919</a>
<a href="#920" id="920">920</a>
<a href="#921" id="921">921</a>
<a href="#922" id="922">922</a>
<a href="#923" id="923">923</a>
<a href="#924" id="924">924</a>
<a href="#925" id="925">925</a>
<a href="#926" id="926">926</a>
<a href="#927" id="927">927</a>
<a href="#928" id="928">928</a>
<a href="#929" id="929">929</a>
<a href="#930" id="930">930</a>
<a href="#931" id="931">931</a>
<a href="#932" id="932">932</a>
<a href="#933" id="933">933</a>
<a href="#934" id="934">934</a>
<a href="#935" id="935">935</a>
<a href="#936" id="936">936</a>
<a href="#937" id="937">937</a>
<a href="#938" id="938">938</a>
<a href="#939" id="939">939</a>
<a href="#940" id="940">940</a>
<a href="#941" id="941">941</a>
<a href="#942" id="942">942</a>
<a href="#943" id="943">943</a>
<a href="#944" id="944">944</a>
<a href="#945" id="945">945</a>
<a href="#946" id="946">946</a>
<a href="#947" id="947">947</a>
<a href="#948" id="948">948</a></pre></div><pre class="rust"><code><span class="doccomment">//! HTTP client for the Scrapfly API.
//!
//! Built on `reqwest` with `rustls`. Single shared [`reqwest::Client`]
//! re-used across every call.
</span><span class="kw">use </span>std::sync::Arc;
<span class="kw">use </span>std::time::Duration;
<span class="kw">use </span>futures_util::stream::{Stream, StreamExt};
<span class="kw">use </span>reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE, USER_AGENT};
<span class="kw">use </span>reqwest::{Method, Response, Url};
<span class="kw">use </span><span class="kw">crate</span>::config::crawler::CrawlerConfig;
<span class="kw">use </span><span class="kw">crate</span>::config::extraction::ExtractionConfig;
<span class="kw">use </span><span class="kw">crate</span>::config::scrape::ScrapeConfig;
<span class="kw">use </span><span class="kw">crate</span>::config::screenshot::ScreenshotConfig;
<span class="kw">use </span><span class="kw">crate</span>::enums::HttpMethod;
<span class="kw">use </span><span class="kw">crate</span>::error::{from_response, parse_retry_after, ApiError, ScrapflyError};
<span class="kw">use </span><span class="kw">crate</span>::result::account::{AccountData, VerifyApiKeyResult};
<span class="kw">use </span><span class="kw">crate</span>::result::crawler::{
CrawlerArtifact, CrawlerArtifactType, CrawlerContents, CrawlerStartResponse, CrawlerStatus,
CrawlerUrls,
};
<span class="kw">use </span><span class="kw">crate</span>::result::extraction::ExtractionResult;
<span class="kw">use </span><span class="kw">crate</span>::result::scrape::{ResultData, ScrapeResult};
<span class="kw">use </span><span class="kw">crate</span>::result::screenshot::{ScreenshotMetadata, ScreenshotResult};
<span class="kw">const </span>DEFAULT_HOST: <span class="kw-2">&</span>str = <span class="string">"https://api.scrapfly.io"</span>;
<span class="kw">const </span>DEFAULT_CLOUD_BROWSER_HOST: <span class="kw-2">&</span>str = <span class="string">"https://browser.scrapfly.io"</span>;
<span class="kw">const </span>SDK_USER_AGENT: <span class="kw-2">&</span>str = <span class="string">"Scrapfly-Rust-SDK"</span>;
<span class="kw">const </span>DEFAULT_RETRIES: usize = <span class="number">3</span>;
<span class="kw">const </span>DEFAULT_RETRY_DELAY: Duration = Duration::from_secs(<span class="number">1</span>);
<span class="kw">const </span>DEFAULT_TIMEOUT: Duration = Duration::from_secs(<span class="number">150</span>);
<span class="doccomment">/// Request-inspection callback. Fires right before `send()`.
///
/// Used by the integration harness to record the outgoing method/URL/headers
/// without wrapping the `reqwest::Client` in a middleware layer.
</span><span class="kw">pub type </span>OnRequest = Arc<<span class="kw">dyn </span>Fn(<span class="kw-2">&</span>Method, <span class="kw-2">&</span>Url, <span class="kw-2">&</span>HeaderMap) + Send + Sync>;
<span class="doccomment">/// Scrapfly API client. Cheap to `Clone` (the inner `reqwest::Client` is
/// `Arc`'d so all clones share one connection pool).
</span><span class="attr">#[derive(Clone)]
</span><span class="kw">pub struct </span>Client {
http: reqwest::Client,
key: String,
host: String,
cloud_browser_host: String,
on_request: <span class="prelude-ty">Option</span><OnRequest>,
}
<span class="kw">impl </span>std::fmt::Debug <span class="kw">for </span>Client {
<span class="kw">fn </span>fmt(<span class="kw-2">&</span><span class="self">self</span>, f: <span class="kw-2">&mut </span>std::fmt::Formatter<<span class="lifetime">'_</span>>) -> std::fmt::Result {
f.debug_struct(<span class="string">"Client"</span>)
.field(<span class="string">"host"</span>, <span class="kw-2">&</span><span class="self">self</span>.host)
.field(<span class="string">"cloud_browser_host"</span>, <span class="kw-2">&</span><span class="self">self</span>.cloud_browser_host)
.finish()
}
}
<span class="doccomment">/// Builder for [`Client`].
</span><span class="attr">#[derive(Default)]
</span><span class="kw">pub struct </span>ClientBuilder {
api_key: <span class="prelude-ty">Option</span><String>,
host: <span class="prelude-ty">Option</span><String>,
cloud_browser_host: <span class="prelude-ty">Option</span><String>,
timeout: <span class="prelude-ty">Option</span><Duration>,
danger_accept_invalid_certs: bool,
http_client: <span class="prelude-ty">Option</span><reqwest::Client>,
on_request: <span class="prelude-ty">Option</span><OnRequest>,
}
<span class="kw">impl </span>ClientBuilder {
<span class="doccomment">/// Set the API key (required).
</span><span class="kw">pub fn </span>api_key(<span class="kw-2">mut </span><span class="self">self</span>, key: <span class="kw">impl </span>Into<String>) -> <span class="self">Self </span>{
<span class="self">self</span>.api_key = <span class="prelude-val">Some</span>(key.into());
<span class="self">self
</span>}
<span class="doccomment">/// Override the API host.
</span><span class="kw">pub fn </span>host(<span class="kw-2">mut </span><span class="self">self</span>, host: <span class="kw">impl </span>Into<String>) -> <span class="self">Self </span>{
<span class="self">self</span>.host = <span class="prelude-val">Some</span>(host.into());
<span class="self">self
</span>}
<span class="doccomment">/// Override the Cloud Browser host (`https://browser.scrapfly.io`).
</span><span class="kw">pub fn </span>cloud_browser_host(<span class="kw-2">mut </span><span class="self">self</span>, host: <span class="kw">impl </span>Into<String>) -> <span class="self">Self </span>{
<span class="self">self</span>.cloud_browser_host = <span class="prelude-val">Some</span>(host.into());
<span class="self">self
</span>}
<span class="doccomment">/// Override the HTTP timeout (default 150s).
</span><span class="kw">pub fn </span>timeout(<span class="kw-2">mut </span><span class="self">self</span>, t: Duration) -> <span class="self">Self </span>{
<span class="self">self</span>.timeout = <span class="prelude-val">Some</span>(t);
<span class="self">self
</span>}
<span class="doccomment">/// Accept invalid TLS certificates (tests / self-signed dev hosts).
</span><span class="kw">pub fn </span>danger_accept_invalid_certs(<span class="kw-2">mut </span><span class="self">self</span>, v: bool) -> <span class="self">Self </span>{
<span class="self">self</span>.danger_accept_invalid_certs = v;
<span class="self">self
</span>}
<span class="doccomment">/// Inject a pre-built `reqwest::Client`. Bypasses the timeout /
/// TLS-verify options.
</span><span class="kw">pub fn </span>http_client(<span class="kw-2">mut </span><span class="self">self</span>, client: reqwest::Client) -> <span class="self">Self </span>{
<span class="self">self</span>.http_client = <span class="prelude-val">Some</span>(client);
<span class="self">self
</span>}
<span class="doccomment">/// Install a pre-send request callback (used by the integration runner
/// to capture SDK-layer attribution without installing middleware).
</span><span class="kw">pub fn </span>on_request(<span class="kw-2">mut </span><span class="self">self</span>, cb: OnRequest) -> <span class="self">Self </span>{
<span class="self">self</span>.on_request = <span class="prelude-val">Some</span>(cb);
<span class="self">self
</span>}
<span class="doccomment">/// Build the client.
</span><span class="kw">pub fn </span>build(<span class="self">self</span>) -> <span class="prelude-ty">Result</span><Client, ScrapflyError> {
<span class="kw">let </span>key = <span class="self">self</span>.api_key.ok_or(ScrapflyError::BadApiKey)<span class="question-mark">?</span>;
<span class="kw">if </span>key.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::BadApiKey);
}
<span class="kw">let </span>http = <span class="kw">if let </span><span class="prelude-val">Some</span>(c) = <span class="self">self</span>.http_client {
c
} <span class="kw">else </span>{
<span class="kw">let </span><span class="kw-2">mut </span>builder = reqwest::Client::builder()
.timeout(<span class="self">self</span>.timeout.unwrap_or(DEFAULT_TIMEOUT))
.user_agent(SDK_USER_AGENT);
<span class="kw">if </span><span class="self">self</span>.danger_accept_invalid_certs {
builder = builder.danger_accept_invalid_certs(<span class="bool-val">true</span>);
}
builder.build().map_err(ScrapflyError::Transport)<span class="question-mark">?
</span>};
<span class="prelude-val">Ok</span>(Client {
http,
key,
host: <span class="self">self</span>.host.unwrap_or_else(|| DEFAULT_HOST.to_string()),
cloud_browser_host: <span class="self">self
</span>.cloud_browser_host
.unwrap_or_else(|| DEFAULT_CLOUD_BROWSER_HOST.to_string()),
on_request: <span class="self">self</span>.on_request,
})
}
}
<span class="kw">impl </span>Client {
<span class="doccomment">/// Start a new [`ClientBuilder`].
</span><span class="kw">pub fn </span>builder() -> ClientBuilder {
ClientBuilder::default()
}
<span class="doccomment">/// Return the configured API key.
</span><span class="kw">pub fn </span>api_key(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="kw-2">&</span>str {
<span class="kw-2">&</span><span class="self">self</span>.key
}
<span class="doccomment">/// Return the configured API host.
</span><span class="kw">pub fn </span>host(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="kw-2">&</span>str {
<span class="kw-2">&</span><span class="self">self</span>.host
}
<span class="doccomment">/// Return the configured Cloud Browser host.
</span><span class="kw">pub fn </span>cloud_browser_host(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="kw-2">&</span>str {
<span class="kw-2">&</span><span class="self">self</span>.cloud_browser_host
}
<span class="doccomment">/// Build a URL by joining `path` onto the configured host.
</span><span class="kw">fn </span>build_url(<span class="kw-2">&</span><span class="self">self</span>, path: <span class="kw-2">&</span>str, query: <span class="kw-2">&</span>[(String, String)]) -> <span class="prelude-ty">Result</span><Url, ScrapflyError> {
<span class="kw">let </span><span class="kw-2">mut </span>u = Url::parse(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"{}{}"</span>, <span class="self">self</span>.host, path))
.map_err(|e| ScrapflyError::Config(<span class="macro">format!</span>(<span class="string">"invalid url: {}"</span>, e)))<span class="question-mark">?</span>;
{
<span class="kw">let </span><span class="kw-2">mut </span>pairs = u.query_pairs_mut();
pairs.append_pair(<span class="string">"key"</span>, <span class="kw-2">&</span><span class="self">self</span>.key);
<span class="kw">for </span>(k, v) <span class="kw">in </span>query {
pairs.append_pair(k, v);
}
}
<span class="prelude-val">Ok</span>(u)
}
<span class="doccomment">/// Verify the API key by hitting `/account`.
</span><span class="kw">pub async fn </span>verify_api_key(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="prelude-ty">Result</span><VerifyApiKeyResult, ScrapflyError> {
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="string">"/account"</span>, <span class="kw-2">&</span>[])<span class="question-mark">?</span>;
<span class="kw">let </span>resp = <span class="self">self</span>.send_simple(Method::GET, url, <span class="prelude-val">None</span>, <span class="prelude-val">None</span>).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="prelude-val">Ok</span>(VerifyApiKeyResult {
valid: resp.status().is_success(),
})
}
<span class="doccomment">/// Fetch account info.
</span><span class="kw">pub async fn </span>account(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="prelude-ty">Result</span><AccountData, ScrapflyError> {
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="string">"/account"</span>, <span class="kw-2">&</span>[])<span class="question-mark">?</span>;
<span class="kw">let </span>resp = <span class="self">self</span>.send_simple(Method::GET, url, <span class="prelude-val">None</span>, <span class="prelude-val">None</span>).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _headers, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">false</span>));
}
<span class="prelude-val">Ok</span>(serde_json::from_slice(<span class="kw-2">&</span>body)<span class="question-mark">?</span>)
}
<span class="doccomment">/// Scrape a URL.
</span><span class="kw">pub async fn </span>scrape(<span class="kw-2">&</span><span class="self">self</span>, config: <span class="kw-2">&</span>ScrapeConfig) -> <span class="prelude-ty">Result</span><ScrapeResult, ScrapflyError> {
<span class="kw">let </span>pairs = config.to_query_pairs()<span class="question-mark">?</span>;
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="string">"/scrape"</span>, <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span>method = <span class="kw">match </span>config.method {
<span class="prelude-val">Some</span>(m) => Method::from_bytes(m.as_str().as_bytes())
.map_err(|e| ScrapflyError::Config(<span class="macro">format!</span>(<span class="string">"invalid method: {}"</span>, e)))<span class="question-mark">?</span>,
<span class="prelude-val">None </span>=> Method::GET,
};
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(<span class="string">"application/json"</span>));
<span class="kw">let </span>body = config.body.clone();
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(method, url, <span class="prelude-val">Some</span>(headers), body.map(|b| b.into_bytes()))
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body_bytes) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body_bytes, <span class="number">0</span>, <span class="bool-val">false</span>));
}
<span class="comment">// HEAD has no body per HTTP spec, so the Scrapfly API returns a 200
// with an empty body — there's no JSON envelope to parse. Synthesize
// a minimal ScrapeResult so callers still get a typed response with
// status_code=200 and an empty content string. Matches Python SDK
// behavior, which tolerates an empty body_handler read on HEAD.
</span><span class="kw">if </span><span class="macro">matches!</span>(config.method, <span class="prelude-val">Some</span>(HttpMethod::Head)) && body_bytes.is_empty() {
<span class="kw">return </span><span class="prelude-val">Ok</span>(ScrapeResult {
uuid: String::new(),
config: serde_json::Value::Null,
context: serde_json::Value::Null,
result: ResultData {
status_code: <span class="number">200</span>,
success: <span class="bool-val">true</span>,
..Default::default()
},
});
}
<span class="kw">let </span><span class="kw-2">mut </span>result: ScrapeResult = serde_json::from_slice(<span class="kw-2">&</span>body_bytes)<span class="question-mark">?</span>;
<span class="comment">// Upstream failure handling: the Scrapfly API call itself may succeed
// (HTTP 200) while the *target* site returned a failure. In that case
// result.result.success is false and we must surface it as an error
// variant so callers can `match` on it. Mirrors the Go SDK behavior
// in `sdk/go/client.go::checkResult` (4xx → UpstreamClient,
// 5xx → UpstreamServer).
</span><span class="kw">if </span>!result.result.success {
<span class="kw">let </span>(err_code, err_message, err_doc) = <span class="kw">match </span><span class="kw-2">&</span>result.result.error {
<span class="prelude-val">Some</span>(e) => (e.code.clone(), e.message.clone(), e.doc_url.clone()),
<span class="prelude-val">None </span>=> (
result.result.status.clone(),
<span class="macro">format!</span>(
<span class="string">"scrape failed with status_code={}"</span>,
result.result.status_code
),
String::new(),
),
};
<span class="kw">let </span>api_err = ApiError {
code: err_code,
message: err_message,
http_status: result.result.status_code,
documentation_url: err_doc,
hint: String::new(),
retry_after_ms: <span class="number">0</span>,
};
<span class="kw">let </span>sc = result.result.status_code;
<span class="kw">if </span>(<span class="number">400</span>..<span class="number">500</span>).contains(<span class="kw-2">&</span>sc) {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::UpstreamClient(api_err));
}
<span class="kw">if </span>(<span class="number">500</span>..<span class="number">600</span>).contains(<span class="kw-2">&</span>sc) {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::UpstreamServer(api_err));
}
<span class="comment">// Unknown status code (e.g. 0, timeouts) — fall through to generic
// Api error rather than silently returning a failed result.
</span><span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Api(api_err));
}
<span class="comment">// Transparent large-object handling: when a scrape response is too
// large, the engine offloads the body to a signed URL and sets
// `format=clob|blob`, stashing the URL in `content`. The SDK must
// auto-fetch and surface the final bytes + a user-friendly format
// marker (clob→text, blob→binary). Mirrors `sdk/go/client.go::handleLargeObjects`.
</span><span class="kw">if </span>result.result.success && result.result.status == <span class="string">"DONE" </span>{
<span class="kw">let </span>fmt = result.result.format.as_str();
<span class="kw">if </span>fmt == <span class="string">"clob" </span>|| fmt == <span class="string">"blob" </span>{
<span class="kw">let </span>(new_content, new_format) =
<span class="self">self</span>.fetch_large_object(<span class="kw-2">&</span>result.result.content, fmt).<span class="kw">await</span><span class="question-mark">?</span>;
result.result.content = new_content;
result.result.format = new_format;
}
}
<span class="prelude-val">Ok</span>(result)
}
<span class="doccomment">/// Fetch an offloaded large-object body from its signed URL, re-attaching
/// the API key as a query param. Returns `(content, format)`:
/// `clob → ("…text…", "text")`, `blob → ("…bytes as lossy utf8…", "binary")`.
</span><span class="kw">async fn </span>fetch_large_object(
<span class="kw-2">&</span><span class="self">self</span>,
content_url: <span class="kw-2">&</span>str,
format: <span class="kw-2">&</span>str,
) -> <span class="prelude-ty">Result</span><(String, String), ScrapflyError> {
<span class="kw">let </span><span class="kw-2">mut </span>url = Url::parse(content_url)
.map_err(|e| ScrapflyError::Config(<span class="macro">format!</span>(<span class="string">"invalid large-object url: {}"</span>, e)))<span class="question-mark">?</span>;
<span class="comment">// Append the API key without clobbering existing query params.
</span>{
<span class="kw">let </span>existing: Vec<(String, String)> = url
.query_pairs()
.filter(|(k, <span class="kw">_</span>)| k != <span class="string">"key"</span>)
.map(|(k, v)| (k.into_owned(), v.into_owned()))
.collect();
<span class="kw">let </span><span class="kw-2">mut </span>qs = url.query_pairs_mut();
qs.clear();
<span class="kw">for </span>(k, v) <span class="kw">in </span>existing {
qs.append_pair(<span class="kw-2">&</span>k, <span class="kw-2">&</span>v);
}
qs.append_pair(<span class="string">"key"</span>, <span class="self">self</span>.api_key());
}
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(<span class="string">"application/json"</span>));
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::GET, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">None</span>)
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">false</span>));
}
<span class="kw">let </span>new_format = <span class="kw">match </span>format {
<span class="string">"clob" </span>=> <span class="string">"text"</span>,
<span class="string">"blob" </span>=> <span class="string">"binary"</span>,
<span class="kw">_ </span>=> {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="macro">format!</span>(
<span class="string">"unsupported large-object format: {}"</span>,
format
)))
}
};
<span class="comment">// For blob (binary PDF, image, etc.) we use from_utf8_lossy to
// preserve the raw bytes in the `content` string field, matching
// the Go/Python SDKs' behavior.
</span><span class="kw">let </span>content = String::from_utf8_lossy(<span class="kw-2">&</span>body).into_owned();
<span class="prelude-val">Ok</span>((content, new_format.to_string()))
}
<span class="doccomment">/// Concurrent-scrape stream. Emits results in completion order.
</span><span class="kw">pub fn </span>concurrent_scrape<<span class="lifetime">'a</span>, I>(
<span class="kw-2">&</span><span class="lifetime">'a </span><span class="self">self</span>,
configs: I,
concurrency_limit: usize,
) -> <span class="kw">impl </span>Stream<Item = <span class="prelude-ty">Result</span><ScrapeResult, ScrapflyError>> + <span class="lifetime">'a
</span><span class="kw">where
</span>I: IntoIterator<Item = ScrapeConfig> + <span class="lifetime">'a</span>,
<I <span class="kw">as </span>IntoIterator>::IntoIter: <span class="lifetime">'a</span>,
{
<span class="kw">let </span>limit = <span class="kw">if </span>concurrency_limit == <span class="number">0 </span>{
<span class="number">5
</span>} <span class="kw">else </span>{
concurrency_limit
};
futures_util::stream::iter(
configs
.into_iter()
.map(<span class="kw">move </span>|cfg| <span class="kw">async move </span>{ <span class="self">self</span>.scrape(<span class="kw-2">&</span>cfg).<span class="kw">await </span>}),
)
.buffer_unordered(limit)
}
<span class="doccomment">/// Scrape a URL with `proxified_response=true`, returning the raw
/// upstream `reqwest::Response` (target's status, headers, body).
///
/// Unlike [`scrape()`], no JSON parsing occurs — the response body is
/// the target page's raw content. Scrapfly metadata is available on
/// the `X-Scrapfly-*` response headers (`Api-Cost`, `Content-Format`,
/// `Log`, etc.).
///
/// Automatically forces `proxified_response=true` regardless of the
/// config's field value.
</span><span class="kw">pub async fn </span>scrape_proxified(
<span class="kw-2">&</span><span class="self">self</span>,
config: <span class="kw-2">&</span>ScrapeConfig,
) -> <span class="prelude-ty">Result</span><reqwest::Response, ScrapflyError> {
<span class="kw">let </span><span class="kw-2">mut </span>cfg = config.clone();
cfg.proxified_response = <span class="bool-val">true</span>;
<span class="kw">let </span>pairs = cfg.to_query_pairs()<span class="question-mark">?</span>;
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="string">"/scrape"</span>, <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span>method = <span class="kw">match </span>cfg.method {
<span class="prelude-val">Some</span>(m) => Method::from_bytes(m.as_str().as_bytes())
.map_err(|e| ScrapflyError::Config(<span class="macro">format!</span>(<span class="string">"invalid method: {}"</span>, e)))<span class="question-mark">?</span>,
<span class="prelude-val">None </span>=> Method::GET,
};
<span class="kw">let </span>body = cfg.body.clone();
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(method, url, <span class="prelude-val">None</span>, body.map(|b| b.into_bytes()))
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="comment">// Error restoration: if X-Scrapfly-Reject-Code is present, the
// scrape failed. Return a typed error so callers get the same
// interface as non-proxified mode.
</span><span class="kw">if let </span><span class="prelude-val">Some</span>(reject_code) = resp.headers().get(<span class="string">"x-scrapfly-reject-code"</span>) {
<span class="kw">let </span>code = reject_code.to_str().unwrap_or(<span class="string">""</span>).to_string();
<span class="kw">let </span>desc = resp.headers().get(<span class="string">"x-scrapfly-reject-description"</span>)
.and_then(|v| v.to_str().ok()).unwrap_or(<span class="string">""</span>).to_string();
<span class="kw">let </span>retryable = resp.headers().get(<span class="string">"x-scrapfly-reject-retryable"</span>)
.and_then(|v| v.to_str().ok()).unwrap_or(<span class="string">"false"</span>) == <span class="string">"true"</span>;
<span class="kw">let </span>retry_after_ms: u64 = <span class="kw">if </span>retryable {
resp.headers().get(<span class="string">"retry-after"</span>)
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(<span class="number">0</span>) * <span class="number">1000 </span><span class="comment">// Retry-After header is in seconds
</span>} <span class="kw">else </span>{ <span class="number">0 </span>};
<span class="kw">let </span>status = resp.status().as_u16();
<span class="kw">let </span>doc = resp.headers().get(<span class="string">"x-scrapfly-reject-doc"</span>)
.and_then(|v| v.to_str().ok()).unwrap_or(<span class="string">""</span>).to_string();
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Api(<span class="kw">crate</span>::error::ApiError {
code,
message: <span class="macro">format!</span>(<span class="string">"Proxified scrape error: {}"</span>, desc),
http_status: status,
documentation_url: doc,
hint: String::new(),
retry_after_ms,
}));
}
<span class="prelude-val">Ok</span>(resp)
}
<span class="doccomment">/// Screenshot a URL.
</span><span class="kw">pub async fn </span>screenshot(
<span class="kw-2">&</span><span class="self">self</span>,
config: <span class="kw-2">&</span>ScreenshotConfig,
) -> <span class="prelude-ty">Result</span><ScreenshotResult, ScrapflyError> {
<span class="kw">let </span>pairs = config.to_query_pairs()<span class="question-mark">?</span>;
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="string">"/screenshot"</span>, <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span>resp = <span class="self">self</span>.send_with_retry(Method::GET, url, <span class="prelude-val">None</span>, <span class="prelude-val">None</span>).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, headers, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">false</span>));
}
<span class="kw">let </span>content_type = headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or(<span class="string">"application/octet-stream"</span>);
<span class="kw">let </span>ext = content_type
.split(<span class="string">'/'</span>)
.nth(<span class="number">1</span>)
.and_then(|s| s.split(<span class="string">';'</span>).next())
.unwrap_or(<span class="string">"bin"</span>)
.to_string();
<span class="kw">let </span>upstream_status_code: u16 = headers
.get(<span class="string">"x-scrapfly-upstream-http-code"</span>)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse().ok())
.unwrap_or(<span class="number">0</span>);
<span class="kw">let </span>upstream_url = headers
.get(<span class="string">"x-scrapfly-upstream-url"</span>)
.and_then(|v| v.to_str().ok())
.unwrap_or(<span class="string">""</span>)
.to_string();
<span class="prelude-val">Ok</span>(ScreenshotResult {
image: body,
metadata: ScreenshotMetadata {
extension_name: ext,
upstream_status_code,
upstream_url,
},
})
}
<span class="doccomment">/// Run AI extraction on a document.
</span><span class="kw">pub async fn </span>extract(
<span class="kw-2">&</span><span class="self">self</span>,
config: <span class="kw-2">&</span>ExtractionConfig,
) -> <span class="prelude-ty">Result</span><ExtractionResult, ScrapflyError> {
<span class="kw">let </span>pairs = config.to_query_pairs()<span class="question-mark">?</span>;
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="string">"/extraction"</span>, <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(
CONTENT_TYPE,
HeaderValue::from_str(<span class="kw-2">&</span>config.content_type)
.map_err(|e| ScrapflyError::Config(<span class="macro">format!</span>(<span class="string">"invalid content-type: {}"</span>, e)))<span class="question-mark">?</span>,
);
headers.insert(ACCEPT, HeaderValue::from_static(<span class="string">"application/json"</span>));
<span class="kw">if let </span><span class="prelude-val">Some</span>(fmt) = config.document_compression_format {
headers.insert(
<span class="string">"content-encoding"</span>,
HeaderValue::from_str(fmt.as_str())
.map_err(|e| ScrapflyError::Config(<span class="macro">format!</span>(<span class="string">"invalid encoding: {}"</span>, e)))<span class="question-mark">?</span>,
);
}
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::POST, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">Some</span>(config.body.clone()))
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body_bytes) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body_bytes, <span class="number">0</span>, <span class="bool-val">false</span>));
}
<span class="prelude-val">Ok</span>(serde_json::from_slice(<span class="kw-2">&</span>body_bytes)<span class="question-mark">?</span>)
}
<span class="comment">// ==============================================================================
// Crawler methods
// ==============================================================================
</span><span class="doccomment">/// Schedule a new crawler job.
</span><span class="kw">pub async fn </span>start_crawl(
<span class="kw-2">&</span><span class="self">self</span>,
config: <span class="kw-2">&</span>CrawlerConfig,
) -> <span class="prelude-ty">Result</span><CrawlerStartResponse, ScrapflyError> {
<span class="kw">let </span>body = config.to_json_body()<span class="question-mark">?</span>;
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="string">"/crawl"</span>, <span class="kw-2">&</span>[])<span class="question-mark">?</span>;
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(<span class="string">"application/json"</span>));
headers.insert(ACCEPT, HeaderValue::from_static(<span class="string">"application/json"</span>));
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::POST, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">Some</span>(body))
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body_bytes) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>&& status != <span class="number">201 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body_bytes, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="kw">let </span>parsed: CrawlerStartResponse = serde_json::from_slice(<span class="kw-2">&</span>body_bytes)<span class="question-mark">?</span>;
<span class="kw">if </span>parsed.crawler_uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::UnexpectedResponseFormat(
<span class="string">"crawler start response missing crawler_uuid"</span>.into(),
));
}
<span class="prelude-val">Ok</span>(parsed)
}
<span class="doccomment">/// Fetch crawler status.
</span><span class="kw">pub async fn </span>crawl_status(<span class="kw-2">&</span><span class="self">self</span>, uuid: <span class="kw-2">&</span>str) -> <span class="prelude-ty">Result</span><CrawlerStatus, ScrapflyError> {
<span class="kw">if </span>uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"uuid cannot be empty"</span>.into()));
}
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"/crawl/{}/status"</span>, uuid), <span class="kw-2">&</span>[])<span class="question-mark">?</span>;
<span class="kw">let </span>resp = <span class="self">self</span>.send_with_retry(Method::GET, url, <span class="prelude-val">None</span>, <span class="prelude-val">None</span>).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="prelude-val">Ok</span>(serde_json::from_slice(<span class="kw-2">&</span>body)<span class="question-mark">?</span>)
}
<span class="doccomment">/// List crawled URLs (streaming text endpoint).
</span><span class="kw">pub async fn </span>crawl_urls(
<span class="kw-2">&</span><span class="self">self</span>,
uuid: <span class="kw-2">&</span>str,
status_filter: <span class="prelude-ty">Option</span><<span class="kw-2">&</span>str>,
page: u32,
per_page: u32,
) -> <span class="prelude-ty">Result</span><CrawlerUrls, ScrapflyError> {
<span class="kw">if </span>uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"uuid cannot be empty"</span>.into()));
}
<span class="kw">let </span>page = <span class="kw">if </span>page == <span class="number">0 </span>{ <span class="number">1 </span>} <span class="kw">else </span>{ page };
<span class="kw">let </span>per_page = <span class="kw">if </span>per_page == <span class="number">0 </span>{ <span class="number">100 </span>} <span class="kw">else </span>{ per_page };
<span class="kw">let </span>status_hint = status_filter.unwrap_or(<span class="string">"visited"</span>);
<span class="kw">let </span><span class="kw-2">mut </span>pairs: Vec<(String, String)> = <span class="macro">vec!</span>[
(<span class="string">"page"</span>.into(), page.to_string()),
(<span class="string">"per_page"</span>.into(), per_page.to_string()),
];
<span class="kw">if let </span><span class="prelude-val">Some</span>(s) = status_filter {
pairs.push((<span class="string">"status"</span>.into(), s.to_string()));
}
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"/crawl/{}/urls"</span>, uuid), <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(
ACCEPT,
HeaderValue::from_static(<span class="string">"text/plain, application/json"</span>),
);
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::GET, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">None</span>)
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, resp_headers, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="kw">let </span>ct = resp_headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or(<span class="string">""</span>);
<span class="kw">if </span>ct.contains(<span class="string">"application/json"</span>) {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::UnexpectedResponseFormat(<span class="macro">format!</span>(
<span class="string">"GET /crawl/{}/urls returned JSON on a 200 response (expected text/plain)"</span>,
uuid
)));
}
<span class="kw">let </span>body_str = std::str::from_utf8(<span class="kw-2">&</span>body)
.map_err(|e| ScrapflyError::UnexpectedResponseFormat(<span class="macro">format!</span>(<span class="string">"invalid utf8: {}"</span>, e)))<span class="question-mark">?</span>;
<span class="prelude-val">Ok</span>(CrawlerUrls::from_text(
body_str,
status_hint,
page,
per_page,
))
}
<span class="doccomment">/// Bulk `GET /crawl/{uuid}/contents` in JSON mode.
</span><span class="kw">pub async fn </span>crawl_contents_json(
<span class="kw-2">&</span><span class="self">self</span>,
uuid: <span class="kw-2">&</span>str,
format: <span class="kw">crate</span>::enums::CrawlerContentFormat,
limit: <span class="prelude-ty">Option</span><u32>,
offset: <span class="prelude-ty">Option</span><u32>,
) -> <span class="prelude-ty">Result</span><CrawlerContents, ScrapflyError> {
<span class="kw">if </span>uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"uuid cannot be empty"</span>.into()));
}
<span class="kw">let </span><span class="kw-2">mut </span>pairs: Vec<(String, String)> = <span class="macro">vec!</span>[(<span class="string">"formats"</span>.into(), format.as_str().into())];
<span class="kw">if let </span><span class="prelude-val">Some</span>(l) = limit {
pairs.push((<span class="string">"limit"</span>.into(), l.to_string()));
}
<span class="kw">if let </span><span class="prelude-val">Some</span>(o) = offset {
pairs.push((<span class="string">"offset"</span>.into(), o.to_string()));
}
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"/crawl/{}/contents"</span>, uuid), <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(<span class="string">"application/json"</span>));
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::GET, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">None</span>)
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, resp_headers, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="kw">let </span>ct = resp_headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or(<span class="string">""</span>);
<span class="kw">if </span>!ct.contains(<span class="string">"application/json"</span>) {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::UnexpectedResponseFormat(<span class="macro">format!</span>(
<span class="string">"expected JSON, got Content-Type={}"</span>,
ct
)));
}
<span class="prelude-val">Ok</span>(serde_json::from_slice(<span class="kw-2">&</span>body)<span class="question-mark">?</span>)
}
<span class="doccomment">/// Plain single-URL `GET /crawl/{uuid}/contents?plain=true`.
</span><span class="kw">pub async fn </span>crawl_contents_plain(
<span class="kw-2">&</span><span class="self">self</span>,
uuid: <span class="kw-2">&</span>str,
target_url: <span class="kw-2">&</span>str,
format: <span class="kw">crate</span>::enums::CrawlerContentFormat,
) -> <span class="prelude-ty">Result</span><String, ScrapflyError> {
<span class="kw">if </span>uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"uuid cannot be empty"</span>.into()));
}
<span class="kw">if </span>target_url.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(
<span class="string">"plain mode requires a single url argument"</span>.into(),
));
}
<span class="kw">let </span>pairs: Vec<(String, String)> = <span class="macro">vec!</span>[
(<span class="string">"formats"</span>.into(), format.as_str().into()),
(<span class="string">"url"</span>.into(), target_url.into()),
(<span class="string">"plain"</span>.into(), <span class="string">"true"</span>.into()),
];
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"/crawl/{}/contents"</span>, uuid), <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(<span class="string">"*/*"</span>));
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::GET, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">None</span>)
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="prelude-val">Ok</span>(String::from_utf8_lossy(<span class="kw-2">&</span>body).into_owned())
}
<span class="doccomment">/// Bulk-batch `POST /crawl/{uuid}/contents/batch`.
/// Returns `url → format → content` (multipart/related response).
</span><span class="kw">pub async fn </span>crawl_contents_batch(
<span class="kw-2">&</span><span class="self">self</span>,
uuid: <span class="kw-2">&</span>str,
urls: <span class="kw-2">&</span>[String],
formats: <span class="kw-2">&</span>[<span class="kw">crate</span>::enums::CrawlerContentFormat],
) -> <span class="prelude-ty">Result</span><
std::collections::BTreeMap<String, std::collections::BTreeMap<String, String>>,
ScrapflyError,
> {
<span class="kw">if </span>uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"uuid cannot be empty"</span>.into()));
}
<span class="kw">if </span>urls.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"at least one URL is required"</span>.into()));
}
<span class="kw">if </span>urls.len() > <span class="number">100 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="macro">format!</span>(
<span class="string">"batch is limited to 100 URLs per request, got {}"</span>,
urls.len()
)));
}
<span class="kw">if </span>formats.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(
<span class="string">"at least one format is required"</span>.into(),
));
}
<span class="kw">let </span>format_strs: Vec<<span class="kw-2">&</span><span class="lifetime">'static </span>str> = formats.iter().map(|f| f.as_str()).collect();
<span class="kw">let </span>pairs: Vec<(String, String)> = <span class="macro">vec!</span>[(<span class="string">"formats"</span>.into(), format_strs.join(<span class="string">","</span>))];
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"/crawl/{}/contents/batch"</span>, uuid), <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span>body = urls.join(<span class="string">"\n"</span>).into_bytes();
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(<span class="string">"text/plain"</span>));
headers.insert(
ACCEPT,
HeaderValue::from_static(<span class="string">"multipart/related, application/json"</span>),
);
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::POST, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">Some</span>(body))
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, resp_headers, body_bytes) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body_bytes, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="kw">let </span>ct = resp_headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or(<span class="string">""</span>);
<span class="kw">if </span>ct.contains(<span class="string">"application/json"</span>) {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::UnexpectedResponseFormat(
<span class="string">"CrawlContentsBatch expected multipart/related, got JSON"</span>.into(),
));
}
parse_multipart_related(
std::str::from_utf8(<span class="kw-2">&</span>body_bytes).unwrap_or(<span class="string">""</span>),
ct,
<span class="kw-2">&</span>format_strs,
)
}
<span class="doccomment">/// Cancel a crawler job.
</span><span class="kw">pub async fn </span>crawl_cancel(<span class="kw-2">&</span><span class="self">self</span>, uuid: <span class="kw-2">&</span>str) -> <span class="prelude-ty">Result</span><(), ScrapflyError> {
<span class="kw">if </span>uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"uuid cannot be empty"</span>.into()));
}
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"/crawl/{}/cancel"</span>, uuid), <span class="kw-2">&</span>[])<span class="question-mark">?</span>;
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(<span class="string">"application/json"</span>));
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::POST, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">None</span>)
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>&& status != <span class="number">202 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="prelude-val">Ok</span>(())
}
<span class="doccomment">/// Download a crawler artifact (WARC or HAR).
</span><span class="kw">pub async fn </span>crawl_artifact(
<span class="kw-2">&</span><span class="self">self</span>,
uuid: <span class="kw-2">&</span>str,
artifact_type: CrawlerArtifactType,
) -> <span class="prelude-ty">Result</span><CrawlerArtifact, ScrapflyError> {
<span class="kw">if </span>uuid.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::Config(<span class="string">"uuid cannot be empty"</span>.into()));
}
<span class="kw">let </span>pairs: Vec<(String, String)> = <span class="macro">vec!</span>[(<span class="string">"type"</span>.into(), artifact_type.as_str().into())];
<span class="kw">let </span>url = <span class="self">self</span>.build_url(<span class="kw-2">&</span><span class="macro">format!</span>(<span class="string">"/crawl/{}/artifact"</span>, uuid), <span class="kw-2">&</span>pairs)<span class="question-mark">?</span>;
<span class="kw">let </span><span class="kw-2">mut </span>headers = HeaderMap::new();
<span class="comment">// HAR is plain JSON — asking for `application/gzip` makes the server
// gzip-wrap it, and reqwest can't auto-decode it without a matching
// `Content-Encoding` header. Match `sdk/go/crawler.go::CrawlArtifact`
// which sends different Accept per artifact type.
</span><span class="kw">let </span>accept = <span class="kw">match </span>artifact_type {
CrawlerArtifactType::Har => <span class="string">"application/json, application/octet-stream"</span>,
CrawlerArtifactType::Warc => {
<span class="string">"application/gzip, application/octet-stream, application/json"
</span>}
};
headers.insert(ACCEPT, HeaderValue::from_static(accept));
<span class="kw">let </span>resp = <span class="self">self
</span>.send_with_retry(Method::GET, url, <span class="prelude-val">Some</span>(headers), <span class="prelude-val">None</span>)
.<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">let </span>(status, _h, body) = read_response(resp).<span class="kw">await</span><span class="question-mark">?</span>;
<span class="kw">if </span>status != <span class="number">200 </span>{
<span class="kw">return </span><span class="prelude-val">Err</span>(from_response(status, <span class="kw-2">&</span>body, <span class="number">0</span>, <span class="bool-val">true</span>));
}
<span class="prelude-val">Ok</span>(CrawlerArtifact {
artifact_type,
data: body,
})
}
<span class="comment">// ==============================================================================
// Cloud browser methods (implementations in cloud_browser.rs)
// ==============================================================================
</span><span class="doccomment">/// Fire a request through the retry loop.
</span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>send_with_retry(
<span class="kw-2">&</span><span class="self">self</span>,
method: Method,
url: Url,
headers: <span class="prelude-ty">Option</span><HeaderMap>,
body: <span class="prelude-ty">Option</span><Vec<u8>>,
) -> <span class="prelude-ty">Result</span><Response, ScrapflyError> {
<span class="kw">let </span><span class="kw-2">mut </span>last_err: <span class="prelude-ty">Option</span><ScrapflyError> = <span class="prelude-val">None</span>;
<span class="kw">for </span>attempt <span class="kw">in </span><span class="number">0</span>..DEFAULT_RETRIES {
<span class="kw">let </span><span class="kw-2">mut </span>req = <span class="self">self</span>.http.request(method.clone(), url.clone());
<span class="kw">let </span><span class="kw-2">mut </span>hmap = headers.clone().unwrap_or_default();
<span class="kw">if </span>!hmap.contains_key(USER_AGENT) {
hmap.insert(USER_AGENT, HeaderValue::from_static(SDK_USER_AGENT));
}
<span class="kw">if let </span><span class="prelude-val">Some</span>(cb) = <span class="kw-2">&</span><span class="self">self</span>.on_request {
cb(<span class="kw-2">&</span>method, <span class="kw-2">&</span>url, <span class="kw-2">&</span>hmap);
}
req = req.headers(hmap);
<span class="kw">if let </span><span class="prelude-val">Some</span>(b) = <span class="kw-2">&</span>body {
req = req.body(b.clone());
}
<span class="kw">match </span>req.send().<span class="kw">await </span>{
<span class="prelude-val">Ok</span>(resp) => {
<span class="kw">let </span>status = resp.status().as_u16();
<span class="kw">if </span>(<span class="number">500</span>..<span class="number">600</span>).contains(<span class="kw-2">&</span>status) && attempt + <span class="number">1 </span>< DEFAULT_RETRIES {
last_err = <span class="prelude-val">Some</span>(ScrapflyError::ApiServer(<span class="kw">crate</span>::error::ApiError {
message: <span class="string">"server error"</span>.into(),
http_status: status,
..Default::default()
}));
tokio::time::sleep(DEFAULT_RETRY_DELAY).<span class="kw">await</span>;
<span class="kw">continue</span>;
}
<span class="kw">return </span><span class="prelude-val">Ok</span>(resp);
}
<span class="prelude-val">Err</span>(e) => {
last_err = <span class="prelude-val">Some</span>(ScrapflyError::Transport(e));
<span class="kw">if </span>attempt + <span class="number">1 </span>< DEFAULT_RETRIES {
tokio::time::sleep(DEFAULT_RETRY_DELAY).<span class="kw">await</span>;
<span class="kw">continue</span>;
}
}
}
}
<span class="prelude-val">Err</span>(last_err.unwrap_or_else(|| ScrapflyError::Config(<span class="string">"retry loop exhausted"</span>.into())))
}
<span class="doccomment">/// Single-shot send, no retry (for `verify_api_key`/`account` style calls).
</span><span class="kw">async fn </span>send_simple(
<span class="kw-2">&</span><span class="self">self</span>,
method: Method,
url: Url,
headers: <span class="prelude-ty">Option</span><HeaderMap>,
body: <span class="prelude-ty">Option</span><Vec<u8>>,
) -> <span class="prelude-ty">Result</span><Response, ScrapflyError> {
<span class="kw">let </span><span class="kw-2">mut </span>req = <span class="self">self</span>.http.request(method.clone(), url.clone());
<span class="kw">let </span><span class="kw-2">mut </span>hmap = headers.unwrap_or_default();
<span class="kw">if </span>!hmap.contains_key(USER_AGENT) {
hmap.insert(USER_AGENT, HeaderValue::from_static(SDK_USER_AGENT));
}
<span class="kw">if let </span><span class="prelude-val">Some</span>(cb) = <span class="kw-2">&</span><span class="self">self</span>.on_request {
cb(<span class="kw-2">&</span>method, <span class="kw-2">&</span>url, <span class="kw-2">&</span>hmap);
}
req = req.headers(hmap);
<span class="kw">if let </span><span class="prelude-val">Some</span>(b) = body {
req = req.body(b);
}
req.send().<span class="kw">await</span>.map_err(ScrapflyError::Transport)
}
}
<span class="doccomment">/// Drain a response into (status, headers, body bytes) and propagate
/// `Retry-After` into the retry-ms field when present.
</span><span class="kw">async fn </span>read_response(resp: Response) -> <span class="prelude-ty">Result</span><(u16, HeaderMap, bytes::Bytes), ScrapflyError> {
<span class="kw">let </span>status = resp.status().as_u16();
<span class="kw">let </span>headers = resp.headers().clone();
<span class="kw">let </span>body = resp.bytes().<span class="kw">await</span>.map_err(ScrapflyError::Transport)<span class="question-mark">?</span>;
<span class="kw">let _ </span>= parse_retry_after(headers.get(<span class="string">"retry-after"</span>).and_then(|v| v.to_str().ok()));
<span class="prelude-val">Ok</span>((status, headers, body))
}
<span class="doccomment">/// Minimal RFC 2387 multipart/related parser — ported from
/// `sdk/go/crawler.go::parseMultipartRelated`.
</span><span class="kw">fn </span>parse_multipart_related(
body: <span class="kw-2">&</span>str,
content_type: <span class="kw-2">&</span>str,
formats: <span class="kw-2">&</span>[<span class="kw-2">&</span>str],
) -> <span class="prelude-ty">Result</span><
std::collections::BTreeMap<String, std::collections::BTreeMap<String, String>>,
ScrapflyError,
> {
<span class="kw">let </span><span class="kw-2">mut </span>boundary = String::new();
<span class="kw">for </span>part <span class="kw">in </span>content_type.split(<span class="string">';'</span>) {
<span class="kw">let </span>p = part.trim();
<span class="kw">if let </span><span class="prelude-val">Some</span>(stripped) = p.strip_prefix(<span class="string">"boundary="</span>) {
boundary = stripped.trim_matches(<span class="string">'"'</span>).to_string();
<span class="kw">break</span>;
}
}
<span class="kw">if </span>boundary.is_empty() {
<span class="kw">return </span><span class="prelude-val">Err</span>(ScrapflyError::UnexpectedResponseFormat(<span class="macro">format!</span>(
<span class="string">"multipart response has no boundary in Content-Type: {}"</span>,
content_type
)));
}
<span class="kw">let </span>delimiter = <span class="macro">format!</span>(<span class="string">"--{}"</span>, boundary);
<span class="kw">let </span><span class="kw-2">mut </span>result: std::collections::BTreeMap<String, std::collections::BTreeMap<String, String>> =
std::collections::BTreeMap::new();
<span class="kw">let </span>segments: Vec<<span class="kw-2">&</span>str> = body.split(<span class="kw-2">&</span>delimiter <span class="kw">as </span><span class="kw-2">&</span>str).collect();
<span class="kw">for </span>segment <span class="kw">in </span>segments.iter().skip(<span class="number">1</span>) {
<span class="kw">let </span><span class="kw-2">mut </span>seg = <span class="kw-2">*</span>segment;
seg = seg.trim_start_matches(<span class="string">"\r\n"</span>).trim_start_matches(<span class="string">'\n'</span>);
<span class="kw">if </span>seg.starts_with(<span class="string">"--"</span>) {
<span class="kw">break</span>;
}
seg = seg.trim_end_matches(<span class="string">"\r\n"</span>).trim_end_matches(<span class="string">'\n'</span>);
<span class="kw">let </span>(headers_raw, part_body) = <span class="kw">if let </span><span class="prelude-val">Some</span>(idx) = seg.find(<span class="string">"\r\n\r\n"</span>) {
(<span class="kw-2">&</span>seg[..idx], <span class="kw-2">&</span>seg[idx + <span class="number">4</span>..])
} <span class="kw">else if let </span><span class="prelude-val">Some</span>(idx) = seg.find(<span class="string">"\n\n"</span>) {
(<span class="kw-2">&</span>seg[..idx], <span class="kw-2">&</span>seg[idx + <span class="number">2</span>..])
} <span class="kw">else </span>{
<span class="kw">continue</span>;
};
<span class="kw">let </span><span class="kw-2">mut </span>part_url = String::new();
<span class="kw">let </span><span class="kw-2">mut </span>part_format = String::new();
<span class="kw">for </span>line <span class="kw">in </span>headers_raw.split(<span class="string">'\n'</span>) {
<span class="kw">let </span>line = line.trim_end_matches(<span class="string">'\r'</span>);
<span class="kw">if let </span><span class="prelude-val">Some</span>(colon) = line.find(<span class="string">':'</span>) {
<span class="kw">let </span>name = line[..colon].trim().to_ascii_lowercase();
<span class="kw">let </span>value = line[colon + <span class="number">1</span>..].trim().to_string();
<span class="kw">match </span>name.as_str() {
<span class="string">"content-location" </span>=> part_url = value,
<span class="string">"content-type" </span>=> part_format = infer_format_from_content_type(<span class="kw-2">&</span>value),
<span class="kw">_ </span>=> {}
}
}
}
<span class="kw">if </span>part_url.is_empty() {
<span class="kw">continue</span>;
}
<span class="kw">if </span>part_format.is_empty() {
part_format = formats.first().copied().unwrap_or(<span class="string">"html"</span>).to_string();
}
result
.entry(part_url)
.or_default()
.insert(part_format, part_body.to_string());
}
<span class="prelude-val">Ok</span>(result)
}
<span class="kw">fn </span>infer_format_from_content_type(ct: <span class="kw-2">&</span>str) -> String {
<span class="kw">let </span>lc = ct
.split(<span class="string">';'</span>)
.next()
.unwrap_or(<span class="string">""</span>)
.trim()
.to_ascii_lowercase();
<span class="kw">match </span>lc.as_str() {
<span class="string">"text/html" </span>=> <span class="string">"html"</span>.into(),
<span class="string">"text/markdown" </span>=> <span class="string">"markdown"</span>.into(),
<span class="string">"text/plain" </span>=> <span class="string">"text"</span>.into(),
<span class="string">"application/json" </span>=> <span class="string">"json"</span>.into(),
<span class="kw">_ </span>=> String::new(),
}
}
</code></pre></div></section></main></body></html>