sofie_cli/fuzz/
mod.rs

1mod report;
2
3use std::collections::BTreeMap;
4use std::num::NonZeroUsize;
5use std::path::Path;
6use std::time::Duration;
7
8use sofie_core::encoding::percent;
9use sofie_model::Method;
10use sofie_model::naming::path_parameters;
11use sofie_values::Seed;
12use sofie_values::fixture::field_json;
13
14pub use report::{Finding, FuzzReport, Outcome};
15
16use crate::gen_command::GenError;
17use crate::http_client::{Answer, CookieJars, HttpError, Request};
18use crate::inputs::{Case, clock_seed, fuzz_case, fuzz_query, valid_case};
19use crate::local_app::{LocalApp, wait_for_trees};
20use crate::targets::{Target, Targets, member_areas, targets};
21
22const DEFAULT_REQUESTS: NonZeroUsize = NonZeroUsize::new(300).expect("300 is not zero");
23const LOG_LINES: usize = 40;
24const SHOWN_LOG_LINES: usize = 20;
25/// A dropped connection is told apart from a crash once the process had time to exit.
26const EXIT_GRACE: Duration = Duration::from_millis(200);
27const MALFORMED_IDS: [&str; 5] = [
28    "not-an-id",
29    "0",
30    "-1",
31    "%2e%2e%2fadmin",
32    "00000000-0000-0000-0000-000000000000x",
33];
34const MEMBERS: [&str; 2] = ["a", "b"];
35const FOREIGN_RECORDS: usize = 3;
36
37#[derive(Clone, Copy, Debug, PartialEq, Eq)]
38pub struct FuzzOptions {
39    pub seed: u64,
40    /// A run sending nothing would report an app with no findings it never tested.
41    pub requests: NonZeroUsize,
42}
43
44impl FuzzOptions {
45    /// Without a seed each run draws another, and the report names it.
46    #[must_use]
47    pub fn new(seed: Option<u64>, requests: Option<NonZeroUsize>) -> Self {
48        Self {
49            seed: seed.unwrap_or_else(clock_seed),
50            requests: requests.unwrap_or(DEFAULT_REQUESTS),
51        }
52    }
53}
54
55/// The routes take turns; what each request sends is drawn from the seed,
56/// so a run sends the same requests again, apart from the record ids the
57/// database lists.
58pub fn fuzz(root: &Path, options: FuzzOptions) -> Result<FuzzReport, GenError> {
59    let app = sofie_model::load(root).map_err(GenError::Model)?;
60    let Targets { targets, skipped } = targets(&app)?;
61    let mut report = FuzzReport {
62        seed: options.seed,
63        sent: 0,
64        routes: targets.len(),
65        statuses: BTreeMap::new(),
66        skipped,
67        findings: Vec::new(),
68    };
69    if targets.is_empty() {
70        return Ok(report);
71    }
72    let mut run = Run::start(root, &targets)?;
73    run.give_the_other_member_records(&targets, options.seed)?;
74    let mut seed = Seed::new(options.seed);
75    for index in 0..options.requests.get() {
76        let target = &targets[index % targets.len()];
77        let planned = Planned::draw(target, &mut seed);
78        let (status, finding) = run.send(target, &planned)?;
79        if let Some(status) = status {
80            *report.statuses.entry(status).or_default() += 1;
81        }
82        if let Some(finding) = finding {
83            report.add(finding);
84        }
85        report.sent += 1;
86    }
87    Ok(report)
88}
89
90#[derive(Clone, Copy)]
91enum IdChoice {
92    Own,
93    Foreign,
94    Random,
95    Malformed(&'static str),
96}
97
98struct Planned {
99    answer: Answer,
100    signed_in: bool,
101    ids: Vec<(IdChoice, u64)>,
102    random_id: String,
103    query: String,
104    case: Option<Case>,
105}
106
107impl Planned {
108    fn draw(target: &Target, seed: &mut Seed) -> Self {
109        let answer = *seed.pick(&Answer::ALL);
110        let signed_in = target.member.is_some() && seed.between(0, 7) != 0;
111        let parameters = path_parameters(&target.path).len();
112        let ids = (0..parameters)
113            .map(|_| {
114                let choice = match seed.between(0, 3) {
115                    0 => IdChoice::Own,
116                    1 => IdChoice::Foreign,
117                    2 => IdChoice::Random,
118                    _ => IdChoice::Malformed(seed.pick(&MALFORMED_IDS)),
119                };
120                (choice, seed.draw())
121            })
122            .collect();
123        let random_id = field_json::<sofie_core::Uuid>(seed, &[])
124            .as_str()
125            .unwrap_or_default()
126            .to_owned();
127        let (query, case) = if target.method == Method::Get {
128            (fuzz_query(seed), None)
129        } else {
130            (String::new(), Some(fuzz_case(&target.fields, seed)))
131        };
132        Self {
133            answer,
134            signed_in,
135            ids,
136            random_id,
137            query,
138            case,
139        }
140    }
141}
142
143struct Run {
144    local: LocalApp,
145    jars: CookieJars,
146}
147
148impl Run {
149    fn start(root: &Path, targets: &[Target]) -> Result<Self, GenError> {
150        let local = LocalApp::start(root, "fuzz")?;
151        let mut jars = CookieJars::default();
152        for (area, paths) in member_areas(targets) {
153            for member in MEMBERS {
154                let email = format!("fuzz-{member}-{area}@example.com");
155                local.sign_in(&mut jars, &jar(area, member), paths, &email)?;
156            }
157        }
158        Ok(Self { local, jars })
159    }
160
161    /// So a foreign id names a record that exists; a refused input only
162    /// leaves the random ids.
163    fn give_the_other_member_records(&self, targets: &[Target], seed: u64) -> Result<(), GenError> {
164        let mut seed = Seed::new(seed.wrapping_add(1));
165        for target in targets.iter().filter(|target| target.is_creating()) {
166            let cookie = self.jars.cookie_header(&jar(&target.area, MEMBERS[1]));
167            for _ in 0..FOREIGN_RECORDS {
168                let case = valid_case(&target.fields, &mut seed);
169                self.local
170                    .send_json("POST", &target.path, cookie.clone(), &case.body)
171                    .map_err(|error| {
172                        GenError::App(format!("{} for the other member: {error}", target.name))
173                    })?;
174            }
175        }
176        Ok(())
177    }
178
179    /// The status answered, if any, and the finding it makes.
180    fn send(
181        &mut self,
182        target: &Target,
183        planned: &Planned,
184    ) -> Result<(Option<u16>, Option<Finding>), GenError> {
185        let (path, foreign) = self.filled_path(target, planned);
186        let mut headers = vec![("origin", self.local.origin())];
187        headers.extend(planned.answer.headers());
188        let cookie = planned
189            .signed_in
190            .then(|| self.jars.cookie_header(&jar(&target.area, MEMBERS[0])))
191            .flatten();
192        headers.extend(cookie.map(|cookie| ("cookie", cookie)));
193        let body = planned.case.as_ref().map_or("", |case| case.body.as_str());
194        if let Some(case) = &planned.case {
195            headers.push(("content-type", case.content_type.to_owned()));
196        }
197        let cursor = self.local.log.cursor();
198        let request = Request {
199            method: target.method.name(),
200            path: &path,
201            headers,
202            body: body.as_bytes(),
203        };
204        let reaches_foreign = foreign && planned.signed_in;
205        let (status, outcome, trace_id) = match self.local.send(&request) {
206            Ok(response) => {
207                let Some(outcome) = answered(response.status, &response.reason, reaches_foreign)
208                else {
209                    return Ok((Some(response.status), None));
210                };
211                (
212                    Some(response.status),
213                    outcome,
214                    response.trace_id().map(str::to_owned),
215                )
216            }
217            // Never sent, so it says nothing of the app: the fuzzer drew what it cannot send.
218            Err(HttpError::Refused(reason)) => {
219                return Err(GenError::App(format!(
220                    "sofie fuzz drew a request to {} it cannot send: {reason}",
221                    target.name
222                )));
223            }
224            Err(error) => (None, self.failed(&error)?, None),
225        };
226        let trees = trace_id
227            .as_deref()
228            .map(|trace_id| wait_for_trees(&self.local.log, trace_id))
229            .unwrap_or_default();
230        let finding = Finding {
231            route: target.name.clone(),
232            method: target.method.name(),
233            path,
234            answer: planned.answer.name(),
235            signed_in: planned.signed_in,
236            input: planned
237                .case
238                .as_ref()
239                .map(|case| (case.label.clone(), case.body.clone())),
240            outcome,
241            trace_id,
242            trees,
243            log: self.log_lines(cursor),
244            repeats: 0,
245        };
246        Ok((status, Some(finding)))
247    }
248
249    /// A panicking handler drops the connection; a crash ends the process,
250    /// which is started again so the run goes on.
251    fn failed(&mut self, error: &HttpError) -> Result<Outcome, GenError> {
252        std::thread::sleep(EXIT_GRACE);
253        let Some(status) = self.local.app.exited()? else {
254            return Ok(Outcome::NoAnswer(error.to_string()));
255        };
256        let outcome = Outcome::Exited(status.to_string());
257        self.local.restart()?;
258        Ok(outcome)
259    }
260
261    /// The path, and whether it names a record of the other member's.
262    fn filled_path(&self, target: &Target, planned: &Planned) -> (String, bool) {
263        let mut path = target.path.clone();
264        let mut foreign = false;
265        for (parameter, (choice, pick)) in
266            path_parameters(&target.path).into_iter().zip(&planned.ids)
267        {
268            let marker = format!("{{{parameter}}}");
269            let collection = path[..path.find(&marker).unwrap_or(0)]
270                .trim_end_matches('/')
271                .to_owned();
272            let id = match choice {
273                IdChoice::Malformed(id) => (*id).to_owned(),
274                IdChoice::Random => planned.random_id.clone(),
275                IdChoice::Own | IdChoice::Foreign => {
276                    let own = matches!(choice, IdChoice::Own);
277                    match pick_id(&self.listed_ids(target, &collection, own), *pick) {
278                        Some(id) => {
279                            foreign |= !own;
280                            path_segment(&id)
281                        }
282                        None => planned.random_id.clone(),
283                    }
284                }
285            };
286            path = path.replacen(&marker, &id, 1);
287        }
288        (format!("{path}{}", planned.query), foreign)
289    }
290
291    /// The other member's records are foreign: what this member must not reach.
292    fn listed_ids(&self, target: &Target, collection: &str, own: bool) -> Vec<String> {
293        let own_ids = self.ids_at(collection, &jar(&target.area, MEMBERS[0]));
294        if own {
295            return own_ids;
296        }
297        self.ids_at(collection, &jar(&target.area, MEMBERS[1]))
298            .into_iter()
299            .filter(|id| !own_ids.contains(id))
300            .collect()
301    }
302
303    fn ids_at(&self, collection: &str, jar: &str) -> Vec<String> {
304        let cookie = self.jars.cookie_header(jar);
305        let Ok(response) = self.local.send_json("GET", collection, cookie, "") else {
306            return Vec::new();
307        };
308        let rows: serde_json::Value = serde_json::from_slice(&response.body).unwrap_or_default();
309        listed_ids(&rows)
310    }
311
312    fn log_lines(&self, cursor: u64) -> Vec<String> {
313        self.local
314            .log
315            .after(cursor, None, LOG_LINES)
316            .lines
317            .into_iter()
318            .filter_map(|line| line.split_once(' ').map(|(_, text)| text.to_owned()))
319            .filter(|text| !text.starts_with(' ') && !text.contains(" sofie.dev: trace_id="))
320            .take(SHOWN_LOG_LINES)
321            .collect()
322    }
323}
324
325/// A server error, or a success that let one member reach another's record, is a finding.
326fn answered(status: u16, reason: &str, reaches_foreign: bool) -> Option<Outcome> {
327    if status >= 500 {
328        return Some(Outcome::Status(status, reason.to_owned()));
329    }
330    let success = (200..300).contains(&status);
331    (success && reaches_foreign).then(|| Outcome::ForeignRecord(status, reason.to_owned()))
332}
333
334fn listed_ids(rows: &serde_json::Value) -> Vec<String> {
335    rows.as_array()
336        .into_iter()
337        .flatten()
338        .filter_map(|row| match &row["id"] {
339            serde_json::Value::String(id) => Some(id.clone()),
340            serde_json::Value::Number(id) => Some(id.to_string()),
341            _ => None,
342        })
343        .collect()
344}
345
346/// An id the app listed reaches the route as it is, whatever it holds.
347fn path_segment(id: &str) -> String {
348    percent::encode(id, percent::is_unreserved)
349}
350
351fn jar(area: &str, member: &str) -> String {
352    format!("{area}/{member}")
353}
354
355fn pick_id(ids: &[String], pick: u64) -> Option<String> {
356    let count = u64::try_from(ids.len()).ok().filter(|count| *count > 0)?;
357    let index = usize::try_from(pick % count).ok()?;
358    ids.get(index).cloned()
359}
360
361#[cfg(test)]
362mod tests {
363    use serde_json::json;
364
365    use super::{Outcome, answered, listed_ids, path_segment};
366
367    #[test]
368    fn a_success_is_a_finding_only_when_it_reached_another_member_s_record() {
369        assert_eq!(
370            answered(200, "OK", true),
371            Some(Outcome::ForeignRecord(200, "OK".to_owned()))
372        );
373        assert_eq!(answered(200, "OK", false), None);
374        assert_eq!(answered(404, "Not Found", true), None);
375        assert_eq!(
376            answered(500, "Internal Server Error", false),
377            Some(Outcome::Status(500, "Internal Server Error".to_owned()))
378        );
379    }
380
381    #[test]
382    fn a_listed_id_is_a_string_or_a_number() {
383        let rows = json!([{ "id": "a1" }, { "id": 42 }, { "id": null }, { "name": "x" }]);
384
385        assert_eq!(listed_ids(&rows), ["a1", "42"]);
386    }
387
388    #[test]
389    fn a_listed_id_is_sent_as_one_path_segment() {
390        assert_eq!(path_segment("a b/ç"), "a%20b%2F%C3%A7");
391        assert_eq!(
392            path_segment("0190a6c4-7b8e-7000-8000-000000000000"),
393            "0190a6c4-7b8e-7000-8000-000000000000"
394        );
395    }
396}
sofie_cli/fuzz/mod.rs

sofie_cli/fuzz/
mod.rs
