Skip to main content

code_system_graph_core/
client_flows.rs

1//! Repository-level composition of HTTP client calls through wrappers, helpers, and fixtures.
2//!
3//! Extractors record client calls whose URL depends on parameters of the enclosing function, and
4//! the calls between functions with their string arguments. Composition first instantiates those
5//! wrapper URLs at every call site that binds them, also through wrappers of wrappers. It then
6//! attributes to each test every client call reached through the helpers and fixtures it calls,
7//! so that a test is linked to the endpoints it exercises indirectly.
8//!
9//! A client call issued through a parameter of its function, such as a pytest `client` fixture,
10//! is confirmed only when the parameter is a fixture returning an in-process test client, or is
11//! passed such a fixture by every resolved caller path that reaches it.
12
13use std::collections::{BTreeMap, BTreeSet};
14
15use crate::repository_symbols::{
16    RepositorySourceFile, RepositorySymbols, SymbolFile, SymbolKey, files_by_repository
17};
18use crate::source_http::instantiated_consumer;
19use crate::url_template::BoundArguments;
20use crate::{
21    SourceEpistemicStatus, SourceFramework, SourceObservation, SourceRole, SymbolRef, UrlPart, UrlTemplate
22};
23
24/// Rounds of wrapper instantiation, which is also the longest chain of wrappers that pass a URL
25/// parameter through to another wrapper.
26const MAX_WRAPPER_HOPS: usize = 3;
27/// Longest call chain followed from a test to the client calls of its helpers.
28const MAX_TEST_CALL_DEPTH: usize = 3;
29/// Most unresolved wrapper URLs retained for one function.
30const MAX_OPEN_URLS: usize = 16;
31
32/// A named function of one file, identified by the file's index within its repository.
33type Function<'a> = (usize, &'a str);
34
35/// Composes client calls through wrappers, helpers, and fixtures.
36///
37/// The result has one observation list per input file, in input order. Call observations are
38/// consumed; each function that binds a wrapper URL to an exact path gains a consumer
39/// observation, and each test gains the client calls of the helpers and fixtures it reaches.
40#[must_use]
41pub fn compose_client_flows(files: &[RepositorySourceFile<'_>]) -> Vec<Vec<SourceObservation>> {
42    let mut output = files
43        .iter()
44        .map(|file| file.observations.to_vec())
45        .collect::<Vec<_>>();
46    for members in files_by_repository(files).into_values() {
47        let composable = members.iter().any(|&index| {
48            files[index]
49                .observations
50                .iter()
51                .any(|observation| observation.role == SourceRole::Call || is_received(observation))
52        });
53        if !composable {
54            continue;
55        }
56        let flows = RepositoryFlows::new(files, &members);
57        for (local, additions) in flows.compose() {
58            output[members[local]].extend(additions);
59        }
60    }
61    for observations in &mut output {
62        observations.retain(|observation| {
63            !matches!(observation.role, SourceRole::Call | SourceRole::Client)
64                && !is_received(observation)
65        });
66    }
67    output
68}
69
70/// Whether `observation` is a client call issued through a parameter of its function.
71fn is_received(observation: &SourceObservation) -> bool {
72    observation.role == SourceRole::Consumer
73        && matches!(observation.router, Some(SymbolRef::Parameter { .. }))
74}
75
76/// A client call whose URL still depends on parameters of the function issuing it.
77#[derive(Clone)]
78struct OpenUrl<'a> {
79    origin: &'a SourceObservation,
80    url: UrlTemplate,
81}
82
83struct RepositoryFlows<'a> {
84    symbols: RepositorySymbols<'a>,
85    functions: BTreeMap<Vec<String>, Option<Function<'a>>>,
86    locals: BTreeMap<(usize, String), Function<'a>>,
87    consumers: BTreeMap<Function<'a>, Vec<&'a SourceObservation>>,
88    calls: BTreeMap<Function<'a>, Vec<&'a SourceObservation>>,
89    callers: BTreeMap<Function<'a>, Vec<(Function<'a>, &'a SourceObservation)>>,
90    tests: BTreeSet<Function<'a>>,
91    clients: BTreeMap<(usize, String), SourceFramework>,
92    received: BTreeMap<Function<'a>, Vec<SourceObservation>>,
93}
94
95impl<'a> RepositoryFlows<'a> {
96    fn new(files: &'a [RepositorySourceFile<'a>], members: &[usize]) -> Self {
97        let mut symbols = RepositorySymbols::new(
98            members
99                .iter()
100                .map(|&index| SymbolFile {
101                    path: files[index].path,
102                    observations: files[index].observations,
103                })
104                .collect(),
105        );
106        let mut consumers = BTreeMap::<Function<'a>, Vec<&'a SourceObservation>>::new();
107        let mut calls = BTreeMap::<Function<'a>, Vec<&'a SourceObservation>>::new();
108        let mut tests = BTreeSet::new();
109        let mut clients = BTreeMap::new();
110        for (local, &index) in members.iter().enumerate() {
111            for observation in files[index].observations {
112                let Some(name) = observation.symbol_name.as_deref() else {
113                    continue;
114                };
115                match observation.role {
116                    SourceRole::Consumer => consumers
117                        .entry((local, name))
118                        .or_default()
119                        .push(observation),
120                    SourceRole::Call => calls.entry((local, name)).or_default().push(observation),
121                    SourceRole::Test => {
122                        tests.insert((local, name));
123                    }
124                    SourceRole::Client => {
125                        clients.insert((local, name.to_owned()), observation.framework);
126                    }
127                    SourceRole::Provider | SourceRole::Factory | SourceRole::Mount => {}
128                }
129            }
130        }
131        let mut functions = BTreeMap::<Vec<String>, Option<Function<'a>>>::new();
132        let mut locals = BTreeMap::new();
133        for &function in consumers.keys().chain(calls.keys()) {
134            locals.insert((function.0, function.1.to_owned()), function);
135            let key = symbols.declare_function(function.0, function.1);
136            functions
137                .entry(key)
138                .and_modify(|declared| {
139                    if *declared != Some(function) {
140                        *declared = None;
141                    }
142                })
143                .or_insert(Some(function));
144        }
145        let mut flows = Self {
146            symbols,
147            functions,
148            locals,
149            consumers,
150            calls,
151            callers: BTreeMap::new(),
152            tests,
153            clients,
154            received: BTreeMap::new(),
155        };
156        for (&caller, calls) in &flows.calls {
157            for &call in calls {
158                if let Some(callee) = flows.callee(caller.0, call) {
159                    flows
160                        .callers
161                        .entry(callee)
162                        .or_default()
163                        .push((caller, call));
164                }
165            }
166        }
167        flows.received = flows.resolve_received();
168        flows
169    }
170
171    /// Client calls issued through a parameter that receives an in-process test client.
172    fn resolve_received(&self) -> BTreeMap<Function<'a>, Vec<SourceObservation>> {
173        let mut received = BTreeMap::<Function<'a>, Vec<SourceObservation>>::new();
174        for (&function, consumers) in &self.consumers {
175            for &consumer in consumers {
176                let Some(SymbolRef::Parameter { name, index }) = &consumer.router else {
177                    continue;
178                };
179                let Some(framework) = self.received_client(function, name, *index, 0) else {
180                    continue;
181                };
182                let mut confirmed = consumer.clone();
183                confirmed.framework = framework;
184                confirmed.router = None;
185                if confirmed.method.is_some() && confirmed.path.is_some() {
186                    confirmed.status = SourceEpistemicStatus::Confirmed;
187                    confirmed.confidence = 1.0;
188                }
189                received.entry(function).or_default().push(confirmed);
190            }
191        }
192        received
193    }
194
195    /// Test client framework passed to parameter `name` at position `index` of `function`.
196    fn received_client(
197        &self,
198        function: Function<'a>,
199        name: &str,
200        index: usize,
201        depth: usize,
202    ) -> Option<SourceFramework> {
203        let fixture = SymbolRef::Fixture(name.to_owned());
204        let requests_fixture = self.calls.get(&function).is_some_and(|calls| {
205            calls.iter().any(|call| {
206                call.call
207                    .as_ref()
208                    .is_some_and(|call| call.callee == fixture)
209            })
210        });
211        if requests_fixture {
212            let Some(SymbolKey::Local { file, name }) = self.symbols.resolve(function.0, &fixture)
213            else {
214                return None;
215            };
216            return self.clients.get(&(file, name)).copied();
217        }
218        if depth >= MAX_TEST_CALL_DEPTH {
219            return None;
220        }
221        let mut frameworks = self.callers.get(&function)?.iter().map(|(caller, call)| {
222            let arguments = bound_arguments(call);
223            let argument = arguments
224                .by_name
225                .get(name)
226                .or_else(|| arguments.by_index.get(&index))?;
227            match argument.parts.as_slice() {
228                [UrlPart::Parameter { name, index }] => {
229                    self.received_client(*caller, name, *index, depth + 1)
230                }
231                _ => None,
232            }
233        });
234        let first = frameworks.next()??;
235        frameworks
236            .all(|framework| framework == Some(first))
237            .then_some(first)
238    }
239
240    /// Function called by `call` from file `local`, when it is known in this repository.
241    fn callee(&self, local: usize, call: &SourceObservation) -> Option<Function<'a>> {
242        let reference = &call.call.as_ref()?.callee;
243        match self.symbols.resolve(local, reference)? {
244            SymbolKey::Local { file, name } => self.locals.get(&(file, name)).copied(),
245            SymbolKey::Function(key) => self.functions.get(&key).copied().flatten(),
246        }
247    }
248
249    /// Observations added to each file, keyed by the file's index within the repository.
250    fn compose(&self) -> BTreeMap<usize, Vec<SourceObservation>> {
251        let resolved = self.instantiate_wrappers();
252        let mut additions = BTreeMap::<usize, Vec<SourceObservation>>::new();
253        for (function, observations) in self.received.iter().chain(&resolved) {
254            additions
255                .entry(function.0)
256                .or_default()
257                .extend(observations.iter().cloned());
258        }
259        for &test in &self.tests {
260            let mut seen = self
261                .exact_consumers(test, &resolved)
262                .into_iter()
263                .map(consumer_identity)
264                .collect::<BTreeSet<_>>();
265            for observation in self.reached_consumers(test, &resolved) {
266                if seen.insert(consumer_identity(&observation)) {
267                    additions.entry(test.0).or_default().push(observation);
268                }
269            }
270        }
271        additions
272    }
273
274    /// Consumers obtained by binding wrapper URLs at call sites, keyed by the calling function.
275    fn instantiate_wrappers(&self) -> BTreeMap<Function<'a>, Vec<SourceObservation>> {
276        let mut open = BTreeMap::<Function<'a>, Vec<OpenUrl<'a>>>::new();
277        for (&function, consumers) in &self.consumers {
278            for &origin in consumers {
279                if let (Some(url), None) = (&origin.url, &origin.path) {
280                    open.entry(function).or_default().push(OpenUrl {
281                        origin,
282                        url: url.clone(),
283                    });
284                }
285            }
286        }
287        let mut resolved = BTreeMap::<Function<'a>, Vec<SourceObservation>>::new();
288        for _ in 0..MAX_WRAPPER_HOPS {
289            let snapshot = open.clone();
290            let mut changed = false;
291            for (&caller, calls) in &self.calls {
292                for &call in calls {
293                    let Some(callee) = self
294                        .callee(caller.0, call)
295                        .filter(|callee| *callee != caller)
296                    else {
297                        continue;
298                    };
299                    let Some(wrappers) = snapshot.get(&callee) else {
300                        continue;
301                    };
302                    let arguments = bound_arguments(call);
303                    for wrapper in wrappers {
304                        let url = wrapper.url.bind(&arguments);
305                        if let Some(consumer) =
306                            instantiated_consumer(wrapper.origin, &url, caller.1, call.lines)
307                        {
308                            let consumers = resolved.entry(caller).or_default();
309                            let identity = consumer_identity(&consumer);
310                            if !consumers
311                                .iter()
312                                .any(|known| consumer_identity(known) == identity)
313                            {
314                                consumers.push(consumer);
315                                changed = true;
316                            }
317                        } else if url.has_parameters() {
318                            let urls = open.entry(caller).or_default();
319                            if urls.len() < MAX_OPEN_URLS
320                                && !urls.iter().any(|known| known.url == url)
321                            {
322                                urls.push(OpenUrl {
323                                    origin: wrapper.origin,
324                                    url,
325                                });
326                                changed = true;
327                            }
328                        }
329                    }
330                }
331            }
332            if !changed {
333                break;
334            }
335        }
336        resolved
337    }
338
339    /// Exact consumers issued by `function`, directly or through an instantiated wrapper.
340    fn exact_consumers<'r>(
341        &'r self,
342        function: Function<'a>,
343        resolved: &'r BTreeMap<Function<'a>, Vec<SourceObservation>>,
344    ) -> Vec<&'r SourceObservation>
345    where
346        'a: 'r,
347    {
348        self.consumers
349            .get(&function)
350            .into_iter()
351            .flatten()
352            .copied()
353            .filter(|observation| {
354                observation.status == SourceEpistemicStatus::Confirmed
355                    && observation.method.is_some()
356                    && observation.path.is_some()
357            })
358            .chain(self.received.get(&function).into_iter().flatten())
359            .chain(resolved.get(&function).into_iter().flatten())
360            .collect()
361    }
362
363    /// Exact consumers of the functions reached from `test`, attributed to `test` at the call
364    /// site in the test through which each one is reached.
365    fn reached_consumers(
366        &self,
367        test: Function<'a>,
368        resolved: &BTreeMap<Function<'a>, Vec<SourceObservation>>,
369    ) -> Vec<SourceObservation> {
370        let mut reached = Vec::new();
371        let mut visited = BTreeSet::from([test]);
372        let mut frontier = self
373            .calls
374            .get(&test)
375            .into_iter()
376            .flatten()
377            .filter_map(|call| Some((self.callee(test.0, call)?, call.lines)))
378            .collect::<Vec<_>>();
379        for _ in 0..MAX_TEST_CALL_DEPTH {
380            let mut next = Vec::new();
381            for (function, lines) in frontier {
382                if !visited.insert(function) {
383                    continue;
384                }
385                for consumer in self.exact_consumers(function, resolved) {
386                    let mut attributed = consumer.clone();
387                    attributed.symbol_name = Some(test.1.to_owned());
388                    attributed.lines = lines;
389                    attributed.url = None;
390                    reached.push(attributed);
391                }
392                next.extend(
393                    self.calls
394                        .get(&function)
395                        .into_iter()
396                        .flatten()
397                        .filter_map(|call| Some((self.callee(function.0, call)?, lines))),
398                );
399            }
400            frontier = next;
401        }
402        reached
403    }
404}
405
406/// Arguments of `call` keyed by keyword, or by position among the positional arguments.
407fn bound_arguments(call: &SourceObservation) -> BoundArguments {
408    let mut bound = BoundArguments::default();
409    let mut position = 0;
410    for argument in call.call.iter().flat_map(|call| &call.arguments) {
411        match (&argument.keyword, &argument.value) {
412            (Some(keyword), Some(value)) => {
413                bound.by_name.insert(keyword.clone(), value.clone());
414            }
415            (Some(_), None) => {}
416            (None, value) => {
417                if let Some(value) = value {
418                    bound.by_index.insert(position, value.clone());
419                }
420                position += 1;
421            }
422        }
423    }
424    bound
425}
426
427/// Method, path, and authority of a consumer.
428type ConsumerIdentity = (Option<String>, Option<String>, Option<String>);
429
430fn consumer_identity(observation: &SourceObservation) -> ConsumerIdentity {
431    (
432        observation.method.clone(),
433        observation.path.clone(),
434        observation.authority.clone(),
435    )
436}
437
438#[cfg(test)]
439mod tests {
440    use code_system_graph_model::RepoId;
441
442    use super::compose_client_flows;
443    use crate::{
444        RepositorySourceFile, SourceObservation, SourceRole, parse_python_source, parse_typescript_source
445    };
446
447    /// Consumers after composition as `path: symbol METHOD authority path`.
448    fn composed_consumers(files: &[(&str, Vec<SourceObservation>)]) -> Vec<String> {
449        let repo = RepoId::new("repo:shop");
450        let inputs = files
451            .iter()
452            .map(|(path, observations)| RepositorySourceFile {
453                repo_id: &repo,
454                path,
455                observations,
456            })
457            .collect::<Vec<_>>();
458        let composed = compose_client_flows(&inputs);
459        assert!(
460            composed
461                .iter()
462                .flatten()
463                .all(|item| item.role != SourceRole::Call)
464        );
465        files
466            .iter()
467            .zip(composed)
468            .flat_map(|((path, _), observations)| {
469                observations
470                    .into_iter()
471                    .filter(|item| item.role == SourceRole::Consumer && item.path.is_some())
472                    .map(move |item| {
473                        format!(
474                            "{path}: {} {} {} {}",
475                            item.symbol_name.unwrap_or_default(),
476                            item.method.unwrap_or_default(),
477                            item.authority.unwrap_or_default(),
478                            item.path.unwrap_or_default()
479                        )
480                    })
481            })
482            .collect()
483    }
484
485    #[test]
486    fn python_tests_should_reach_fixture_and_wrapped_helper_calls_across_files() {
487        let conftest = r#"
488import pytest
489import requests
490BASE = "http://orders:8000"
491
492@pytest.fixture
493def order():
494    return requests.post(BASE + "/orders", json={}).json()
495"#;
496        let helpers = r#"
497import requests
498API = "http://orders:8000/api"
499
500def get_json(path):
501    return requests.get(API + path).json()
502
503def read_order(order_id):
504    return get_json(f"/orders/{order_id}")
505"#;
506        let test = r#"
507from tests.helpers import read_order
508
509def test_read(order):
510    read_order(order["id"])
511"#;
512        let consumers = composed_consumers(&[
513            ("tests/conftest.py", parse_python_source(conftest)),
514            ("tests/helpers.py", parse_python_source(helpers)),
515            ("tests/test_orders.py", parse_python_source(test)),
516        ]);
517
518        assert_eq!(
519            consumers,
520            [
521                "tests/conftest.py: order POST orders:8000 /orders",
522                "tests/helpers.py: read_order GET orders:8000 /api/orders/{order_id}",
523                "tests/test_orders.py: test_read POST orders:8000 /orders",
524                "tests/test_orders.py: test_read GET orders:8000 /api/orders/{order_id}",
525            ]
526        );
527    }
528
529    #[test]
530    fn python_parameter_clients_should_resolve_through_test_client_fixtures() {
531        let conftest = r"
532import pytest
533from fastapi.testclient import TestClient
534from app.main import app
535
536@pytest.fixture
537def client():
538    with TestClient(app) as test_client:
539        yield test_client
540";
541        let helpers = r#"
542def create_order(client, sku):
543    return client.post("/orders", json={"sku": sku})
544
545def health(session):
546    return session.get("/health")
547"#;
548        let test = r#"
549from tests.helpers import create_order, health
550
551def test_create(client):
552    create_order(client, "A-1")
553
554def test_read(client):
555    client.get("/orders/42")
556
557def test_health():
558    health(object())
559"#;
560        let files = [
561            ("tests/conftest.py", parse_python_source(conftest)),
562            ("tests/helpers.py", parse_python_source(helpers)),
563            ("tests/test_orders.py", parse_python_source(test)),
564        ];
565        let consumers = composed_consumers(&files);
566
567        assert_eq!(
568            consumers,
569            [
570                "tests/helpers.py: create_order POST  /orders",
571                "tests/test_orders.py: test_read GET  /orders/42",
572                "tests/test_orders.py: test_create POST  /orders",
573            ]
574        );
575    }
576
577    #[test]
578    fn script_wrappers_should_be_instantiated_at_call_sites_in_other_modules() {
579        let api = r#"
580const API = "/api";
581export function request(path: string) {
582  return fetch(API + path);
583}
584"#;
585        let orders = r#"
586import { request } from "./api";
587export function getOrder(id: string) {
588  return request(`/orders/${id}`);
589}
590"#;
591        let consumers = composed_consumers(&[
592            ("web/src/api.ts", parse_typescript_source(api)),
593            ("web/src/orders.ts", parse_typescript_source(orders)),
594        ]);
595
596        assert_eq!(
597            consumers,
598            ["web/src/orders.ts: getOrder GET  /api/orders/{id}"]
599        );
600    }
601}