Skip to main content

code_system_graph_core/
client_flows.rs

1//! Repository-level composition of HTTP client calls through wrappers, helpers, and fixtures.
2//!
3//! Extractors record client calls whose URL depends on parameters of the enclosing function, and
4//! the calls between functions with their string arguments. Composition first instantiates those
5//! wrapper URLs at every call site that binds them, also through wrappers of wrappers. It then
6//! attributes to each test every client call reached through the helpers and fixtures it calls,
7//! so that a test is linked to the endpoints it exercises indirectly.
8//!
9//! A client call issued through a parameter of its function, such as a pytest `client` fixture,
10//! is confirmed only when the parameter is a fixture returning an in-process test client, or is
11//! passed such a fixture by every resolved caller path that reaches it.
12
13use std::collections::{BTreeMap, BTreeSet};
14
15use crate::repository_symbols::{
16    RepositorySourceFile, RepositorySymbols, SymbolFile, SymbolKey, files_by_repository
17};
18use crate::source_http::instantiated_consumer;
19use crate::url_template::BoundArguments;
20use crate::{
21    SourceEpistemicStatus, SourceFramework, SourceObservation, SourceRole, SymbolRef, UrlPart, UrlTemplate
22};
23
24/// Rounds of wrapper instantiation, which is also the longest chain of wrappers that pass a URL
25/// parameter through to another wrapper.
26const MAX_WRAPPER_HOPS: usize = 3;
27/// Longest call chain followed from a test to the client calls of its helpers.
28const MAX_TEST_CALL_DEPTH: usize = 3;
29/// Most unresolved wrapper URLs retained for one function.
30const MAX_OPEN_URLS: usize = 16;
31
32/// A named function of one file, identified by the file's index within its repository.
33type Function<'a> = (usize, &'a str);
34
35/// Composes client calls through wrappers, helpers, and fixtures.
36///
37/// The result has one observation list per input file, in input order. Call observations are
38/// consumed; each function that binds a wrapper URL to an exact path gains a consumer
39/// observation, and each test gains the client calls of the helpers and fixtures it reaches.
40#[must_use]
41pub fn compose_client_flows(files: &[RepositorySourceFile<'_>]) -> Vec<Vec<SourceObservation>> {
42    let mut output = files
43        .iter()
44        .map(|file| file.observations.to_vec())
45        .collect::<Vec<_>>();
46    for members in files_by_repository(files).into_values() {
47        let composable = members.iter().any(|&index| {
48            files[index]
49                .observations
50                .iter()
51                .any(|observation| observation.role == SourceRole::Call || is_received(observation))
52        });
53        if !composable {
54            continue;
55        }
56        let flows = RepositoryFlows::new(files, &members);
57        for (local, additions) in flows.compose() {
58            output[members[local]].extend(additions);
59        }
60    }
61    for observations in &mut output {
62        observations.retain(|observation| {
63            !matches!(observation.role, SourceRole::Call | SourceRole::Client)
64                && !is_received(observation)
65        });
66    }
67    output
68}
69
70/// Whether `observation` is a client call issued through a parameter of its function.
71fn is_received(observation: &SourceObservation) -> bool {
72    observation.role == SourceRole::Consumer
73        && matches!(observation.router, Some(SymbolRef::Parameter { .. }))
74}
75
76/// A client call whose URL still depends on parameters of the function issuing it.
77#[derive(Clone)]
78struct OpenUrl<'a> {
79    origin: &'a SourceObservation,
80    url: UrlTemplate,
81}
82
83struct RepositoryFlows<'a> {
84    symbols: RepositorySymbols<'a>,
85    functions: BTreeMap<Vec<String>, Option<Function<'a>>>,
86    locals: BTreeMap<(usize, String), Function<'a>>,
87    consumers: BTreeMap<Function<'a>, Vec<&'a SourceObservation>>,
88    calls: BTreeMap<Function<'a>, Vec<&'a SourceObservation>>,
89    callers: BTreeMap<Function<'a>, Vec<(Function<'a>, &'a SourceObservation)>>,
90    tests: BTreeSet<Function<'a>>,
91    clients: BTreeMap<(usize, String), SourceFramework>,
92    received: BTreeMap<Function<'a>, Vec<SourceObservation>>,
93}
94
95impl<'a> RepositoryFlows<'a> {
96    fn new(files: &'a [RepositorySourceFile<'a>], members: &[usize]) -> Self {
97        let mut symbols = RepositorySymbols::new(
98            members
99                .iter()
100                .map(|&index| SymbolFile {
101                    path: files[index].path,
102                    observations: files[index].observations,
103                })
104                .collect(),
105        );
106        let mut consumers = BTreeMap::<Function<'a>, Vec<&'a SourceObservation>>::new();
107        let mut calls = BTreeMap::<Function<'a>, Vec<&'a SourceObservation>>::new();
108        let mut tests = BTreeSet::new();
109        let mut clients = BTreeMap::new();
110        for (local, &index) in members.iter().enumerate() {
111            for observation in files[index].observations {
112                let Some(name) = observation.symbol_name.as_deref() else {
113                    continue;
114                };
115                match observation.role {
116                    SourceRole::Consumer => consumers
117                        .entry((local, name))
118                        .or_default()
119                        .push(observation),
120                    SourceRole::Call => calls.entry((local, name)).or_default().push(observation),
121                    SourceRole::Test => {
122                        tests.insert((local, name));
123                    }
124                    SourceRole::Client => {
125                        clients.insert((local, name.to_owned()), observation.framework);
126                    }
127                    SourceRole::Provider | SourceRole::Factory | SourceRole::Mount => {}
128                }
129            }
130        }
131        let mut functions = BTreeMap::<Vec<String>, Option<Function<'a>>>::new();
132        let mut locals = BTreeMap::new();
133        for &function in consumers.keys().chain(calls.keys()) {
134            locals.insert((function.0, function.1.to_owned()), function);
135            let key = symbols.declare_function(function.0, function.1);
136            functions
137                .entry(key)
138                .and_modify(|declared| {
139                    if *declared != Some(function) {
140                        *declared = None;
141                    }
142                })
143                .or_insert(Some(function));
144        }
145        let mut flows = Self {
146            symbols,
147            functions,
148            locals,
149            consumers,
150            calls,
151            callers: BTreeMap::new(),
152            tests,
153            clients,
154            received: BTreeMap::new(),
155        };
156        for (&caller, calls) in &flows.calls {
157            for &call in calls {
158                if let Some(callee) = flows.callee(caller.0, call) {
159                    flows
160                        .callers
161                        .entry(callee)
162                        .or_default()
163                        .push((caller, call));
164                }
165            }
166        }
167        flows.received = flows.resolve_received();
168        flows
169    }
170
171    /// Client calls issued through a parameter that receives an in-process test client.
172    fn resolve_received(&self) -> BTreeMap<Function<'a>, Vec<SourceObservation>> {
173        let mut received = BTreeMap::<Function<'a>, Vec<SourceObservation>>::new();
174        for (&function, consumers) in &self.consumers {
175            for &consumer in consumers {
176                let Some(SymbolRef::Parameter { name, index }) = &consumer.router else {
177                    continue;
178                };
179                let Some(framework) = self.received_client(function, name, *index, 0) else {
180                    continue;
181                };
182                let mut confirmed = consumer.clone();
183                confirmed.framework = framework;
184                confirmed.router = None;
185                if confirmed.method.is_some() && confirmed.path.is_some() {
186                    confirmed.status = SourceEpistemicStatus::Confirmed;
187                    confirmed.confidence = 1.0;
188                }
189                received.entry(function).or_default().push(confirmed);
190            }
191        }
192        received
193    }
194
195    /// Test client framework passed to parameter `name` at position `index` of `function`.
196    fn received_client(
197        &self,
198        function: Function<'a>,
199        name: &str,
200        index: usize,
201        depth: usize,
202    ) -> Option<SourceFramework> {
203        let fixture = SymbolRef::Fixture(name.to_owned());
204        let requests_fixture = self.calls.get(&function).is_some_and(|calls| {
205            calls.iter().any(|call| {
206                call.call
207                    .as_ref()
208                    .is_some_and(|call| call.callee == fixture)
209            })
210        });
211        if requests_fixture {
212            let Some(SymbolKey::Local { file, name }) = self.symbols.resolve(function.0, &fixture)
213            else {
214                return None;
215            };
216            return self.clients.get(&(file, name)).copied();
217        }
218        if depth >= MAX_TEST_CALL_DEPTH {
219            return None;
220        }
221        let mut frameworks = self.callers.get(&function)?.iter().map(|(caller, call)| {
222            let arguments = bound_arguments(call);
223            let argument = arguments
224                .by_name
225                .get(name)
226                .or_else(|| arguments.by_index.get(&index))?;
227            match argument.parts.as_slice() {
228                [UrlPart::Parameter { name, index }] => {
229                    self.received_client(*caller, name, *index, depth + 1)
230                }
231                _ => None,
232            }
233        });
234        let first = frameworks.next()??;
235        frameworks
236            .all(|framework| framework == Some(first))
237            .then_some(first)
238    }
239
240    /// Function called by `call` from file `local`, when it is known in this repository.
241    fn callee(&self, local: usize, call: &SourceObservation) -> Option<Function<'a>> {
242        let reference = &call.call.as_ref()?.callee;
243        match self.symbols.resolve(local, reference)? {
244            SymbolKey::Local { file, name } => self.locals.get(&(file, name)).copied(),
245            SymbolKey::Function(key) => self.functions.get(&key).copied().flatten(),
246        }
247    }
248
249    /// Observations added to each file, keyed by the file's index within the repository.
250    fn compose(&self) -> BTreeMap<usize, Vec<SourceObservation>> {
251        let resolved = self.instantiate_wrappers();
252        let mut additions = BTreeMap::<usize, Vec<SourceObservation>>::new();
253        for (function, observations) in self.received.iter().chain(&resolved) {
254            additions
255                .entry(function.0)
256                .or_default()
257                .extend(observations.iter().cloned());
258        }
259        for &test in &self.tests {
260            let mut seen = self
261                .exact_consumers(test, &resolved)
262                .into_iter()
263                .map(consumer_identity)
264                .collect::<BTreeSet<_>>();
265            for observation in self.reached_consumers(test, &resolved) {
266                if seen.insert(consumer_identity(&observation)) {
267                    additions.entry(test.0).or_default().push(observation);
268                }
269            }
270        }
271        additions
272    }
273
274    /// Consumers obtained by binding wrapper URLs at call sites, keyed by the calling function.
275    fn instantiate_wrappers(&self) -> BTreeMap<Function<'a>, Vec<SourceObservation>> {
276        let mut open = BTreeMap::<Function<'a>, Vec<OpenUrl<'a>>>::new();
277        for (&function, consumers) in &self.consumers {
278            for &origin in consumers {
279                if let (Some(url), None) = (&origin.url, &origin.path) {
280                    open.entry(function).or_default().push(OpenUrl {
281                        origin,
282                        url: url.clone(),
283                    });
284                }
285            }
286        }
287        let mut resolved = BTreeMap::<Function<'a>, Vec<SourceObservation>>::new();
288        for _ in 0..MAX_WRAPPER_HOPS {
289            let snapshot = open.clone();
290            let mut changed = false;
291            for (&caller, calls) in &self.calls {
292                for &call in calls {
293                    let Some(callee) = self
294                        .callee(caller.0, call)
295                        .filter(|callee| *callee != caller)
296                    else {
297                        continue;
298                    };
299                    let Some(wrappers) = snapshot.get(&callee) else {
300                        continue;
301                    };
302                    let arguments = bound_arguments(call);
303                    for wrapper in wrappers {
304                        let url = wrapper.url.bind(&arguments);
305                        if let Some(mut consumer) =
306                            instantiated_consumer(wrapper.origin, &url, caller.1, call.lines)
307                        {
308                            consumer.symbol_start_line = call.symbol_start_line;
309                            let consumers = resolved.entry(caller).or_default();
310                            let identity = consumer_identity(&consumer);
311                            if !consumers
312                                .iter()
313                                .any(|known| consumer_identity(known) == identity)
314                            {
315                                consumers.push(consumer);
316                                changed = true;
317                            }
318                        } else if url.has_parameters() {
319                            let urls = open.entry(caller).or_default();
320                            if urls.len() < MAX_OPEN_URLS
321                                && !urls.iter().any(|known| known.url == url)
322                            {
323                                urls.push(OpenUrl {
324                                    origin: wrapper.origin,
325                                    url,
326                                });
327                                changed = true;
328                            }
329                        }
330                    }
331                }
332            }
333            if !changed {
334                break;
335            }
336        }
337        resolved
338    }
339
340    /// Exact consumers issued by `function`, directly or through an instantiated wrapper.
341    fn exact_consumers<'r>(
342        &'r self,
343        function: Function<'a>,
344        resolved: &'r BTreeMap<Function<'a>, Vec<SourceObservation>>,
345    ) -> Vec<&'r SourceObservation>
346    where
347        'a: 'r,
348    {
349        self.consumers
350            .get(&function)
351            .into_iter()
352            .flatten()
353            .copied()
354            .filter(|observation| {
355                observation.status == SourceEpistemicStatus::Confirmed
356                    && observation.method.is_some()
357                    && observation.path.is_some()
358            })
359            .chain(self.received.get(&function).into_iter().flatten())
360            .chain(resolved.get(&function).into_iter().flatten())
361            .collect()
362    }
363
364    /// Exact consumers of the functions reached from `test`, attributed to `test` at the call
365    /// site in the test through which each one is reached.
366    fn reached_consumers(
367        &self,
368        test: Function<'a>,
369        resolved: &BTreeMap<Function<'a>, Vec<SourceObservation>>,
370    ) -> Vec<SourceObservation> {
371        let mut reached = Vec::new();
372        let mut visited = BTreeSet::from([test]);
373        let mut frontier = self
374            .calls
375            .get(&test)
376            .into_iter()
377            .flatten()
378            .filter_map(|call| Some((self.callee(test.0, call)?, call.lines)))
379            .collect::<Vec<_>>();
380        for _ in 0..MAX_TEST_CALL_DEPTH {
381            let mut next = Vec::new();
382            for (function, lines) in frontier {
383                if !visited.insert(function) {
384                    continue;
385                }
386                for consumer in self.exact_consumers(function, resolved) {
387                    let mut attributed = consumer.clone();
388                    attributed.symbol_name = Some(test.1.to_owned());
389                    attributed.symbol_start_line = None;
390                    attributed.lines = lines;
391                    attributed.url = None;
392                    reached.push(attributed);
393                }
394                next.extend(
395                    self.calls
396                        .get(&function)
397                        .into_iter()
398                        .flatten()
399                        .filter_map(|call| Some((self.callee(function.0, call)?, lines))),
400                );
401            }
402            frontier = next;
403        }
404        reached
405    }
406}
407
408/// Arguments of `call` keyed by keyword, or by position among the positional arguments.
409fn bound_arguments(call: &SourceObservation) -> BoundArguments {
410    let mut bound = BoundArguments::default();
411    let mut position = 0;
412    for argument in call.call.iter().flat_map(|call| &call.arguments) {
413        match (&argument.keyword, &argument.value) {
414            (Some(keyword), Some(value)) => {
415                bound.by_name.insert(keyword.clone(), value.clone());
416            }
417            (Some(_), None) => {}
418            (None, value) => {
419                if let Some(value) = value {
420                    bound.by_index.insert(position, value.clone());
421                }
422                position += 1;
423            }
424        }
425    }
426    bound
427}
428
429/// Method, path, and authority of a consumer.
430type ConsumerIdentity = (Option<String>, Option<String>, Option<String>);
431
432fn consumer_identity(observation: &SourceObservation) -> ConsumerIdentity {
433    (
434        observation.method.clone(),
435        observation.path.clone(),
436        observation.authority.clone(),
437    )
438}
439
440#[cfg(test)]
441mod tests {
442    use code_system_graph_model::RepoId;
443
444    use super::compose_client_flows;
445    use crate::{
446        RepositorySourceFile, SourceObservation, SourceRole, parse_python_source, parse_typescript_source
447    };
448
449    /// Consumers after composition as `path: symbol METHOD authority path`.
450    fn composed_consumers(files: &[(&str, Vec<SourceObservation>)]) -> Vec<String> {
451        let repo = RepoId::new("repo:shop");
452        let inputs = files
453            .iter()
454            .map(|(path, observations)| RepositorySourceFile {
455                repo_id: &repo,
456                path,
457                observations,
458            })
459            .collect::<Vec<_>>();
460        let composed = compose_client_flows(&inputs);
461        assert!(
462            composed
463                .iter()
464                .flatten()
465                .all(|item| item.role != SourceRole::Call)
466        );
467        files
468            .iter()
469            .zip(composed)
470            .flat_map(|((path, _), observations)| {
471                observations
472                    .into_iter()
473                    .filter(|item| item.role == SourceRole::Consumer && item.path.is_some())
474                    .map(move |item| {
475                        format!(
476                            "{path}: {} {} {} {}",
477                            item.symbol_name.unwrap_or_default(),
478                            item.method.unwrap_or_default(),
479                            item.authority.unwrap_or_default(),
480                            item.path.unwrap_or_default()
481                        )
482                    })
483            })
484            .collect()
485    }
486
487    #[test]
488    fn python_tests_should_reach_fixture_and_wrapped_helper_calls_across_files() {
489        let conftest = r#"
490import pytest
491import requests
492BASE = "http://orders:8000"
493
494@pytest.fixture
495def order():
496    return requests.post(BASE + "/orders", json={}).json()
497"#;
498        let helpers = r#"
499import requests
500API = "http://orders:8000/api"
501
502def get_json(path):
503    return requests.get(API + path).json()
504
505def read_order(order_id):
506    return get_json(f"/orders/{order_id}")
507"#;
508        let test = r#"
509from tests.helpers import read_order
510
511def test_read(order):
512    read_order(order["id"])
513"#;
514        let consumers = composed_consumers(&[
515            ("tests/conftest.py", parse_python_source(conftest)),
516            ("tests/helpers.py", parse_python_source(helpers)),
517            ("tests/test_orders.py", parse_python_source(test)),
518        ]);
519
520        assert_eq!(
521            consumers,
522            [
523                "tests/conftest.py: order POST orders:8000 /orders",
524                "tests/helpers.py: read_order GET orders:8000 /api/orders/{order_id}",
525                "tests/test_orders.py: test_read POST orders:8000 /orders",
526                "tests/test_orders.py: test_read GET orders:8000 /api/orders/{order_id}",
527            ]
528        );
529    }
530
531    #[test]
532    fn python_parameter_clients_should_resolve_through_test_client_fixtures() {
533        let conftest = r"
534import pytest
535from fastapi.testclient import TestClient
536from app.main import app
537
538@pytest.fixture
539def client():
540    with TestClient(app) as test_client:
541        yield test_client
542";
543        let helpers = r#"
544def create_order(client, sku):
545    return client.post("/orders", json={"sku": sku})
546
547def health(session):
548    return session.get("/health")
549"#;
550        let test = r#"
551from tests.helpers import create_order, health
552
553def test_create(client):
554    create_order(client, "A-1")
555
556def test_read(client):
557    client.get("/orders/42")
558
559def test_health():
560    health(object())
561"#;
562        let files = [
563            ("tests/conftest.py", parse_python_source(conftest)),
564            ("tests/helpers.py", parse_python_source(helpers)),
565            ("tests/test_orders.py", parse_python_source(test)),
566        ];
567        let consumers = composed_consumers(&files);
568
569        assert_eq!(
570            consumers,
571            [
572                "tests/helpers.py: create_order POST  /orders",
573                "tests/test_orders.py: test_read GET  /orders/42",
574                "tests/test_orders.py: test_create POST  /orders",
575            ]
576        );
577    }
578
579    #[test]
580    fn script_wrappers_should_be_instantiated_at_call_sites_in_other_modules() {
581        let api = r#"
582const API = "/api";
583export function request(path: string) {
584  return fetch(API + path);
585}
586"#;
587        let orders = r#"
588import { request } from "./api";
589export function getOrder(id: string) {
590  return request(`/orders/${id}`);
591}
592"#;
593        let consumers = composed_consumers(&[
594            ("web/src/api.ts", parse_typescript_source(api)),
595            ("web/src/orders.ts", parse_typescript_source(orders)),
596        ]);
597
598        assert_eq!(
599            consumers,
600            ["web/src/orders.ts: getOrder GET  /api/orders/{id}"]
601        );
602    }
603}