agent-first-data 0.31.0

A naming convention that lets AI agents understand your data without being told what it means, plus a CLI and library for reading Markdown structure and safely editing structured JSON, TOML, YAML, dotenv, and INI documents.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
use serde_json::Value;
use std::collections::HashSet;

// ═══════════════════════════════════════════
// Public API: Output Formatters
// ═══════════════════════════════════════════

/// Which fields a [`Redactor`] scrubs. The default is [`RedactionPolicy::All`].
///
/// The policy selects a *scope* inside a structured value. A command line
/// ([`Redactor::argv`]) and a bare URL string ([`Redactor::url`]) have no
/// `result`/`trace` split to scope to, so on those two paths `TraceOnly`
/// redacts in full like `All`, and only [`RedactionPolicy::Off`] turns
/// redaction off.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum RedactionPolicy {
    /// Redact every secret field anywhere in the value (the default).
    #[default]
    All,
    /// Redact only inside the top-level `trace` object.
    TraceOnly,
    /// Do not redact anything.
    Off,
}

impl RedactionPolicy {
    /// Whether this policy redacts an input that carries no scope of its own —
    /// a command line or a single URL string, as opposed to a JSON value with a
    /// `trace` object to narrow to.
    ///
    /// `TraceOnly` narrows *where* redaction applies within a value; it does not
    /// weaken redaction. A standalone argv or URL has no non-`trace` half to
    /// leave alone — and both are diagnostic material by construction, the very
    /// thing `TraceOnly` scrubs — so `TraceOnly` redacts them in full, exactly
    /// like `All`. Only `Off`, the caller explicitly asking for raw output,
    /// disables redaction, and it does so on all three paths alike.
    fn redacts_unscoped_input(self) -> bool {
        !matches!(self, RedactionPolicy::Off)
    }
}

/// Rendering style for plain (logfmt) output only. JSON and YAML are always
/// structure-preserving and ignore this.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum PlainStyle {
    /// Human-readable AFDATA rendering: strip suffixes and format values.
    #[default]
    Readable,
    /// Schema-preserving rendering: keep keys and values unchanged after redaction.
    Raw,
}

/// Configurable redaction builder for secrets and legacy field names.
///
/// `Redactor` encapsulates redaction policy, custom secret field names, and
/// exact legacy URL field names.
/// Build with [`Redactor::new()`], configure via builder methods, then pass to
/// redaction functions like [`redacted_value`] or [`redact_url_secrets`].
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Redactor {
    policy: RedactionPolicy,
    secret_names: Vec<String>,
    url_names: Vec<String>,
}

impl Redactor {
    /// Create a new default redactor (full redaction, no custom secret names).
    pub fn new() -> Self {
        Self::default()
    }

    /// Set custom field names to treat as secrets in addition to `_secret` suffixes.
    ///
    /// Matching is exact field-name equality at any nesting level. The same
    /// list also matches URL query-parameter names inside `_url` fields.
    /// Builder style: returns `self`.
    pub fn secret_names<I: IntoIterator<Item = S>, S: Into<String>>(mut self, names: I) -> Self {
        self.secret_names = names.into_iter().map(|s| s.into()).collect();
        self
    }

    /// Set exact legacy field names whose values should receive URL-aware
    /// redaction in addition to `_url`/`_URL` suffixed fields.
    ///
    /// A matching string is handled like an `_url` value. Arrays and nested
    /// collections recursively apply that treatment to every string leaf.
    /// Matching is exact and does not normalize case, whitespace, or spelling.
    #[must_use]
    pub fn url_names<I: IntoIterator<Item = S>, S: Into<String>>(mut self, names: I) -> Self {
        self.url_names = names.into_iter().map(Into::into).collect();
        self
    }

    /// Set the redaction policy (default: full redaction).
    /// Builder style: returns `self`.
    pub fn policy(mut self, policy: RedactionPolicy) -> Self {
        self.policy = policy;
        self
    }

    /// Redact a JSON value copy using this redactor's policy and secret names.
    ///
    /// Clones `value` first; for a large payload you already own and can
    /// mutate, prefer [`Redactor::redact_in_place`] to avoid the copy.
    pub fn value(&self, value: &Value) -> Value {
        let mut v = value.clone();
        self.redact_in_place(&mut v);
        v
    }

    /// Redact secret components of a URL string using this redactor's settings.
    ///
    /// A query parameter is redacted iff its (form-decoded) name ends in
    /// `_secret`/`_SECRET` or matches an exact entry in `secret_names`. A
    /// fragment written in the same `k=v&k=v` shape — how an OAuth implicit-flow
    /// response carries its token — is redacted by that same rule; any other
    /// fragment passes through byte-for-byte. The userinfo password
    /// (`scheme://user:pass@host`) is always redacted as a structural rule.
    /// Only the secret spans are replaced with `***`; every other byte is
    /// preserved. A string that is not a single, whitespace-free,
    /// scheme-prefixed URL (including a URL embedded in surrounding prose) is
    /// returned unchanged.
    ///
    /// A URL is unscoped input: `RedactionPolicy::Off` returns it unchanged,
    /// every other policy redacts it in full.
    pub fn url(&self, url: &str) -> String {
        if !self.policy.redacts_unscoped_input() {
            return url.to_string();
        }
        let context = RedactionContext::from_redactor(self);
        redact_url_in_str(url, &context).unwrap_or_else(|| url.to_string())
    }

    /// Redact complete scheme-prefixed URLs embedded in prose.
    ///
    /// This is deliberately explicit and independent from structured field
    /// redaction. It recognizes URL spans only; it never scans surrounding text
    /// for secret-looking names or values.
    pub fn urls_in_text(&self, text: &str) -> String {
        if !self.policy.redacts_unscoped_input() {
            return text.to_string();
        }
        let context = RedactionContext::from_redactor(self);
        redact_urls_in_text_with_context(text, &context)
    }

    /// Redact `value` in place, using this redactor's policy and secret names.
    ///
    /// The zero-copy counterpart of [`Redactor::value`] — use it on a large
    /// payload you already own to avoid cloning.
    pub fn redact_in_place(&self, value: &mut Value) {
        let context = RedactionContext::from_redactor(self);
        apply_redaction_policy_with_context(value, self.policy, &context);
    }

    /// Redact secret *values* out of a command line, using this redactor's
    /// policy and secret names.
    ///
    /// A long flag whose name is secret by AFDATA naming (`--api-key-secret`,
    /// or an exact `secret_names` entry) has its value replaced by `***`, in
    /// both `--flag=value` and `--flag value` spellings. Everything else is
    /// preserved byte-for-byte.
    ///
    /// Free text is deliberately never scanned: a bare `api_key_secret=sk-live`
    /// positional, or a secret-looking token after a non-secret flag, is left
    /// alone. AFDATA decides sensitivity from the *field name*, and argv is no
    /// exception — rename the flag rather than pattern-matching values. A flag
    /// with no value (end of argv, or followed by another flag) is likewise
    /// left inspectable.
    ///
    /// Only long (`--`) flags are recognized, matching the convention's
    /// long-flags-only rule.
    ///
    /// A command line is unscoped input: `RedactionPolicy::Off` returns `args`
    /// unchanged, every other policy redacts in full.
    pub fn argv<S: AsRef<str>>(&self, args: &[S]) -> Vec<String> {
        if !self.policy.redacts_unscoped_input() {
            return args.iter().map(|arg| arg.as_ref().to_string()).collect();
        }
        let context = RedactionContext::from_redactor(self);
        let mut out = Vec::with_capacity(args.len());
        let mut redact_next = false;
        for arg in args {
            let arg = arg.as_ref();
            if redact_next {
                redact_next = false;
                if !arg.starts_with('-') {
                    out.push(REDACTED_MARKER.to_string());
                    continue;
                }
            }
            if let Some(rest) = arg.strip_prefix("--") {
                if let Some((name, _)) = rest.split_once('=') {
                    if is_secret_flag_name(name, &context) {
                        out.push(format!("--{name}={REDACTED_MARKER}"));
                        continue;
                    }
                } else if is_secret_flag_name(rest, &context) {
                    redact_next = true;
                }
            }
            out.push(arg.to_string());
        }
        out
    }

    /// True when `name` would be treated as a secret field name by this
    /// redactor: an exact `_secret`/`_SECRET` suffix, or an exact match
    /// against a configured `secret_names` entry.
    ///
    /// Exposed for callers that must gate on a single *targeted* field name
    /// (for example a CLI dot-path leaf) rather than redact a whole value —
    /// [`Redactor::value`] only rewrites fields it finds while walking an
    /// object, so a bare scalar pulled out from under its field name needs
    /// this explicit check instead.
    pub fn is_secret_name(&self, name: &str) -> bool {
        RedactionContext::from_redactor(self).is_secret_key(name)
    }

    /// True when `name` receives URL-aware structured-field redaction: an
    /// exact `_url`/`_URL` suffix or an exact configured `url_names` entry.
    ///
    /// The counterpart to [`Redactor::is_secret_name`], for the same targeted
    /// single-field case. Kept even with no in-tree caller: a consumer that
    /// gates on one name has to ask both questions, and an API that answers
    /// only half of a symmetric pair sends the caller back to reimplementing
    /// the suffix rule — which is exactly how the rule drifts.
    pub fn is_url_name(&self, name: &str) -> bool {
        RedactionContext::from_redactor(self).is_url_key(name)
    }
}

impl From<RedactionPolicy> for Redactor {
    fn from(policy: RedactionPolicy) -> Self {
        Self {
            policy,
            secret_names: Vec::new(),
            url_names: Vec::new(),
        }
    }
}

/// Output options combining redaction and rendering style.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct OutputOptions {
    /// Redactor applied before rendering.
    pub redaction: Redactor,
    /// Rendering style for plain output only.
    pub style: PlainStyle,
}

impl From<RedactionPolicy> for OutputOptions {
    fn from(policy: RedactionPolicy) -> Self {
        Self {
            redaction: Redactor::from(policy),
            style: PlainStyle::default(),
        }
    }
}

// ═══════════════════════════════════════════
// Public API: Redaction & Utility
// ═══════════════════════════════════════════

/// Return a JSON value copy with default `_secret` redaction applied.
pub fn redacted_value(value: &Value) -> Value {
    Redactor::new().value(value)
}

/// Redact secret values out of a command line, using default options.
///
/// Returns `args` with the value of every `_secret`-suffixed long flag replaced
/// by `***`, covering both `--flag=value` and `--flag value`. Use
/// [`Redactor::argv`] for custom `secret_names` or a non-default policy.
///
/// Intended for CLIs that record their own invocation — startup diagnostics,
/// audit trails, crash reports — where writing argv verbatim would put a
/// credential in the log.
pub fn redact_argv<S: AsRef<str>>(args: &[S]) -> Vec<String> {
    Redactor::new().argv(args)
}

/// Redact secret components of a single URL string, using default options.
///
/// Returns `url` with its userinfo password and any `_secret`-suffixed query
/// parameter values replaced by `***`.
pub fn redact_url_secrets(url: &str) -> String {
    Redactor::new().url(url)
}

/// Redact complete scheme-prefixed URLs embedded in prose, using default
/// secret-name rules.
///
/// This helper is opt-in. Ordinary structured redaction never scans prose.
pub fn redact_urls_in_text(text: &str) -> String {
    Redactor::new().urls_in_text(text)
}

// ═══════════════════════════════════════════
// Secret Redaction
// ═══════════════════════════════════════════

/// The scalar every redacted span, value, and subtree is replaced with. Also
/// the signal plain rendering reads to tell a hidden field from a live one.
pub(crate) const REDACTED_MARKER: &str = "***";

#[derive(Default)]
pub(crate) struct RedactionContext {
    secret_names: HashSet<String>,
    url_names: HashSet<String>,
}

impl RedactionContext {
    fn from_redactor(redactor: &Redactor) -> Self {
        let secret_names = redactor.secret_names.iter().cloned().collect();
        let url_names = redactor.url_names.iter().cloned().collect();
        Self {
            secret_names,
            url_names,
        }
    }

    fn is_secret_key(&self, key: &str) -> bool {
        key_has_secret_suffix(key) || self.secret_names.contains(key)
    }

    fn is_url_key(&self, key: &str) -> bool {
        key_has_url_suffix(key) || self.url_names.contains(key)
    }
}

fn key_has_secret_suffix(key: &str) -> bool {
    key.ends_with("_secret") || key.ends_with("_SECRET")
}

fn key_has_url_suffix(key: &str) -> bool {
    key.ends_with("_url") || key.ends_with("_URL")
}

/// Whether a long flag's name is secret, normalizing the kebab-case flag
/// spelling to the snake_case field spelling the convention is defined in.
///
/// Ungated: core argv redaction relies on it.
pub(crate) fn is_secret_flag_name(flag_name: &str, context: &RedactionContext) -> bool {
    let normalized = flag_name.replace('-', "_");
    context.is_secret_key(&normalized) || context.is_secret_key(flag_name)
}

const MAX_DEPTH: usize = 256;
const MAX_DEPTH_MARKER: &str = "<afdata:max-depth>";

fn redact_secrets_with_context(value: &mut Value, context: &RedactionContext) {
    redact_secrets_with_context_depth(value, context, 0);
}

fn redact_secrets_with_context_depth(value: &mut Value, context: &RedactionContext, depth: usize) {
    if depth >= MAX_DEPTH {
        *value = Value::String(MAX_DEPTH_MARKER.into());
        return;
    }
    match value {
        Value::Object(map) => {
            let keys: Vec<String> = map.keys().cloned().collect();
            for key in keys {
                if context.is_secret_key(&key) {
                    // A null secret is an *absent* secret. Masking it would
                    // manufacture the appearance of a configured credential:
                    // readers cannot tell `"***"`-because-set from
                    // `"***"`-because-null, so a tool showing its own config
                    // would report every unset secret as configured. Redaction
                    // hides a value that exists; it does not invent one.
                    if !map.get(&key).is_some_and(Value::is_null) {
                        map.insert(key, Value::String(REDACTED_MARKER.into()));
                    }
                } else if context.is_url_key(&key) {
                    if let Some(v) = map.get_mut(&key) {
                        redact_url_value_depth(v, context, depth + 1);
                    }
                } else if let Some(v) = map.get_mut(&key) {
                    redact_secrets_with_context_depth(v, context, depth + 1);
                }
            }
        }
        Value::Array(arr) => {
            for v in arr {
                redact_secrets_with_context_depth(v, context, depth + 1);
            }
        }
        _ => {}
    }
}

fn redact_url_value_depth(value: &mut Value, context: &RedactionContext, depth: usize) {
    if depth >= MAX_DEPTH {
        *value = Value::String(MAX_DEPTH_MARKER.into());
        return;
    }
    match value {
        Value::String(text) => {
            *text = redact_url_field_value(text, context);
        }
        Value::Array(values) => {
            for value in values {
                redact_url_value_depth(value, context, depth + 1);
            }
        }
        Value::Object(values) => {
            let keys = values.keys().cloned().collect::<Vec<_>>();
            for key in keys {
                if context.is_secret_key(&key) {
                    if !values.get(&key).is_some_and(Value::is_null) {
                        values.insert(key, Value::String(REDACTED_MARKER.into()));
                    }
                } else if let Some(value) = values.get_mut(&key) {
                    redact_url_value_depth(value, context, depth + 1);
                }
            }
        }
        _ => {}
    }
}

/// Redact secret components of a single URL string, returning `Some(redacted)`
/// when `s` is a processable URL, or `None` when it is not (so callers can keep
/// the original). Only secret spans change; all other bytes are preserved.
fn redact_url_in_str(s: &str, context: &RedactionContext) -> Option<String> {
    // Precondition (spec): a single, whitespace-free, scheme-prefixed URL.
    // The gate is scheme + no-whitespace only — NOT "parses as a URL library
    // object". Span location below is purely byte-wise, so we never re-serialize
    // the URL; adding a `url::Url::parse` gate here would diverge across
    // languages (e.g. ports > 65535 or empty hosts that one library rejects and
    // another accepts) and silently leak secrets in the values it rejects.
    if !s.contains("://") || !is_single_url(s) {
        return None;
    }
    let scheme_sep = s.find("://")?;
    let scheme = &s[..scheme_sep];
    let rest = &s[scheme_sep + 3..];

    // Authority runs from after "://" to the first '/', '?', or '#'.
    let auth_end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
    let authority = &rest[..auth_end];
    let remainder = &rest[auth_end..];

    let new_authority = redact_userinfo_password(authority);

    // `remainder` is `path[?query][#fragment]`; '#' ends the query, so split the
    // fragment off first and the query out of what is left.
    let (before_fragment, fragment) = match remainder.split_once('#') {
        Some((before, fragment)) => (before, Some(fragment)),
        None => (remainder, None),
    };
    let new_before_fragment = match before_fragment.split_once('?') {
        Some((path, query)) => format!("{path}?{}", redact_query(query, context)),
        None => before_fragment.to_string(),
    };
    // A fragment gets the same treatment as the query: `k=v&k=v` after the '#'
    // is exactly how an OAuth implicit-flow response hands back a token, so a
    // secret-named fragment parameter must not survive where the identically
    // named query parameter would not. A fragment that is not in that shape has
    // no '=' in its segments and passes through byte-for-byte.
    let new_fragment = match fragment {
        Some(fragment) => format!("#{}", redact_query(fragment, context)),
        None => String::new(),
    };

    Some(format!(
        "{scheme}://{new_authority}{new_before_fragment}{new_fragment}"
    ))
}

fn redact_urls_in_text_with_context(text: &str, context: &RedactionContext) -> String {
    let mut output = String::with_capacity(text.len());
    let mut copied_through = 0;
    let mut search_from = 0;
    let mut found = false;
    while let Some((start, end)) = next_scheme_url_span(text, search_from) {
        found = true;
        output.push_str(&text[copied_through..start]);
        let url = &text[start..end];
        output.push_str(&redact_url_in_str(url, context).unwrap_or_else(|| url.to_string()));
        copied_through = end;
        search_from = end;
    }
    if !found {
        return text.to_string();
    }
    output.push_str(&text[copied_through..]);
    output
}

fn next_scheme_url_span(text: &str, from: usize) -> Option<(usize, usize)> {
    let bytes = text.as_bytes();
    let mut start = from;
    while start < bytes.len() {
        // Only consider the head of a maximal scheme-byte run: a scheme cannot
        // begin mid-run, so a position whose predecessor is a scheme byte was
        // already covered by the run that contains it.
        if start == 0 || !is_scheme_byte(bytes[start - 1]) {
            let mut scheme_end = start;
            while scheme_end < bytes.len() && is_scheme_byte(bytes[scheme_end]) {
                scheme_end += 1;
            }
            // A scheme must start with a letter, but the run need not: prose
            // like `2https://h/?token_secret=x` glues a digit onto the URL. Retry
            // from the first letter *inside* the run instead of abandoning the
            // span — giving up there fails open and leaks the whole query.
            let scheme_start = (start..scheme_end).find(|&i| bytes[i].is_ascii_alphabetic());
            if let Some(scheme_start) = scheme_start
                && bytes
                    .get(scheme_end..scheme_end.saturating_add(3))
                    .is_some_and(|separator| separator == b"://")
            {
                let mut end = scheme_end + 3;
                while end < bytes.len() {
                    let Some(character) = text[end..].chars().next() else {
                        break;
                    };
                    if is_span_terminator_whitespace(character) || is_text_url_delimiter(character)
                    {
                        break;
                    }
                    end += character.len_utf8();
                }
                while end > scheme_end + 3 {
                    let Some(character) = text[..end].chars().next_back() else {
                        break;
                    };
                    if !is_trailing_text_url_punctuation(character) {
                        break;
                    }
                    end -= character.len_utf8();
                }
                if end > scheme_end + 3 {
                    return Some((scheme_start, end));
                }
            }
        }
        start += 1;
    }
    None
}

fn is_scheme_byte(byte: u8) -> bool {
    byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.')
}

/// Whitespace that ends a URL span in prose: the Unicode `White_Space`
/// property, written out here instead of delegated to `char::is_whitespace`.
///
/// The four AFDATA implementations must cut the span at the same byte, and each
/// language's native predicate covers a different set — JS `\s` counts U+FEFF
/// but not U+0085, Python's `str.isspace()` counts U+001C–U+001F, Rust and Go
/// count neither. Divergence is a defect in both directions: ending a span early
/// leaves the rest of the query readable, and ending it late pulls the following
/// prose into the URL, where redacting the parameter it lands in deletes it. So
/// the set is enumerated, and it is generous — a redaction helper exists to keep
/// a log line readable, so anything a human reads as a space must end the URL.
///
/// Two exclusions are deliberate, not oversights:
/// * **U+FEFF** (zero-width no-break space) is not `White_Space`. Treating it as
///   one would cut `https://h/x\u{feff}?token_secret=leak` short and leave the
///   secret in the clear — the exact leak JS `\s` used to cause here.
/// * **U+001C–U+001F** (the file/group/record/unit separators) are not
///   `White_Space` either; only Python's `str.isspace()` counted them.
///
/// This is a different question from [`is_single_url`], which asks whether a
/// whole string is one URL and stays ASCII-only. A span this scanner produces
/// contains no whitespace at all, so it satisfies that stricter gate either way.
fn is_span_terminator_whitespace(character: char) -> bool {
    matches!(
        character,
        // TAB, LF, VT, FF, CR, SPACE
        '\u{0009}'..='\u{000d}' | '\u{0020}'
            // NEL, NBSP, OGHAM SPACE MARK
            | '\u{0085}' | '\u{00a0}' | '\u{1680}'
            // EN QUAD .. HAIR SPACE
            | '\u{2000}'..='\u{200a}'
            // LINE SEPARATOR, PARAGRAPH SEPARATOR, NARROW NBSP,
            // MEDIUM MATHEMATICAL SPACE, IDEOGRAPHIC SPACE
            | '\u{2028}' | '\u{2029}' | '\u{202f}' | '\u{205f}' | '\u{3000}'
    )
}

fn is_text_url_delimiter(character: char) -> bool {
    matches!(
        character,
        '"' | '\''
            | '<'
            | '>'
            | '`'
            | ''
            | ''
            | ''
            | ''
            | ''
            | ''
            | ''
            | ''
            | ''
            | ''
    )
}

fn is_trailing_text_url_punctuation(character: char) -> bool {
    matches!(
        character,
        '.' | ',' | ';' | '!' | ')' | ']' | '}' | '' | '' | '' | '' | '' | '' | ''
    )
}

fn redact_url_field_value(s: &str, context: &RedactionContext) -> String {
    if let Some(redacted) = redact_url_in_str(s, context) {
        return redacted;
    }
    let trimmed = s.trim();
    if trimmed != s
        && let Some(redacted) = redact_url_in_str(trimmed, context)
    {
        return redacted;
    }
    // Fail closed: a `_url` value we could not parse as a clean scheme-prefixed
    // URL, yet which carries a credential sigil (`@` userinfo) or internal
    // whitespace, is redacted wholesale rather than passed through. A schemeless
    // connection string like `user:pass@host/db` has no scheme anchor for the
    // surgical span logic above, so blanket redaction is the safe default.
    if s.chars().any(char::is_whitespace) || s.contains('@') {
        return REDACTED_MARKER.to_string();
    }
    s.to_string()
}

/// Replace the userinfo password (`user:pass@`) with `***`, preserving the
/// username. Authority without `@`, or userinfo without `:`, is unchanged.
fn redact_userinfo_password(authority: &str) -> String {
    let Some(at) = authority.rfind('@') else {
        return authority.to_string();
    };
    let userinfo = &authority[..at];
    match userinfo.find(':') {
        Some(colon) => format!(
            "{}:{REDACTED_MARKER}{}",
            &authority[..colon],
            &authority[at..]
        ),
        None => authority.to_string(),
    }
}

/// Redact the values of secret-named query parameters, preserving raw bytes of
/// every other segment (keys, benign values, encoding, ordering, separators).
fn redact_query(query: &str, context: &RedactionContext) -> String {
    query
        .split('&')
        .map(|segment| {
            let Some(eq) = segment.find('=') else {
                return segment.to_string();
            };
            let raw_key = &segment[..eq];
            // Form-decode the name (`+` → space, percent-decode) for the check.
            let name = url::form_urlencoded::parse(segment.as_bytes())
                .next()
                .map(|(k, _)| k.into_owned())
                .unwrap_or_default();
            if context.is_secret_key(&name) {
                format!("{raw_key}={REDACTED_MARKER}")
            } else {
                segment.to_string()
            }
        })
        .collect::<Vec<_>>()
        .join("&")
}

/// True when `s` begins with a URL scheme (`ALPHA *(ALPHA / DIGIT / "+" / "-" /
/// ".") "://"`) and contains no ASCII whitespace — i.e. a single bare URL, not
/// a URL embedded in prose.
fn is_single_url(s: &str) -> bool {
    if s.bytes().any(|b| b.is_ascii_whitespace()) {
        return false;
    }
    let bytes = s.as_bytes();
    if !bytes.first().is_some_and(|b| b.is_ascii_alphabetic()) {
        return false;
    }
    let mut i = 1;
    while i < bytes.len() {
        let c = bytes[i];
        if c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.') {
            i += 1;
        } else {
            break;
        }
    }
    s[i..].starts_with("://")
}

fn apply_redaction_policy_with_context(
    value: &mut Value,
    redaction_policy: RedactionPolicy,
    context: &RedactionContext,
) {
    match redaction_policy {
        RedactionPolicy::All => redact_secrets_with_context(value, context),
        RedactionPolicy::TraceOnly => {
            if let Value::Object(map) = value
                && let Some(trace) = map.get_mut("trace")
            {
                redact_secrets_with_context(trace, context);
            }
        }
        RedactionPolicy::Off => {}
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    // ── URL fragments carry secrets too ──────────

    #[test]
    fn url_fragment_params_redacted_like_query_params() {
        assert_eq!(
            redact_url_secrets("https://h/p?token_secret=QUERYLEAK#token_secret=FRAGLEAK"),
            "https://h/p?token_secret=***#token_secret=***"
        );
    }

    #[test]
    fn url_fragment_params_redacted_without_a_query() {
        // The OAuth implicit-flow shape: the credential exists only after '#'.
        assert_eq!(
            redact_url_secrets("https://h/cb#access_token_secret=abc&state=xyz"),
            "https://h/cb#access_token_secret=***&state=xyz"
        );
    }

    #[test]
    fn url_fragment_without_params_is_preserved() {
        for url in [
            "https://h/p?a=1#section",
            "https://h/p#",
            "https://h/p#a/b?c",
        ] {
            assert_eq!(redact_url_secrets(url), url);
        }
    }

    #[test]
    fn url_fragment_honors_secret_names() {
        let redactor = Redactor::new().secret_names(vec!["token".to_string()]);
        assert_eq!(
            redactor.url("https://h/cb#token=abc&page=2"),
            "https://h/cb#token=***&page=2"
        );
    }

    #[test]
    fn url_names_are_exact_and_recurse_through_collections() {
        let redactor = Redactor::new().secret_names(["token"]).url_names([
            "url",
            "relays",
            "synapse_selector",
        ]);
        let input = json!({
            "url": "https://u:pw@h/?token=one",
            "URL": "https://u:visible@h/?token=two",
            "relays": [
                "wss://relay.example/?token=three",
                {"nested": "https://u:pw@nested.example/"}
            ],
            "synapse_selector": "user:pass@host/db",
            "other": "https://u:visible@h/?token=four"
        });

        assert_eq!(
            redactor.value(&input),
            json!({
                "url": "https://u:***@h/?token=***",
                "URL": "https://u:visible@h/?token=two",
                "relays": [
                    "wss://relay.example/?token=***",
                    {"nested": "https://u:***@nested.example/"}
                ],
                "synapse_selector": "***",
                "other": "https://u:visible@h/?token=four"
            })
        );
        assert!(redactor.is_url_name("url"));
        assert!(!redactor.is_url_name("URL"));
    }

    #[test]
    fn prose_url_redaction_is_explicit_and_preserves_punctuation() {
        let message = "connect (https://u:pw@h/?token_secret=one), then wss://h/?token_secret=two. bare user:pw@h stays";
        assert_eq!(
            redact_urls_in_text(message),
            "connect (https://u:***@h/?token_secret=***), then wss://h/?token_secret=***. bare user:pw@h stays"
        );
        assert_eq!(
            redacted_value(&json!({"message": message})),
            json!({"message": message})
        );
    }

    #[test]
    fn prose_url_redaction_honors_policy_and_secret_names() {
        let text = "see https://h/?token=abc";
        assert_eq!(
            Redactor::new().secret_names(["token"]).urls_in_text(text),
            "see https://h/?token=***"
        );
        assert_eq!(
            Redactor::new()
                .policy(RedactionPolicy::Off)
                .secret_names(["token"])
                .urls_in_text(text),
            text
        );
    }

    // ── One policy meaning across value, argv, url ──────────

    fn scoped_value() -> Value {
        json!({
            "result": {"api_key_secret": "sk-result"},
            "trace": {"api_key_secret": "sk-trace"}
        })
    }

    fn argv() -> Vec<String> {
        vec!["tool".to_string(), "--api-key-secret=sk-live".to_string()]
    }

    #[test]
    fn all_policy_redacts_every_path() {
        let redactor = Redactor::new().policy(RedactionPolicy::All);
        assert_eq!(
            redactor.value(&scoped_value()),
            json!({
                "result": {"api_key_secret": "***"},
                "trace": {"api_key_secret": "***"}
            })
        );
        assert_eq!(redactor.argv(&argv()), vec!["tool", "--api-key-secret=***"]);
        assert_eq!(
            redactor.url("https://u:pw@h/cb?token_secret=abc"),
            "https://u:***@h/cb?token_secret=***"
        );
    }

    #[test]
    fn trace_only_scopes_a_value_but_redacts_argv_and_url_in_full() {
        let redactor = Redactor::new().policy(RedactionPolicy::TraceOnly);
        // Scoped input: only the `trace` half is scrubbed.
        assert_eq!(
            redactor.value(&scoped_value()),
            json!({
                "result": {"api_key_secret": "sk-result"},
                "trace": {"api_key_secret": "***"}
            })
        );
        // Unscoped input: a command line and a bare URL have no non-`trace`
        // half to leave alone, so they are redacted like `All`.
        assert_eq!(redactor.argv(&argv()), vec!["tool", "--api-key-secret=***"]);
        assert_eq!(
            redactor.url("https://u:pw@h/cb?token_secret=abc"),
            "https://u:***@h/cb?token_secret=***"
        );
    }

    #[test]
    fn off_policy_disables_every_path() {
        let redactor = Redactor::new().policy(RedactionPolicy::Off);
        assert_eq!(redactor.value(&scoped_value()), scoped_value());
        assert_eq!(redactor.argv(&argv()), argv());
        assert_eq!(
            redactor.url("https://u:pw@h/cb?token_secret=abc"),
            "https://u:pw@h/cb?token_secret=abc"
        );
    }
}