diplodocus 0.15.0

Build coherent documentation websites for polyglot software projects.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
using SHA
using TOML
using Markdown

const EXPORT_POLICY = "julia-native-v1"

# Exact Docs attachments, source receipts from the loader, and Method metadata
# need Julia internals. Keep those reads here and exercise them on every minor.
module NativeCompat
    using Base.JuliaSyntax: JuliaSyntax as JS

    documentation(mod) = Base.Docs.meta(mod; autoinit=false)
    used_modules(mod) = ccall(:jl_module_usings, Any, (Any,), mod)
    function cache_files(cache)
        header = Base.parse_cache_header(cache)[2]
        @static if VERSION < v"1.11"
            return first(header), Base.parse_cache_header(cache; srcfiles_only=true)[2][1]
        else
            return header[1], header[2]
        end
    end
    cache_source(cache, file) = Base.read_dependency_src(cache, file)
    cache_path(id) = Base.pkgorigins[id].cachepath
    public_binding(mod, name) = @static VERSION >= v"1.11" ?
        Base.ispublic(mod, name) : Base.isexported(mod, name)
    arguments(method) = Base.method_argnames(method)
    keywords(method) = Base.kwarg_decl(method)
    function binding_origin(mod, name, visited=())
        reference = GlobalRef(mod, name)
        @static if VERSION < v"1.12"
            isdefined(reference, :binding) || return reference
            binding = reference.binding
            return isdefined(binding, :owner) ? binding.owner.globalref : reference
        else
            world = Base.get_world_counter()
            binding = convert(Core.Binding, reference)
            while true
                partition = Base.lookup_binding_partition(world, binding)
                kind = Base.binding_kind(partition)
                if kind == Base.PARTITION_KIND_IMPLICIT_CONST
                    # These partitions store the imported value in place of a
                    # binding. Follow the exports that resolve to its owner.
                    ref = binding.globalref
                    ref.mod in visited && return ref
                    owner = Base.binding_module(ref.mod, ref.name)
                    owner === ref.mod && return ref
                    origins = unique(binding_origin(used, ref.name, (visited..., ref.mod))
                                     for used in used_modules(ref.mod)
                                     if Base.isexported(used, ref.name) &&
                                        Base.binding_module(used, ref.name) === owner)
                    actual = filter(origin -> origin.mod === owner, origins)
                    return length(actual) == 1 ? only(actual) : ref
                end
                Base.is_some_binding_imported(kind) || return binding.globalref
                binding = Base.partition_restriction(partition)::Core.Binding
            end
        end
    end
    function constructor_methods(type)
        signature = Tuple{Type{<:type}, Vararg{Any}}
        matches = Base._methods_by_ftype(signature, -1, Base.get_world_counter())
        return [match.method for match in matches
                if constructor_head(match.method, type) !== nothing]
    end
    function constructor_head(method, type)
        head = Base.unwrap_unionall(method.sig).parameters[1]
        head isa DataType && head.name === Base.unwrap_unionall(Type).name || return nothing
        value = only(head.parameters)
        base = Base.unwrap_unionall(value)
        base isa DataType && base.name === Base.unwrap_unionall(type).name || return nothing
        return value
    end

    function source_headers(path)
        declarations = Dict{Int,Vector{Pair{Int,Expr}}}()
        attachments = Dict{Int,Vector{Expr}}()
        tree = JS.parseall(JS.SyntaxNode, read(path, String); filename=path, ignore_errors=true)
        function walk(node)
            children = JS.children(node)
            isempty(something(children, ())) && return
            kind = string(JS.kind(node))
            line = JS.source_line(node)
            if kind in ("function", "=", "macro")
                header_node = first(children)
                header = Expr(header_node)
                header isa Expr || return
                # Use the header's byte offset, since a macro argument's
                # declaration node can start in the preceding whitespace.
                entry = (JS.first_byte(header_node) - 1) => header
                push!(get!(() -> Pair{Int,Expr}[], declarations, line), entry)
                header_line = JS.source_line(header_node)
                if header_line != line
                    push!(get!(() -> Pair{Int,Expr}[], declarations, header_line), entry)
                end
            elseif kind in ("doc", "macrocall")
                # Julia parses a literal preceding a definition as Core.@doc.
                callee = kind == "doc" ? Symbol("@doc") : macro_name(Expr(first(children)))
                if callee === Symbol("@doc")
                    target = Expr(last(children))
                    target isa Expr && push!(get!(() -> Expr[], attachments, line), target)
                    walk(last(children))
                elseif callee in (Symbol("@inline"), Symbol("@noinline"), Symbol("@propagate_inbounds"))
                    walk(last(children))
                end
            elseif kind in ("toplevel", "block", "module", "baremodule", "struct")
                foreach(walk, children)
            end
        end
        walk(tree)
        return (; declarations, attachments)
    end

    function macro_name(callee)
        Meta.isexpr(callee, :., 2) && callee.args[1] in (:Base, :Core) &&
            callee.args[2] isa QuoteNode && return callee.args[2].value
        return callee
    end
end

module_name(mod) = join(Base.fullname(mod), '.')
qualified(mod, name) = string(module_name(mod), '.', name)
sha(bytes) = bytes2hex(sha256(bytes))

function name_token(name)
    Base.isidentifier(name) && return string(name)
    # Quoted bindings remain distinct from compiler-generated names.
    return "var" * repr(String(name))
end

function type_spelling(value, variables)
    if value isa TypeVar
        return variables[value]
    elseif value === Union{}
        return "Union{}"
    elseif value isa Union
        branches = sort!([type_spelling(branch, variables) for branch in Base.uniontypes(value)])
        return "Union{" * join(branches, ", ") * "}"
    elseif value isa UnionAll
        body, constraints, scope = type_variables(value, variables)
        return "(" * type_spelling(body, scope) * where_clause(constraints) * ")"
    elseif Base.isvarargtype(value)
        isdefined(value, :T) || return "Vararg"
        parameters = [type_spelling(value.T, variables)]
        isdefined(value, :N) && push!(parameters, type_spelling(value.N, variables))
        return "Vararg{" * join(parameters, ", ") * "}"
    elseif value isa DataType
        owner = parentmodule(value)
        if value <: Function && Base.issingletontype(value)
            singleton = value.instance
            name = nameof(singleton)
            isdefined(owner, name) && getfield(owner, name) === singleton ||
                error("A Julia dispatch type refers to an anonymous function.")
            return "typeof(" * qualified(owner, name_token(name)) * ")"
        end
        head = owner === Core || owner === Base ? name_token(nameof(value)) :
               qualified(owner, name_token(nameof(value)))
        value === Base.unwrap_unionall(value.name.wrapper) && return head
        isempty(value.parameters) && return head
        return head * "{" * join([type_spelling(x, variables) for x in value.parameters], ", ") * "}"
    elseif value isa Union{Bool,Integer,Char,Symbol,AbstractFloat}
        return repr(value)
    elseif value isa Tuple
        values = [type_spelling(part, variables) for part in value]
        return "(" * join(values, ", ") * (length(values) == 1 ? "," : "") * ")"
    end
    error("Unsupported Julia dispatch parameter of type $(typeof(value)).")
end

function type_variables(value, outer=IdDict{TypeVar,String}())
    scope = copy(outer)
    constraints = String[]
    while value isa UnionAll
        var = value.var
        name = "T$(length(scope) + 1)"
        # A bound can refer to an enclosing variable.
        lower = var.lb === Union{} ? "" : type_spelling(var.lb, scope) * "<:"
        upper = var.ub === Any ? "" : "<:" * type_spelling(var.ub, scope)
        push!(constraints, lower * name * upper)
        scope[var] = name
        value = value.body
    end
    return value, constraints, scope
end

where_clause(constraints) = isempty(constraints) ? "" : " where {" * join(constraints, ", ") * "}"

function package_file(root, path)
    isabspath(path) || return nothing
    local_path = relpath(normpath(path), root)
    parts = splitpath(local_path)
    return isabspath(local_path) || any(==(".."), parts) ? nothing : join(parts, '/')
end

function source_receipt(root, path, mod; cached=nothing)
    relative = package_file(root, path)
    relative === nothing && error("A Julia package include escapes the configured checkout.")
    canonical = package_file(root, realpath(path))
    canonical === nothing && error("A Julia package include symlink escapes the configured checkout.")
    bytes = read(path)
    cached === nothing || sha(bytes) == sha(cached) ||
        error("Julia source changed since the loaded package was precompiled: $relative")
    return Dict("path" => relative, "canonical" => canonical, "module" => mod,
                "digest" => sha(bytes))
end

function verify_inputs(root, receipts)
    for input in receipts
        path = joinpath(root, input["path"])
        (package_file(root, realpath(path)) == input["canonical"] &&
         sha(read(path)) == input["digest"]) || error("Julia inputs changed while loading the checkout.")
    end
end

function cached_inputs!(receipts, root, identity)
    cached = NativeCompat.cache_path(identity)
    cached === nothing && return true
    includes, sources = NativeCompat.cache_files(cached)
    source_names = Set(source.filename for source in sources)
    complete = true
    for input in includes
        if input.filename ∉ source_names
            # Files registered via include_dependency may have a timestamp only
            # on older Julia releases. The compiler still checks them on load.
            complete = false
            continue
        end
        text = NativeCompat.cache_source(cached, input.filename)
        mod = join([identity.name; input.modpath], '.')
        push!(receipts, source_receipt(root, input.filename, mod; cached=text))
    end
    return complete
end

function location(root, path, line)
    path isa Symbol && (path = String(path))
    path isa AbstractString || return Dict{String,Any}()
    relative = package_file(root, path)
    relative === nothing && return Dict{String,Any}()
    result = Dict{String,Any}("path" => relative)
    line isa Integer && line > 0 && (result["line"] = Int(line))
    return result
end

function documentation_text(doc)::Dict{String,Any}
    if !isempty(doc.text) && all(part -> part isa Union{String,SubString{String},Bool,Number,Char,Symbol},
                                 doc.text)
        return Dict("text" => join(doc.text))
    elseif doc.object isa Markdown.MD
        return Dict("text" => sprint(show, MIME"text/markdown"(), doc.object))
    end
    return Dict("unsupported" => "Unsupported Julia documentation object.")
end

function document(root, doc, registered)
    value = documentation_text(doc)
    value["module"] = module_name(registered)
    merge!(value, location(root, get(doc.data, :path, nothing), get(doc.data, :linenumber, 0)))
    return value
end

function methods_for(value, modules)
    entries = value isa Type ? NativeCompat.constructor_methods(value) : collect(methods(value, modules))
    return unique(filter(method -> method.module in modules, entries))
end

function positional(method, macro_binding)
    sig = Base.unwrap_unionall(method.sig)
    first = macro_binding ? 4 : 2
    return [sig.parameters[index] for index in first:length(sig.parameters)]
end

function method_arguments(method, macro_binding)
    removed = macro_binding ? 3 : 1
    arguments = NativeCompat.arguments(method)
    return [Base.isidentifier(name) && !startswith(String(name), '#') ? String(name) : "arg$index"
            for (index, name) in enumerate(arguments[removed+1:end])]
end

# Resolve type syntax without evaluating package expressions a second time.
# Unsupported expressions leave the reflected signature in charge.
function dispatch_value(expr, mod, scope)
    if expr isa Symbol
        return haskey(scope, expr) ? scope[expr] : getfield(mod, expr)
    elseif expr isa QuoteNode && expr.value isa Symbol
        return expr.value
    elseif expr isa Union{Bool,Integer,Char,AbstractFloat}
        return expr
    elseif Meta.isexpr(expr, :., 2)
        owner = dispatch_value(expr.args[1], mod, scope)
        name = expr.args[2]
        owner isa Module && name isa QuoteNode && return getfield(owner, name.value)
    elseif Meta.isexpr(expr, :curly)
        head = dispatch_value(first(expr.args), mod, scope)
        anonymous = TypeVar[]
        params = map(expr.args[2:end]) do arg
            if Meta.isexpr(arg, (:<:, :>:), 1)
                bound = dispatch_value(only(arg.args), mod, scope)
                variable = arg.head === :<: ? TypeVar(gensym(), Union{}, bound) : TypeVar(gensym(), bound, Any)
                push!(anonymous, variable)
                return variable
            end
            return dispatch_value(arg, mod, scope)
        end
        return wrap_dispatch(Core.apply_type(head, params...), anonymous)
    elseif Meta.isexpr(expr, :where)
        nested = copy(scope)
        variables = dispatch_variables(expr.args[2:end], mod, nested)
        return wrap_dispatch(dispatch_value(first(expr.args), mod, nested), variables)
    elseif Meta.isexpr(expr, :tuple)
        return Tuple(dispatch_value(arg, mod, scope) for arg in expr.args)
    elseif Meta.isexpr(expr, :call, 2) && expr.args[1] === :typeof && getfield(mod, :typeof) === typeof
        return typeof(dispatch_value(expr.args[2], mod, scope))
    end
    throw(ArgumentError("Unsupported source type expression."))
end

function dispatch_variables(specs, mod, scope)
    return map(specs) do spec
        name, lower, upper = if spec isa Symbol
            (spec, Union{}, Any)
        elseif Meta.isexpr(spec, :<:, 2)
            (spec.args[1], Union{}, dispatch_value(spec.args[2], mod, scope))
        elseif Meta.isexpr(spec, :>:, 2)
            (spec.args[1], dispatch_value(spec.args[2], mod, scope), Any)
        elseif Meta.isexpr(spec, :comparison, 5) && spec.args[2] === spec.args[4] === :<:
            (spec.args[3], dispatch_value(spec.args[1], mod, scope),
             dispatch_value(spec.args[5], mod, scope))
        else
            throw(ArgumentError("Unsupported source type variable."))
        end
        variable = TypeVar(name, lower, upper)
        scope[name] = variable
        return variable
    end
end

wrap_dispatch(type, variables) = foldr(UnionAll, variables; init=type)

function dispatch_argument(arg, mod, scope)
    Meta.isexpr(arg, :kw) && return dispatch_argument(first(arg.args), mod, scope)
    if Meta.isexpr(arg, :(...), 1)
        return Vararg{dispatch_argument(only(arg.args), mod, scope)}
    end
    Meta.isexpr(arg, :(::)) && return dispatch_value(last(arg.args), mod, scope)
    (arg isa Symbol || Meta.isexpr(arg, :tuple)) && return Any
    throw(ArgumentError("Unsupported source argument."))
end

function source_dispatch(expr, mod, value, macro_binding)
    scope = Dict{Symbol,TypeVar}()
    variables = TypeVar[]
    try
        while expr isa Expr
            if expr.head === :where
                append!(variables, dispatch_variables(expr.args[2:end], mod, scope))
            elseif expr.head === :(::) && Meta.isexpr(last(expr.args), :where)
                # A return annotation can carry the method's where clause.
                append!(variables, dispatch_variables(last(expr.args).args[2:end], mod, scope))
            elseif expr.head === :macrocall
                NativeCompat.macro_name(first(expr.args)) in
                    (Symbol("@inline"), Symbol("@noinline"), Symbol("@propagate_inbounds")) || return nothing
                expr = last(expr.args)
                continue
            elseif expr.head === :call
                callee = first(expr.args)
                macro_binding && callee isa Symbol && (callee = Symbol('@', callee))
                target = dispatch_value(callee, mod, scope)
                head = if value isa Type
                    base = Base.unwrap_unionall(target)
                    base isa DataType && base.name === Base.unwrap_unionall(value).name ||
                        return nothing
                    Type{target}
                else
                    target === value || return nothing
                    typeof(value)
                end
                types = [dispatch_argument(arg, mod, scope) for arg in expr.args[2:end]
                         if !Meta.isexpr(arg, :parameters)]
                return wrap_dispatch(Tuple{head, types...}, variables)
            elseif expr.head ∉ (:function, :(=), :macro, :(::))
                return nothing
            end
            expr = first(expr.args)
        end
    catch err
        err isa Union{ArgumentError,TypeError,MethodError,UndefVarError} || rethrow()
    end
    return nothing
end

function live_dispatch(method, macro_binding)
    macro_binding || return method.sig
    head = Base.unwrap_unionall(method.sig).parameters[1]
    return Base.rewrap_unionall(Tuple{head, positional(method, true)...}, method.sig)
end

function method_record(root, method, value, name, macro_binding, headers)
    _, constraints, variables = type_variables(method.sig)
    arguments = method_arguments(method, macro_binding)
    positionals = positional(method, macro_binding)
    types = [type_spelling(type, variables) for type in positionals]
    live = live_dispatch(method, macro_binding)
    offsets = [offset for (offset, header) in headers
               if source_dispatch(header, method.module, value, macro_binding) == live]
    keywords = [String(key) for key in NativeCompat.keywords(method) if key != Symbol("...")]
    name = macro_binding ? name[2:end] : name
    binding = Base.isoperator(Symbol(name)) ? name : name_token(Symbol(name))
    if value isa Type
        head = NativeCompat.constructor_head(method, value)
        if head !== value
            parameters = Base.unwrap_unionall(head).parameters
            binding *= "{" * join([type_spelling(type, variables) for type in parameters], ", ") * "}"
        end
    end
    signatures = [type == "Any" ? argument : argument * "::" * type
                  for (argument, type) in zip(arguments, types)]
    args = join(signatures, ", ")
    isempty(keywords) || (args *= "; " * join(keywords, ", "))
    display = binding * "(" * args * ")" * where_clause(constraints)
    selector = binding * "(" * join(["::" * type for type in types], ", ") * ")" * where_clause(constraints)
    return merge(Dict{String,Any}(
        "module" => module_name(method.module), "types" => types, "arguments" => arguments,
        "source_offsets" => offsets,
        "keywords" => keywords, "display" => display, "selector" => selector, "docs" => Any[],
    ), location(root, method.file, method.line))
end

function attached_method(entries, signature, macro_binding, doc, value, registered, headers)
    docs_body = Base.unwrap_unionall(signature)
    branches = Base.uniontypes(docs_body)
    isempty(branches) && return nothing
    # Docs uses singleton type-variable tuples to preserve bindings, but one
    # can also be the full method after Docs adds the optional-arity branches.
    # Match the live methods first and prefer the widest matching arity.
    marker(branch) = length(branch.parameters) == 1 && branch.parameters[1] isa TypeVar
    required = maximum((length(branch.parameters) for branch in branches if !marker(branch)); init=0)
    keys = [Base.rewrap_unionall(branch, signature) => !marker(branch) for branch in branches
            if length(branch.parameters) >= required]
    priorities = map(entries) do method
        # Rewrapping preserves bound variables; recreating them from display
        # strings would conflate unrelated types with the same display name.
        types = positional(method, macro_binding)
        call = Base.rewrap_unionall(Tuple{types...}, method.sig)
        # At the same arity a concrete branch is stronger evidence than a
        # marker, which could also match an unrelated overload.
        maximum(((length(types), concrete) for (key, concrete) in keys if key == call);
                init=(-1, false))
    end
    matches = findall(priority -> first(priority) >= 0, priorities)
    if length(matches) > 1
        authored = [source_dispatch(header, registered, value, macro_binding) for header in headers]
        selected = filter(index -> any(==(live_dispatch(entries[index], macro_binding)), authored), matches)
        length(selected) == 1 && return only(selected)
        best = maximum(index -> priorities[index], matches)
        filter!(index -> priorities[index] == best, matches)
        path = get(doc.data, :path, nothing)
        line = get(doc.data, :linenumber, 0)
        local_matches = filter(index -> String(entries[index].file) == path &&
                                        entries[index].line >= line, matches)
        if !isempty(local_matches)
            nearest = minimum(index -> entries[index].line, local_matches)
            matches = filter(index -> entries[index].line == nearest, local_matches)
        end
    end
    isempty(matches) && return nothing
    length(matches) == 1 || error("a documentation attachment matches multiple live methods")
    return only(matches)
end

function type_record(value)
    body, parameters, variables = type_variables(value)
    flavor = isabstracttype(body) ? "abstract" : isprimitivetype(body) ? "primitive" : "struct"
    fields = [Dict("name" => String(name), "type" => type_spelling(fieldtype(body, index), variables),
                   "const" => ismutabletype(body) && Base.isconst(body, index), "docs" => Any[])
              for (index, name) in enumerate(flavor == "abstract" ? () : fieldnames(body))]
    return Dict{String,Any}("flavor" => flavor, "mutable" => ismutabletype(body),
                            "bits" => flavor == "primitive" ? 8 * sizeof(body) : 0,
                            "supertype" => type_spelling(supertype(body), variables),
                            "parameters" => parameters, "fields" => fields)
end

function canonical_binding(mod, name, value)
    if value isa Module
        return parentmodule(value), nameof(value)
    end
    if value isa Function || Base.unwrap_unionall(value) isa DataType
        owner = parentmodule(value)
        symbol = nameof(value)
        if isdefined(owner, symbol) && getfield(owner, symbol) === value
            return owner, symbol
        end
    end
    origin = NativeCompat.binding_origin(mod, name)
    return origin.mod, origin.name
end

function binding_name(mod, name, value)
    value isa Module && return module_name(value)
    owner, symbol = canonical_binding(mod, name, value)
    return qualified(owner, symbol)
end

function package_modules(root)
    modules = Module[root]
    for mod in modules
        for name in names(mod; all=true, imported=true)
            isdefined(mod, name) || continue
            value = getfield(mod, name)
            if value isa Module && parentmodule(value) === mod && value ∉ modules
                push!(modules, value)
            end
        end
    end
    return modules
end

function collect_surface(root, loaded, receipts)
    modules = package_modules(loaded)
    bindings = Dict{String,Tuple{Module,Symbol,Any}}()
    aliases = Dict{String,Vector{Dict{String,String}}}()
    attachments = Dict{String,Vector{Tuple{Module,Any,Any}}}()
    public = Dict{String,String}()
    exports = Dict{String,String}[]
    errors = String[]
    for mod in sort!(collect(modules); by=module_name)
        bindings[module_name(mod)] = (parentmodule(mod), nameof(mod), mod)
        for name in sort!(union(names(mod; all=true, imported=true), names(mod)); by=string)
            name === nameof(mod) && continue
            startswith(String(name), '#') && continue
            visible = NativeCompat.public_binding(mod, name)
            if !isdefined(mod, name)
                visible && push!(errors, "Public Julia binding `$(qualified(mod, name))` is undefined.")
                continue
            end
            value = getfield(mod, name)
            canonical = binding_name(mod, name, value)
            local_name = qualified(mod, name)
            owner, symbol = canonical_binding(mod, name, value)
            bindings[canonical] = (owner, symbol, value)
            if canonical != local_name
                push!(get!(aliases, canonical, []),
                      Dict("name" => local_name, "kind" => Base.binding_module(mod, name) === mod ?
                           "assignment" : "import"))
            end
            if visible
                visibility = Base.isexported(mod, name) ? "exported" : "public"
                push!(exports, Dict("module" => module_name(mod), "name" => String(name),
                                    "visibility" => visibility))
                if get(public, canonical, "private") != "exported"
                    public[canonical] = visibility
                end
            end
        end
        table = NativeCompat.documentation(mod)
        table === nothing && continue
        for (binding, docs) in table
            if !Base.Docs.defined(binding)
                push!(errors, "Julia documentation targets an undefined binding `$(qualified(binding.mod, binding.var))`.")
                continue
            end
            value = Base.Docs.resolve(binding)
            canonical = binding_name(binding.mod, binding.var, value)
            owner, symbol = canonical_binding(binding.mod, binding.var, value)
            bindings[canonical] = (owner, symbol, value)
            for signature in docs.order
                push!(get!(attachments, canonical, []), (mod, signature, docs.docs[signature]))
            end
        end
    end

    retained = union(keys(public), keys(attachments), [module_name(loaded)])
    for name in collect(retained)
        haskey(bindings, name) || continue
        mod, _, _ = bindings[name]
        while mod in modules
            push!(retained, module_name(mod))
            mod === loaded && break
            mod = parentmodule(mod)
        end
    end

    for mod in modules
        explicit = Set(names(mod; all=true, imported=true))
        candidates = Set(name for used in NativeCompat.used_modules(mod) for name in names(used)
                         if Base.isexported(used, name))
        for name in setdiff(candidates, explicit)
            # Ask Julia to resolve the name so that shadows and ambiguous
            # exports cannot introduce a lookup alias.
            isdefined(mod, name) && Base.binding_module(mod, name) !== mod || continue
            canonical = binding_name(mod, name, getfield(mod, name))
            canonical in retained || continue
            local_name = qualified(mod, name)
            canonical == local_name && continue
            entries = get!(aliases, canonical, [])
            any(alias -> alias["name"] == local_name, entries) ||
                push!(entries, Dict("name" => local_name, "kind" => "import"))
        end
    end

    tracked = Set(joinpath(root, input[key]) for input in receipts for key in ("path", "canonical"))
    sources = Dict{String,NamedTuple}()
    function headers(path)
        path isa Union{String,Symbol} || return nothing
        file = normpath(String(path))
        file in tracked || return nothing
        return get!(() -> NativeCompat.source_headers(file), sources, file)
    end

    items = Any[]
    for canonical in sort!(collect(retained))
        owner, symbol, value = bindings[canonical]
        name = String(symbol)
        observed = Dict{String,Any}(
            "name" => name, "qualified_name" => canonical,
            "module" => value === loaded ? "" : owner in modules ? module_name(owner) : "",
            "aliases" => sort!(get(aliases, canonical, []); by=alias -> alias["name"]),
            "docs" => Any[], "methods" => Any[],
        )
        docs = get(attachments, canonical, [])
        try
            macro_binding = value isa Function && startswith(name, '@')
            entries = Method[]
            if value isa Module && value in modules
                observed["kind"] = "module"
            elseif value isa Union{DataType,UnionAll} && owner in modules && nameof(value) === symbol
                observed["kind"] = "type"
                observed["type"] = type_record(value)
                entries = methods_for(value, modules)
            elseif value isa Function && !(value isa Core.Builtin)
                observed["kind"] = macro_binding ? "macro" : "function"
                entries = methods_for(value, modules)
                if owner ∉ modules
                    isempty(entries) && error("external-only reexports have no package-owned methods")
                    observed["module"] = module_name(first(sort!(entries; by=m -> module_name(m.module))).module)
                end
            elseif owner in modules && isconst(owner, symbol)
                observed["kind"] = "constant"
                observed["annotation"] = type_spelling(typeof(value), IdDict{TypeVar,String}())
            else
                error("the binding's runtime declaration kind is unsupported")
            end
            for entry in entries
                source = headers(entry.file)
                declarations = source === nothing ? () : get(source.declarations, entry.line, ())
                push!(observed["methods"], method_record(root, entry, value, name, macro_binding, declarations))
            end
            for (registered, signature, doc) in docs
                parsed = document(root, doc, registered)
                if signature === Union{} || macro_binding
                    push!(observed["docs"], parsed)
                    if observed["kind"] == "type"
                        for field in observed["type"]["fields"]
                            field_doc = get(get(doc.data, :fields, Dict()), Symbol(field["name"]), nothing)
                            if field_doc isa AbstractString
                                push!(field["docs"], merge(copy(parsed),
                                                           Dict("text" => String(field_doc))))
                                delete!(only(field["docs"]), "unsupported")
                            end
                        end
                    end
                else
                    source = headers(get(doc.data, :path, nothing))
                    authored = source === nothing ? () : get(source.attachments, get(doc.data, :linenumber, 0), ())
                    index = attached_method(entries, signature, macro_binding, doc, value, registered, authored)
                    index === nothing && error("a documentation attachment has no exact live method")
                    push!(observed["methods"][index]["docs"], parsed)
                end
            end
            sort!(observed["methods"]; by=method -> (get(method, "path", ""),
                   get(method, "line", 0), method["selector"], method["module"]))
            push!(items, observed)
        catch err
            push!(errors, "Julia binding `$canonical`: $(sprint(showerror, err))")
        end
    end
    return items, exports, sort!(unique(errors))
end

# Julia resolves dependencies in the prepared environment. Verify both the
# checkout identity and the source actually used by the loader.
function observe_checkout(package_path, entry_path, output_path)
    VERSION.major == 1 && 10 <= VERSION.minor <= 13 ||
        error("Julia native extraction supports Julia 1.10 through 1.13.")
    timings = splitext(output_path)[1] * ".timings"
    started = time_ns()
    root = realpath(package_path)
    project_path = joinpath(root, "Project.toml")
    project_bytes = read(project_path)
    project = TOML.parse(String(copy(project_bytes)))
    identity = Base.PkgId(Base.UUID(project["uuid"]), project["name"])
    receipts = Dict{String,String}[]
    entry = realpath(entry_path)
    callback = (mod, file) -> begin
        if Base.PkgId(mod).uuid == identity.uuid || file == entry
            scope = file == entry ? "" : module_name(mod)
            push!(receipts, source_receipt(root, file, scope))
        end
        nothing
    end
    push!(Base.include_callbacks, callback)
    loaded = try
        Base.require(identity)
    finally
        filter!(value -> value !== callback, Base.include_callbacks)
    end
    Base.PkgId(loaded) == identity || error("Loaded package identity conflicts with Project.toml.")
    nameof(loaded) == Symbol(project["name"]) || error("Loaded package name conflicts with Project.toml.")
    version = Base.pkgversion(loaded)
    if haskey(project, "version")
        version == VersionNumber(project["version"]) || error("Loaded package version conflicts with Project.toml.")
    end
    loaded_entry = pathof(loaded)
    loaded_entry !== nothing && realpath(loaded_entry) == entry ||
        error("Loaded package does not match the supplied checkout.")
    complete = cached_inputs!(receipts, root, identity)
    for input in receipts
        input["canonical"] == package_file(root, entry) && (input["module"] = "")
    end
    any(input -> input["canonical"] == package_file(root, entry), receipts) ||
        error("Julia did not report the loaded checkout entry.")
    read(project_path) == project_bytes || error("Julia project changed while loading the checkout.")
    verify_inputs(root, receipts)

    write(timings, "package-loading\t$(Float64(time_ns() - started) / 1e9)\n")
    started = time_ns()
    items, exports, errors = Base.invokelatest(collect_surface, root, loaded, receipts)
    dependencies = Dict(string(id.uuid) => string(version) for (id, dependency) in Base.loaded_modules
                        if (version = Base.pkgversion(dependency)) !== nothing)
    result = Dict(
        "policy" => EXPORT_POLICY, "julia_version" => string(VERSION),
        "package_name" => identity.name, "package_uuid" => string(identity.uuid),
        "package_version" => string(something(Base.pkgversion(loaded), "")),
        "project_digest" => sha(project_bytes), "dependencies" => dependencies,
        "inputs" => sort!(unique(receipts); by=input -> (input["path"], input["module"])),
        "complete" => complete, "items" => items, "exports" => exports, "errors" => errors,
    )
    verify_inputs(root, receipts)
    open(timings, "a") do io
        println(io, "extraction\t$(Float64(time_ns() - started) / 1e9)")
    end
    open(output_path, "w") do io
        TOML.print(io, result; sorted=true)
    end
end

if abspath(PROGRAM_FILE) == @__FILE__
    observe_checkout(ARGS...)
end