midenup 1.0.0

The Miden toolchain manager
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
use std::{
    borrow::Cow,
    collections::{BTreeMap, HashMap},
    path::PathBuf,
    time::SystemTime,
};

use anyhow::Context;
use serde::Deserialize;

use crate::{
    channel::Channel,
    commands,
    config::Config,
    fault,
    options::{InstallationOptions, IntentUpdate},
    paths,
    resolve::Intent,
    state::{Installation, LocalState, PublicationId, PublicationRef},
    utils,
    version::{Authority, GitTarget},
};

/// Installs `channel` as `options` describes it.
///
/// **`channel` is always the full upstream channel**, and what gets installed is decided here, by
/// resolving the effective intent against it. Every operation -- a direct install, a toolchain-file
/// activation, an update, a channel migration -- differs only in how that intent is derived, and
/// they all arrive here. A caller that narrowed the channel before passing it would be deciding
/// "what should be installed" a second time, in terms the intent recorded here cannot see.
pub fn install(
    config: &Config,
    channel: &Channel,
    state: &mut LocalState,
    options: &InstallationOptions,
) -> anyhow::Result<()> {
    commands::setup_midenup(config)?;

    let home = &config.midenup_home;

    // Every install produces a *new* publication, named opaquely. Nothing may infer identity from
    // the name: a name derived from the plan key would invite treating equal keys as equal bytes,
    // which nothing verifies.
    let publication_id = PublicationId::generate();
    let publication = paths::publication_dir(home, &channel.name, &publication_id);

    // The single decision this whole function turns on.
    let intent = carry_migrated_intent(state, channel, effective_intent(state, channel, options));
    let plan = crate::plan::build_plan(channel, &intent, config.target(), &publication)?;

    // What the publication being replaced owns, if this build published it. Only a receipt can
    // say; a directory listing cannot distinguish installed content from anything else that
    // happens to be there.
    let previous = previous_publication(config, state, &channel.name);
    let stale = options.stale.clone();

    // 1. PREPARE. The record this operation intends to commit is written down *before* any of it
    // happens, so that a crash anywhere after this point can be completed or discarded rather than
    // reconstructed by inspection.
    let mut entry = crate::publish::JournalEntry::install(
        channel.name.clone(),
        previous_publication_id(state, &channel.name),
        publication_id.clone(),
        target_installation(config, channel, &intent, options, &publication_id, &plan)?,
    );
    crate::publish::journal::prepare(home, &entry)?;
    fault::fail_at(fault::FaultPoint::PostPrepare)?;

    // 2. STAGE.
    crate::install::prepare(&publication)?;
    if let Some((previous_dir, receipt)) = &previous {
        crate::install::seed(
            &plan,
            &publication,
            &crate::install::Seed {
                publication: previous_dir,
                receipt,
                stale: &stale,
            },
        )?;
    }

    let realized = crate::install::execute(
        &plan,
        &publication,
        crate::report::subprocess_output_visible(),
        config.debug,
    )?;

    // Spec section 9.2: a `path` source that moves *during* the build produces an installation
    // matching neither the tree we pinned nor the one on disk now, and nothing else would ever
    // report it. Checked before the commit point, so the answer is to retry rather than to work
    // out what was published.
    for step in &plan.steps {
        if let Some(authority) = step.authority() {
            crate::plan::recheck_path(authority)?;
        }
    }

    fault::fail_at(fault::FaultPoint::PostStage)?;

    // 3. VERIFY. Structural check before anything is published: every planned file exists, is a
    // regular file, and carries the planned mode. Contents are not verified -- digests are
    // recorded but never checked -- so this asserts the plan was carried out, not what was
    // installed.
    crate::install::verify(&plan, &publication)?;

    // The receipt makes the publication self-describing: from here on, what it owns is a fact
    // recorded inside it rather than something re-derived from a manifest that may have moved on.
    let receipt = crate::publish::receipt_for(
        &plan,
        &publication,
        &publication_id,
        &realized,
        previous.as_ref().map(|(_, receipt)| receipt),
    );
    crate::publish::write_receipt(&publication, &receipt)?;

    // A `cargo install --path` build can touch its own source tree, so a `path` component's
    // modification time is only knowable once the build is done. Refreshing it here and amending
    // the journal -- atomically, and still before the commit point -- keeps the recorded time
    // equal to what the *next* run will observe before building. Recording the pre-build time
    // instead makes every subsequent update believe the source changed.
    if let Some(installation) = entry.target_installation.as_mut() {
        refresh_path_modification_times(config, &mut installation.components);
    }
    crate::publish::journal::prepare(home, &entry)?;
    fault::fail_at(fault::FaultPoint::PostVerify)?;

    // ======================== 4. COMMIT — the commit point ======================
    //
    // A single atomic rename of a symlink. Before it, this operation never happened and recovery
    // discards it; after it, recovery completes it. Nothing else distinguishes the two.
    crate::publish::journal::commit_symlink(home, &entry)?;
    fault::fail_at(fault::FaultPoint::PostCommit)?;

    // 5. RECORD.
    crate::publish::journal::record(home, &entry, state)?;
    fault::fail_at(fault::FaultPoint::PostRecord)?;

    // 6. DERIVE. Which channel a network names is a property of the upstream manifest, recomputed
    // from it rather than remembered, so a stale local copy can never disagree with upstream. A
    // loop rather than a conditional because several networks may name one channel -- the state
    // right after a testnet toolchain is promoted to mainnet.
    //
    // Only the channel being installed gets links. Repointing a network at a channel that is not
    // installed would leave a dangling symlink; `midenup update <network>` is what advances it.
    let relative_channel_target = PathBuf::from(format!("{}", channel.name));
    for network in config.upstream_manifest()?.networks_for(&channel.name) {
        // A network name becomes a path segment under `toolchains/`, and `replace_symlink` renames
        // over whatever is at that path. Loading a manifest is deliberately permissive, so the
        // authoring gate in `manifest::validate` cannot be the only thing standing between a
        // manifest and a symlink written outside `$MIDENUP_HOME`.
        if crate::plan::validate_artifact_id(network).is_err() {
            continue;
        }

        let link = paths::network_link(home, network);
        crate::trace!("linking {} -> {}", link.display(), relative_channel_target.display());
        utils::fs::replace_symlink(&link, &relative_channel_target).with_context(|| {
            format!("failed to point '{network}' at the newly installed channel")
        })?;
    }
    fault::fail_at(fault::FaultPoint::PostDerive)?;

    // 7. CLEAN.
    crate::publish::journal::clean(home, &entry)?;

    crate::info!("installed channel '{}'", channel.name);

    Ok(())
}

/// What this operation wants installed.
///
/// Installing and *recording what the user wants* are separate concerns, and this is where they
/// meet: the effective intent is both what gets resolved into a plan and what gets persisted. A
/// toolchain-file activation may only add to the record, so it unions; a direct install restates
/// it, and is allowed to shrink; an update re-resolves what is already recorded rather than
/// restating it.
pub(crate) fn effective_intent(
    state: &LocalState,
    channel: &Channel,
    options: &InstallationOptions,
) -> Intent {
    // What the caller asked for on this invocation.
    let requested = Intent {
        profiles: [options.profile].into_iter().collect(),
        roots: options.components.iter().cloned().collect(),
    };
    let previous = state.get(&channel.name).map(|installation| installation.intent.clone());

    match options.intent_update.clone() {
        // A direct `midenup install`: record exactly what the command line asked for.
        None => requested,
        Some(IntentUpdate::Replace(intent)) => intent,
        Some(IntentUpdate::Union(intent)) => {
            let mut merged = previous.unwrap_or_default();
            merged.union_with(&intent);
            merged
        },
        Some(IntentUpdate::Preserve) => previous.unwrap_or(requested),
    }
}

/// Carries a migrated selection into the install that replaces it, dropping what upstream no longer
/// has.
///
/// Two rules, both from spec section 12, and both scoped to a record that is still
/// `NeedsReinstall`:
///
/// **The migrated selection is carried, not replaced.** A migrated record exists because the
/// toolchain *was* installed; the install that resolves it is a continuation of that, not a fresh
/// choice, so `midenup install <channel>` reinstalls what the user had rather than silently
/// reducing it to the default profile. Once the record is managed, `install` replaces intent as
/// usual (section 8.1) -- which is also how a user deliberately shrinks it.
///
/// **Roots upstream no longer has are dropped with a warning.** Section 11.3 blocks an update when
/// an explicit root disappears, because the user chose that root deliberately. A migrated root was
/// not chosen in those terms -- it was inferred from a v1 record -- so blocking would strand every
/// v1 user whose channel happened to drop a component, with no way forward but deleting their state
/// by hand.
///
/// Both are one-time by construction rather than by a flag: the install they are part of replaces
/// the migrated record with a managed one.
fn carry_migrated_intent(state: &LocalState, channel: &Channel, intent: Intent) -> Intent {
    use colored::Colorize;

    let Some(migrated) = state.get(&channel.name).filter(|installation| !installation.is_managed())
    else {
        return intent;
    };

    let mut intent = intent;
    intent.union_with(&migrated.intent);

    let (kept, dropped): (Vec<String>, Vec<String>) = intent
        .roots
        .iter()
        .cloned()
        .partition(|root| channel.get_component(root).is_some());

    if dropped.is_empty() {
        return intent;
    }

    crate::report::prepare_stderr_color();
    let listed = dropped.iter().map(|root| format!("\n- {}", root.bold())).collect::<String>();
    crate::warn!(
        "these components are no longer part of channel {} and have been dropped from your \
         selection:{listed}",
        channel.name
    );

    Intent {
        profiles: intent.profiles,
        roots: kept.into_iter().collect(),
    }
}

/// The state record this install intends to commit.
///
/// Built before anything is staged, because the journal carries it: recovery has to be able to
/// complete the operation without re-resolving it against an upstream manifest that may have moved
/// on in the meantime.
///
/// The component snapshot is the **resolved** set, pinned, rather than the whole channel: `miden`
/// dispatch reads it offline to decide what is available, and update needs to know what was
/// *actually* installed. Recording every component in the channel would make a `--profile minimal`
/// installation claim to have everything, so activation could never notice a missing component.
fn target_installation(
    config: &Config,
    channel: &Channel,
    intent: &Intent,
    options: &InstallationOptions,
    publication_id: &PublicationId,
    plan: &crate::plan::InstallationPlan,
) -> anyhow::Result<Installation> {
    let installed_components = {
        let mut installed_components: Vec<crate::manifest::Component> =
            crate::resolve::resolve(channel, intent)?.into_iter().cloned().collect();

        // How a component was really obtained can only be known after the fact.
        for component in installed_components.iter_mut() {
            match &component.version {
                // A branch is not a fixed point, so record the commit that was actually installed;
                // update compares against it to decide whether new commits have landed.
                Authority::Git {
                    repository_url,
                    subpath,
                    target: GitTarget::Branch { name, .. },
                } => {
                    // Leaving this empty on failure means an update is triggered unnecessarily,
                    // which is the safe direction to fail in.
                    let revision_hash = utils::git::find_latest_hash(repository_url, name).ok();

                    component.version = Authority::Git {
                        repository_url: repository_url.clone(),
                        subpath: subpath.clone(),
                        target: GitTarget::Branch {
                            name: name.clone(),
                            latest_revision: revision_hash,
                        },
                    }
                },
                Authority::Git { .. } | Authority::Path { .. } | Authority::Registry { .. } => (),
            }
        }

        // Path authorities are recorded here too, but their modification time is refreshed after
        // staging: see `refresh_path_modification_times`.
        refresh_path_modification_times(config, &mut installed_components);

        // A component the update policy declined to touch keeps the definition it was installed
        // with. Recording the upstream one instead would mark it up to date without having
        // rebuilt it, and the next update would stop offering.
        for component in installed_components.iter_mut() {
            if let Some(held) = options.held_back.iter().find(|held| held.name == component.name) {
                *component = held.clone();
            }
        }

        installed_components
    };

    Ok(Installation {
        channel: channel.name.clone(),
        intent: intent.clone(),
        components: installed_components,
        publication: PublicationRef::Managed {
            id: publication_id.clone(),
            plan_key: plan.key.clone(),
            target: config.target().to_string(),
        },
        installed_at: chrono::Utc::now().timestamp(),
    })
}

/// Records each `path` component's source tree modification time, canonicalizing the path.
///
/// Called once while assembling the record and again after staging, because a build can modify its
/// own source tree -- Cargo writes into it -- and what matters is that the recorded time matches
/// what the next run will see *before* it builds. Anything else makes every subsequent update
/// believe the source changed.
fn refresh_path_modification_times(config: &Config, components: &mut [crate::manifest::Component]) {
    for component in components.iter_mut() {
        let Authority::Path { path, .. } = &component.version else {
            continue;
        };

        let path = if path.is_absolute() {
            Cow::Borrowed(path.as_path())
        } else {
            Cow::Owned(config.working_directory.join(path.as_path()))
        };
        let latest_time = utils::fs::latest_modification(&path)
            .ok()
            .map(|(latest_modification, _)| latest_modification)
            .unwrap_or_else(SystemTime::now);

        component.version = Authority::Path {
            path: path.to_path_buf(),
            last_modification: Some(latest_time),
        };
    }
}

/// The publication currently recorded for `channel`, if this build published it.
fn previous_publication_id(state: &LocalState, channel: &semver::Version) -> Option<PublicationId> {
    match &state.get(channel)?.publication {
        PublicationRef::Managed { id, .. } => Some(id.clone()),
        PublicationRef::NeedsReinstall => None,
    }
}

/// The publication this install is replacing, and what it owns.
///
/// `None` when the channel is not installed, was carried over from v1 (so nothing describes it),
/// or its receipt is unreadable. In every one of those cases the correct behaviour is the same:
/// seed nothing and install from scratch. Guessing ownership from a directory listing is what this
/// exists to avoid.
fn previous_publication(
    config: &Config,
    state: &LocalState,
    channel: &semver::Version,
) -> Option<(PathBuf, crate::state::Receipt)> {
    let installation = state.get(channel)?;
    let PublicationRef::Managed { id, .. } = &installation.publication else {
        return None;
    };
    let dir = paths::publication_dir(&config.midenup_home, channel, id);
    let receipt = crate::publish::read_receipt(&dir).ok()?;
    Some((dir, receipt))
}

#[allow(unused)]
pub struct InstalledBinary {
    pub version: semver::Version,
    pub location: Authority,
    pub bins: Vec<String>,
    pub features: Vec<String>,
}

#[derive(Deserialize)]
struct CargoInstalls {
    #[serde(default)]
    installs: BTreeMap<String, InstalledCrateInfo>,
}

#[derive(Deserialize)]
#[serde(untagged)]
enum InstalledCrateInfo {
    Info {
        #[serde(default)]
        bins: Vec<String>,
        #[serde(default)]
        features: Vec<String>,
    },
    UnknownFormat,
}

/// Returns the names of all packages installed via cargo at the given root.
///
/// Runs `cargo install --list --root <root>` and parses each package header line.
#[allow(unused)]
pub fn get_installed_cargo_binaries(
    root_dir: PathBuf,
) -> anyhow::Result<HashMap<String, InstalledBinary>> {
    let crates2_json = root_dir.join(".crates2.json");
    if !crates2_json.exists() {
        return Ok(HashMap::new());
    }
    let crates2_json_file = std::fs::File::open(&crates2_json).with_context(|| {
        format!(
            "failed to obtain binaries installed via Cargo from '{}'",
            crates2_json.display()
        )
    })?;
    let installs =
        serde_json::from_reader::<_, CargoInstalls>(crates2_json_file).with_context(|| {
            format!("failed to deserialize Cargo's install manifest '{}'", crates2_json.display())
        })?;

    let mut installed = HashMap::new();

    for (crate_id, info) in installs.installs {
        let InstalledCrateInfo::Info { bins, features } = info else {
            continue;
        };
        if bins.is_empty() {
            continue;
        }
        let Some((crate_name, rest)) = crate_id.split_once(' ') else {
            continue;
        };
        let Some((crate_version, source)) = rest.split_once(' ') else {
            continue;
        };
        let crate_name = crate_name.trim();
        let Ok(crate_version) = semver::Version::parse(crate_version.trim()) else {
            continue;
        };
        let source = source.trim().trim_matches(['(', ')']);
        if let Some(path) = source.strip_prefix("path+") {
            installed.insert(
                crate_name.to_string(),
                InstalledBinary {
                    version: crate_version,
                    location: Authority::Path {
                        path: PathBuf::from(path.to_string()),
                        last_modification: None,
                    },
                    bins,
                    features,
                },
            );
        } else if source.starts_with("registry+") {
            let version = crate_version.clone();
            installed.insert(
                crate_name.to_string(),
                InstalledBinary {
                    version: crate_version,
                    location: Authority::Registry { version },
                    bins,
                    features,
                },
            );
        }
    }

    Ok(installed)
}