pub(crate) mod accounts;
mod actor;
pub(crate) use actor::Actor;
mod activity;
mod command_record;
pub mod dismiss;
mod effects;
mod live;
pub(crate) use live::Command as LiveCommand;
pub mod export;
pub(crate) mod github;
pub mod metrics;
pub(crate) mod owner;
pub(crate) mod pass_starts;
mod process_identity;
mod reading;
pub(crate) use reading::Framing;
pub(crate) use reading::catalog;
pub(crate) use reading::run_for_pull_request;
mod recording;
mod run_metrics;
pub(crate) mod session;
pub(crate) mod session_signals;
pub(crate) mod spans;
pub(crate) mod start_notice;
pub mod status;
pub mod stop;
pub(crate) mod trace;
mod usual_range;
use std::fmt::{self, Display};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{LazyLock, Mutex, MutexGuard, Once, PoisonError};
use crate::config::UserConfig;
use crate::github::ListedIssue;
use crate::harness::Choice;
use crate::issue::{IssueUrl, Repo};
use crate::session::Logged;
use activity::Kind;
pub use command_record::StageName as Stage;
pub(crate) use command_record::{
FailureCategory, Issue, TicketRecord, TicketState, experiment_rechecked,
};
use effects::OnMachine;
use recording::Record;
pub(crate) fn live_commands() -> anyhow::Result<Vec<LiveCommand>> {
let configured = record().logs_directory();
let root = match configured {
Some(root) => root,
None => UserConfig::load()?.logs_dir,
};
live::read(&root)
}
pub(crate) fn command_for_process(
config: &UserConfig,
repo: &Repo,
process_id: u32,
started_after: chrono::DateTime<chrono::Local>,
) -> anyhow::Result<Option<String>> {
Ok(
reading::repository_records(&root_under(&config.logs_dir, repo))?
.into_values()
.find(|tree| {
tree.command.process_id == process_id
&& tree.command.parent_record.is_none()
&& chrono::DateTime::parse_from_rfc3339(&tree.command.started_at)
.is_ok_and(|start| start >= started_after)
})
.map(|tree| tree.command.name),
)
}
fn unix_user(action: &str) -> anyhow::Result<String> {
use anyhow::Context;
let failure = format!("could not identify the Unix user asking for {action}");
let user = std::process::Command::new("id")
.arg("-un")
.output()
.with_context(|| failure.clone())?;
if !user.status.success() {
anyhow::bail!(failure);
}
Ok(String::from_utf8(user.stdout)?.trim().to_owned())
}
pub(crate) fn display_name(stored: &str) -> &str {
match stored {
"Run" => "Ticket run",
"dispatched Run" => "dispatched Ticket run",
other => other,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CommandKind {
Issue,
Architect,
Pickup,
Secure,
Scoring,
Triage,
Experiment,
Process,
Review,
Fix,
Release,
}
impl CommandKind {
pub fn folder(self) -> &'static str {
match self {
Self::Issue => "issue",
Self::Architect => "architect",
Self::Pickup => "pickup",
Self::Secure => "secure",
Self::Scoring => "scoring",
Self::Triage => "triage",
Self::Experiment => "experiment",
Self::Process => "process",
Self::Review => "review",
Self::Fix => "fix",
Self::Release => "release",
}
}
pub fn named(name: &str) -> Option<Self> {
match name {
"issue" => Some(Self::Issue),
"architect" => Some(Self::Architect),
"pickup" => Some(Self::Pickup),
"secure" => Some(Self::Secure),
"scoring" => Some(Self::Scoring),
"triage" => Some(Self::Triage),
"experiment" => Some(Self::Experiment),
"process" => Some(Self::Process),
"review" => Some(Self::Review),
"fix" => Some(Self::Fix),
"release" => Some(Self::Release),
_ => None,
}
}
}
pub enum Begin<'a> {
Run(&'a IssueUrl),
ChildRun(&'a str, CommandKind),
ArchitectRun,
PickupRun,
SecurityRun,
ScoringRun,
ReleaseRun {
pass: bool,
},
ReviewRun {
pass: bool,
},
FixRun {
pass: bool,
},
TriageRun {
pass: bool,
},
Experiment,
Process {
name: &'a str,
pass: bool,
},
}
#[derive(Clone, Copy)]
pub enum Work<'a> {
Starting {
repo: &'a Repo,
command: CommandKind,
issue: Option<&'a IssueUrl>,
},
Run(&'a IssueUrl),
BaseFix(&'a IssueUrl),
SpecRun(&'a IssueUrl),
PickupRun(&'a ListedIssue),
ArchitectRun(&'a Repo),
SecurityRun(&'a Repo),
ScoringRun(&'a Repo),
ReviewRun(&'a Repo),
FixRun(&'a Repo),
ReleaseRun(&'a Repo),
TriageRun {
repo: &'a Repo,
issue: Option<&'a IssueUrl>,
},
Experiment(&'a Repo),
Process {
repo: &'a Repo,
name: &'a str,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Pass {
Architect,
Pickup,
Security,
Review,
Fix,
Triage,
Release,
}
impl Pass {
const ALL: [Self; 7] = [
Self::Architect,
Self::Pickup,
Self::Security,
Self::Review,
Self::Fix,
Self::Triage,
Self::Release,
];
pub(crate) fn name(self) -> &'static str {
match self {
Self::Architect => "Architect",
Self::Pickup => "Pickup",
Self::Security => "Security",
Self::Review => "Review",
Self::Fix => "Fix",
Self::Triage => "Triage",
Self::Release => "Release",
}
}
}
impl Display for Pass {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{} run", self.name())
}
}
impl serde::Serialize for Pass {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.collect_str(self)
}
}
impl<'de> serde::Deserialize<'de> for Pass {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let name = <String as serde::Deserialize>::deserialize(deserializer)?;
Self::ALL
.into_iter()
.find(|pass| pass.to_string() == name)
.ok_or_else(|| serde::de::Error::custom("unknown Pass kind"))
}
}
static RECORD: LazyLock<Mutex<Record<OnMachine>>> =
LazyLock::new(|| Mutex::new(Record::new(OnMachine)));
pub(crate) struct Nested(Option<Record<OnMachine>>);
pub(crate) fn nested() -> Nested {
let mut current = record();
let mut child = Record::new(OnMachine);
child.child_of(current.record_path());
Nested(Some(std::mem::replace(&mut *current, child)))
}
impl Drop for Nested {
fn drop(&mut self) {
*record() = self.0.take().expect("nested recording restored once");
attach_quota_budget();
}
}
const PARENT_RECORD: &str = "THIRDSHIFT_PARENT_RECORD";
const PARENT_DISPATCH: &str = "THIRDSHIFT_PARENT_DISPATCH";
pub fn begin(begin: Begin) {
crate::github::quota::command::begin();
static SAMPLER: Once = Once::new();
SAMPLER.call_once(|| {
std::thread::spawn(|| {
loop {
std::thread::sleep(std::time::Duration::from_secs(30));
drop(record());
}
});
});
let mut record = record();
if matches!(begin, Begin::ChildRun(..)) {
record.inherit_owner(owner::Owner::inherited());
record.child_of(std::env::var_os(PARENT_RECORD).map(PathBuf::from));
record.child_dispatch(std::env::var(PARENT_DISPATCH).ok());
}
record.begin(begin);
drop(record);
attach_quota_budget();
}
fn attach_quota_budget() {
if let Ok(owner) = captured_owner()
&& let Some(path) = owner.log_path()
&& let Err(error) = crate::github::quota::command::attach(path.into())
{
crate::progress::step(format!(
"warning: could not attach Command quota budget: {error:#}"
));
}
}
pub fn inherit_record(command: &mut std::process::Command) {
if let Some(path) = record().record_path() {
command.env(PARENT_RECORD, path);
} else {
command.env_remove(PARENT_RECORD);
}
if let Some(issue) = record().dispatch_issue() {
command.env(PARENT_DISPATCH, issue);
} else {
command.env_remove(PARENT_DISPATCH);
}
}
pub(crate) fn tickets(tickets: Vec<TicketRecord>) {
record().tickets(tickets);
}
pub fn configured(config: &UserConfig) {
crate::poll::configure_waiting_lines(config.effective_waiting_line_interval());
record().configured(config);
}
pub(crate) fn preflight_issue(issue: &IssueUrl) {
record().preflight_issue(issue.clone());
}
pub(crate) fn pass_fired(repo: &Repo) {
record().pass_fired(repo);
}
pub fn started(work: Work, harness: &Choice) {
record().started(work, harness);
attach_quota_budget();
}
pub(crate) fn started_custom_runner(work: Work) {
started(work, &Choice::default());
record().name_executor(CUSTOM_RUNNER_EXECUTOR);
}
pub(crate) const CUSTOM_RUNNER_EXECUTOR: &str = "custom runner";
pub(crate) fn process_loaded(identity: crate::processes::Identity) {
record().process_loaded(identity);
}
pub(crate) fn process_running(identity: crate::processes::Identity) {
record().process_running(identity);
}
pub(crate) fn custom_process() -> Option<crate::processes::Identity> {
record().custom_process()
}
pub(crate) fn waiting() {
if record().needs_preflight_repo() {
let repo = crate::launch::repo().ok();
record().preflight_repo(repo);
}
record().waiting();
}
pub fn skipped(pass: Pass, repo: &Repo, reason: impl Display) {
start_notice::skipped(&reason);
record().skipped(pass, repo, reason);
}
pub(crate) fn base_ci_skipped(repo: &Repo, reason: &str) {
record().base_ci_skipped(repo, reason);
}
pub fn skipped_process(
repo: &Repo,
name: &str,
harness: &crate::harness::Choice,
reason: &crate::command_limit::NotAdmitted,
) {
record().skipped_process(repo, name, harness, reason);
}
pub(crate) fn activity(message: impl Display) {
record().activity(message);
}
pub(crate) fn activity_for(repo: &Repo, message: impl Display) {
record().activity_for(repo, message);
}
pub fn ended(outcome: impl Display) {
record().ended(outcome);
}
pub fn issue_title(title: Option<String>) {
record().issue_title(title);
}
pub fn stage(stage: Stage) {
record().stage(stage, None);
}
pub(crate) fn waiting_for(commands: &[String]) {
record().waiting_for(commands);
}
pub(crate) fn quota_wait(until: Option<&str>) {
if until.is_some() && record().needs_preflight_repo() {
let repo = crate::launch::repo().ok();
record().preflight_repo(repo);
}
record().quota_wait(until);
if until.is_some() {
attach_quota_budget();
}
}
pub fn session(session: Logged, path: &Path) {
record().stage(session.stage(), Some(path));
}
pub fn session_choice(choice: &crate::harness::Choice) {
record().session_choice(choice);
}
pub(crate) fn implemented_on(configured: &crate::harness::Choice) -> crate::harness::Choice {
let recorded = record().implemented_on();
match recorded.and_then(|(name, model, effort)| {
Some((crate::harness::Harness::named(&name)?, model, effort))
}) {
Some((harness, model, effort)) => crate::harness::Choice {
harness,
model,
effort,
..configured.clone()
},
None => configured.clone(),
}
}
pub fn issue_branch(branch: Option<&str>) {
record().issue_branch(branch);
}
pub(crate) fn pull_request(url: &str) {
record().pull_request(url);
}
pub(crate) fn release_url(url: &str) {
record().release_url(url);
}
pub(crate) fn pull_request_title(title: Option<String>) {
record().pull_request_title(title);
}
pub(crate) fn base_branch(branch: &str) {
record().base_branch(branch);
}
pub(crate) fn unaddressed_findings(counts: Option<crate::pull_request::UnaddressedFindings>) {
record().unaddressed_findings(counts);
}
pub fn finished(account: &crate::run_ending::Account<'_>) {
record().finished(account);
}
pub(crate) fn experiment(description: &crate::evals::experiment::Description) {
record().experiment(description);
}
pub(crate) fn resumable(command: &str) {
record().resumable(command);
}
pub(crate) fn handed_back_for_pickup() {
record().handed_back_for_pickup();
}
pub(crate) fn recorded_handed_back_for_pickup() -> bool {
record().recorded_handed_back_for_pickup()
}
pub(crate) fn paused(limit: &crate::session::limits::Limit) {
record().paused(limit);
}
pub(crate) fn experiment_finished(outcome: &str, cause: Option<&str>) {
complete(outcome, cause);
}
pub fn complete(outcome: &str, cause: Option<&str>) {
let mut record = record();
if outcome == "paused" {
record.resumable_by_this_command();
}
record.outcome(outcome, cause);
}
pub fn stop_requested() -> Option<stop::Request> {
if !crate::interrupt::requested() {
return None;
}
record().stop_request()
}
pub fn dispatch_ended(ended: &crate::run::Ended) {
record().dispatch_ended(&crate::run_ending::Account::of_run(ended));
}
pub fn dispatching(
work: Work,
harness: &Choice,
security_fix_findings: &[crate::security::RecordedFinding],
) {
record().dispatching(work, harness, security_fix_findings);
}
pub fn show_held() {
record().show_held();
}
pub fn root(repo: &Repo) -> PathBuf {
record().root(repo)
}
pub(crate) fn root_if_configured(repo: &Repo) -> Option<PathBuf> {
record().root_if_configured(repo)
}
pub fn command_kind() -> CommandKind {
record().command_kind()
}
pub(crate) fn record_for_pull_request(root: &Path, url: &str) -> anyhow::Result<Option<PathBuf>> {
Ok(run_for_pull_request(root, url)?.map(|(path, _)| path))
}
pub fn session_dir(repo: &Repo) -> PathBuf {
let record = record();
record
.root(repo)
.join("commands")
.join(record.command_kind().folder())
}
pub fn stamp() -> String {
record().stamp()
}
pub fn command_log_path() -> Option<PathBuf> {
record().command_log_path()
}
pub(crate) fn captured_owner() -> Result<owner::Owner, String> {
record().captured_owner()
}
pub(crate) fn trace_context() -> Option<trace::Context> {
record().trace_context()
}
static OTLP_SUPPRESSED: AtomicUsize = AtomicUsize::new(0);
pub(crate) struct SuppressOtlp;
impl SuppressOtlp {
pub(crate) fn new() -> Self {
OTLP_SUPPRESSED.fetch_add(1, Ordering::AcqRel);
Self
}
}
impl Drop for SuppressOtlp {
fn drop(&mut self) {
OTLP_SUPPRESSED.fetch_sub(1, Ordering::AcqRel);
}
}
pub(crate) fn otlp_suppressed() -> bool {
OTLP_SUPPRESSED.load(Ordering::Acquire) > 0
}
pub(crate) fn otlp_endpoint() -> Option<String> {
if otlp_suppressed() {
None
} else {
record().otlp_endpoint()
}
}
pub fn eprint(line: &str) {
record().eprint(line);
}
pub fn print(line: &str) {
record().print(line);
}
fn root_under(logs_dir: &Path, repo: &Repo) -> PathBuf {
logs_dir.join(&repo.owner).join(&repo.name)
}
impl<'a> Work<'a> {
fn process(self) -> crate::processes::Identity {
use crate::processes::Identity;
let file = match self {
Self::Starting { .. } => return Identity::default(),
Self::Run(_) | Self::BaseFix(_) => "ticket-run",
Self::SpecRun(_) => "spec-run",
Self::ArchitectRun(_) => "architect-run",
Self::SecurityRun(_) => "security-run",
Self::PickupRun(_) => "pickup-run",
Self::ScoringRun(_) => "scoring-run",
Self::ReviewRun(_) => "review-run",
Self::FixRun(_) => "fix-run",
Self::ReleaseRun(_) => "release-run",
Self::TriageRun { .. } => "triage-run",
Self::Experiment(_) => "experiment",
Self::Process { name, .. } => return Identity::custom(name, None),
};
Identity::built_in(file, &self.kind().to_string())
}
fn initial_stage(self) -> Option<Stage> {
match self {
Work::Starting { .. } => None,
Work::Run(_) | Work::BaseFix(_) => Some(Stage::Implement),
Work::SpecRun(_) => Some(Stage::Tickets),
Work::ArchitectRun(_) => Some(Stage::ArchitectureReview),
Work::ScoringRun(_) => Some(Stage::Sampling),
Work::ReviewRun(_) => Some(Stage::CodeReview),
Work::FixRun(_) => Some(Stage::PullRequest),
Work::ReleaseRun(_) => Some(Stage::PullRequest),
Work::TriageRun { .. } => Some(Stage::Triage),
Work::PickupRun(_) | Work::SecurityRun(_) | Work::Process { .. } => None,
Work::Experiment(_) => Some(Stage::ArmBuilds),
}
}
fn repo(self) -> Repo {
match self {
Work::Starting { repo, .. } => repo.clone(),
Work::Run(issue) | Work::BaseFix(issue) | Work::SpecRun(issue) => issue.repo(),
Work::PickupRun(listed) => listed.issue.repo(),
Work::ArchitectRun(repo)
| Work::SecurityRun(repo)
| Work::ScoringRun(repo)
| Work::ReviewRun(repo)
| Work::FixRun(repo)
| Work::ReleaseRun(repo)
| Work::TriageRun { repo, .. }
| Work::Experiment(repo)
| Work::Process { repo, .. } => repo.clone(),
}
}
fn kind(self) -> Kind {
match self {
Work::Starting { command, .. } => match command {
CommandKind::Issue => Kind::Run,
CommandKind::Architect => Kind::Pass(Pass::Architect),
CommandKind::Pickup => Kind::Pass(Pass::Pickup),
CommandKind::Secure => Kind::Pass(Pass::Security),
CommandKind::Scoring => Kind::ScoringRun,
CommandKind::Review => Kind::ReviewRun,
CommandKind::Fix => Kind::FixRun,
CommandKind::Release => Kind::ReleaseRun,
CommandKind::Triage => Kind::TriageRun,
CommandKind::Experiment => Kind::Experiment,
CommandKind::Process => Kind::Process,
},
Work::Run(_) | Work::BaseFix(_) => Kind::Run,
Work::SpecRun(_) => Kind::SpecRun,
Work::PickupRun(_) => Kind::Pass(Pass::Pickup),
Work::ArchitectRun(_) => Kind::Pass(Pass::Architect),
Work::SecurityRun(_) => Kind::Pass(Pass::Security),
Work::ScoringRun(_) => Kind::ScoringRun,
Work::ReviewRun(_) => Kind::ReviewRun,
Work::FixRun(_) => Kind::FixRun,
Work::ReleaseRun(_) => Kind::ReleaseRun,
Work::TriageRun { .. } => Kind::TriageRun,
Work::Experiment(_) => Kind::Experiment,
Work::Process { .. } => Kind::Process,
}
}
pub(crate) fn issue_url(self) -> Option<&'a IssueUrl> {
match self {
Work::Starting { issue, .. } | Work::TriageRun { issue, .. } => issue,
Work::Run(issue) | Work::BaseFix(issue) | Work::SpecRun(issue) => Some(issue),
Work::PickupRun(listed) => Some(&listed.issue),
Work::ArchitectRun(_)
| Work::SecurityRun(_)
| Work::ScoringRun(_)
| Work::ReviewRun(_)
| Work::FixRun(_)
| Work::ReleaseRun(_)
| Work::Experiment(_)
| Work::Process { .. } => None,
}
}
fn issue(self) -> Option<u64> {
self.issue_url().map(|issue| issue.number)
}
}
fn record() -> MutexGuard<'static, Record<OnMachine>> {
let mut record = RECORD.lock().unwrap_or_else(PoisonError::into_inner);
record.check_suspension();
record
}
pub(crate) fn suspended_seconds() -> u64 {
record().suspended_seconds()
}