Skip to main content

starry_kernel/
entry.rs

1use alloc::{
2    string::{String, ToString},
3    sync::Arc,
4};
5
6use ax_fs_ng::vfs::current_fs_context;
7use ax_runtime::hal::cpu::user::UserContext;
8
9use crate::{
10    file::{FD_TABLE, FileTable, new_file_table_scope},
11    mm::{MmHandle, load_user_app, new_user_image_builder},
12    namespace::NsProxy,
13    pseudofs::{self, dev::tty},
14    sync::{Mutex, RwLock},
15    task::{
16        PidReservation, PidReservationKind, Process, ProcessData, ProcessDataInit, ProcessImage,
17        ROOT_PID_NS, Tgid, Thread, Tid, TidNumber, UserThreadOptions, kernel_thread_builder,
18        new_user_task, prepare_user_thread, sleep, spawn_alarm_task,
19    },
20    tracepoint::tracepoint_init,
21};
22
23/// Initialize and run initproc.
24pub fn init(args: &[String], envs: &[String]) {
25    // Install task-context diagnostics and contention backoff before userspace.
26    crate::rdrive_osal::init();
27
28    crate::stop_machine::init();
29    crate::trap::init_handlers();
30    static_keys::global_init();
31    crate::cgroup::init();
32
33    tracepoint_init().expect("Failed to initialize tracepoints");
34
35    crate::ebpf::init_ebpf();
36    crate::perf::perf_event_init();
37    crate::kmod::init_kmod();
38
39    pseudofs::mount_all().expect("Failed to mount pseudofs");
40    crate::file::epoll::start_epoll_notify_worker();
41    spawn_alarm_task();
42    crate::mm::spawn_reclaimer_task();
43    // DVFS: a one-shot OPP-calibration boot runs the sweep and skips the governor;
44    // otherwise start the ondemand governor. Both run here (early init, before the
45    // console tty handoff) so their kernel logs reach the serial console.
46    if ax_driver::cpufreq::calibrate_wanted() {
47        run_opp_calibration();
48    } else {
49        spawn_cpufreq_governor();
50    }
51    // Read-only cluster frequency snapshot for CPU-bound workload triage.
52    ax_driver::cpufreq::log_frequency_readout();
53    pseudofs::usbfs::start_event_pump();
54
55    ax_alloc::register_page_reclaim_fn(ax_fs_ng::vfs::page_cache_reclaim);
56
57    let loc = current_fs_context()
58        .lock()
59        .resolve(&args[0])
60        .expect("Failed to resolve executable path");
61    let path = loc
62        .absolute_path()
63        .expect("Failed to get executable absolute path");
64    let name = loc.name().into_owned();
65
66    let mut image_builder =
67        new_user_image_builder().expect("Failed to create unpublished user address space");
68    let loaded_image = load_user_app(
69        &mut image_builder,
70        loc,
71        &args[0],
72        args,
73        envs,
74        &crate::task::Cred::root(),
75    )
76    .unwrap_or_else(|error| panic!("Failed to load user app: {error}"));
77    let prepared_image = image_builder
78        .finish(loaded_image)
79        .expect("loaded init image token no longer matches its address space");
80    let (uspace, entry_vaddr, ustack_top, auxv) = prepared_image.into_parts();
81
82    let uctx = UserContext::new(entry_vaddr.into(), ustack_top, 0);
83
84    // PID 1 must really be 1: the init process is the root of the process
85    // hierarchy and userspace (e.g. systemd's `getpid() == 1` system-manager
86    // check) relies on it. The scheduler task id is an internal counter that is
87    // already past 1 by the time we spawn the user init (kernel helper tasks
88    // took the low ids), so we pin the user-visible pid/tid to 1 and leave the
89    // scheduler id untouched. `Thread::tid` is already decoupled from the
90    // scheduler id (see its field doc), so this only requires the table keys to
91    // follow the thread tid rather than `task.id()`.
92    const INIT_PID: u32 = 1;
93    let reservation = PidReservation::reserve(&ROOT_PID_NS, PidReservationKind::ProcessLeader)
94        .expect("failed to reserve init PID identity");
95    let pid = reservation
96        .number_in(&ROOT_PID_NS)
97        .expect("init PID reservation has no root binding")
98        .get();
99    assert_eq!(pid, INIT_PID);
100    let identity = reservation.identity();
101    let tid_lease = identity
102        .acquire_role::<Tid>()
103        .expect("failed to acquire init TID role");
104    let tgid_lease = identity
105        .acquire_role::<Tgid>()
106        .expect("failed to acquire init TGID role");
107    let proc = Process::new_init(identity.clone()).expect("failed to prepare init process");
108    proc.add_thread(TidNumber::try_from(pid).expect("init TID must be non-zero"));
109
110    if let Err(error) = tty::bind_console_to(&proc) {
111        warn!("Failed to bind console tty: {error:?}");
112    }
113
114    let proc = ProcessData::new(
115        proc,
116        identity.clone(),
117        tgid_lease,
118        ProcessDataInit::new(
119            ProcessImage::new(
120                path.to_string(),
121                Arc::new(args.to_vec()),
122                Arc::new(envs.to_vec()),
123                auxv,
124                "/".to_string(),
125                "/".to_string(),
126            ),
127            MmHandle::from_arc(Arc::new(Mutex::new(uspace)))
128                .expect("init MM identity must be unique"),
129            Arc::default(),
130            NsProxy::new_root(),
131            None,
132            TidNumber::try_from(pid).expect("init TID must be non-zero"),
133        ),
134    );
135    // SAFE-EXPECT: failing to attach init would violate the kernel's process accounting invariant.
136    crate::cgroup::attach_initial_process(&identity)
137        .expect("Failed to attach init process to cgroup root");
138
139    let mut scope = scope_local::Scope::new();
140    let mut fd_table = FileTable::new();
141    crate::file::add_stdio(&mut fd_table).expect("Failed to add stdio");
142    *FD_TABLE.scope_mut(&mut scope) = new_file_table_scope(Arc::new(RwLock::new(fd_table)));
143
144    let thr = Thread::new(
145        identity.clone(),
146        tid_lease,
147        proc,
148        None,
149        starry_signal::SignalSet::default(),
150        scope,
151    )
152    .expect("failed to prepare init thread state");
153    let prepared_task = prepare_user_thread(
154        new_user_task(
155            uctx,
156            0,
157            TidNumber::try_from(pid).expect("init TID must be non-zero"),
158        ),
159        thr,
160        UserThreadOptions::new(&name).expect("failed to prepare init thread name"),
161    )
162    .expect("failed to prepare init task");
163    let staged_task = prepared_task.stage().expect("failed to stage init task");
164    let published_identity = reservation
165        .publish()
166        .expect("failed to publish init PID identity");
167    debug_assert!(Arc::ptr_eq(&published_identity, &identity));
168    staged_task.with_task(|task| task.as_thread().attach_pid_task(task));
169    tty::arm_console_irq();
170    let task = staged_task.activate();
171
172    // TODO: wait for all processes to finish
173    let exit_code = task.join();
174    info!("Init process exited with code: {exit_code:?}");
175
176    let fs_context = current_fs_context();
177    let cx = fs_context.lock();
178    // Best-effort teardown, matching Linux's shutdown path. A process that exited while
179    // holding a mount namespace (bind mounts, pivot_root) can leave the mount tree in a
180    // state `unmount_all` rejects; at shutdown that must be logged, not turned into a
181    // kernel panic that fails an otherwise clean run. The rootfs flush below is what
182    // matters for on-disk integrity.
183    if let Err(err) = cx.root_dir().unmount_all() {
184        warn!("shutdown: unmount_all failed (best-effort): {err:?}");
185    }
186    cx.root_dir()
187        .filesystem()
188        .flush()
189        .expect("Failed to flush rootfs");
190}
191
192/// Run the one-shot DVFS OPP calibration sweep (gated by the driver's `CALIBRATE`
193/// const). Each cluster's (voltage x ring) sweep must execute ON a core of that
194/// cluster to read that core's own PMU cycle counter, so we pin a task per cluster
195/// (cpu0=A55, cpu4=A76 big0, cpu6=A76 big1) before run-queue publication and run
196/// them sequentially (the two A76 rails share one I2C bus). Synchronous: it
197/// blocks init briefly so the `CAL` log lines land before the console tty handoff.
198fn run_opp_calibration() {
199    info!("cpufreq: running OPP calibration sweep (governor disabled this boot)");
200    for &(cluster_idx, cpu) in &[(0usize, 0usize), (1, 4), (2, 6)] {
201        let mut affinity = ax_runtime::task::sched::CpuSet::empty(ax_runtime::hal::cpu_num());
202        let cpu_id =
203            u32::try_from(cpu).unwrap_or_else(|_| panic!("cpufreq CPU id {cpu} is out of range"));
204        assert!(
205            affinity.insert(ax_runtime::task::sched::CpuId::new(cpu_id)),
206            "cpufreq calibration CPU {cpu} is outside the runtime topology"
207        );
208        let task = kernel_thread_builder(String::from("cpufreq-cal"))
209            .affinity(affinity)
210            .spawn(move || ax_driver::cpufreq::calibrate_cluster(cluster_idx, cpu))
211            .expect("failed to spawn kernel thread");
212        let _exit_code = task.join().expect("failed to join kernel thread");
213    }
214    info!("cpufreq: OPP calibration sweep complete");
215}
216
217/// Start the CPU DVFS ondemand governor.
218///
219/// The frequency/voltage policy and the SCMI+PMIC apply live in the cpufreq
220/// driver (`ax_driver::cpufreq`); this kernel task is only the driver's periodic
221/// *loop*. The loop must live here, not in the driver, because ax-driver sits
222/// below ax-task/ax-hal in the dependency graph (they pull ax-driver back in via
223/// axplat-dyn), so spawning a task inside the driver would be a cyclic dep. Each
224/// period we snapshot the scheduler's cumulative per-CPU non-idle runtime and
225/// hand it to `governor_poll`, which decides and applies any OPP change.
226///
227/// No-op unless the driver armed the governor (feature on and both CPU-rail PMIC
228/// buses up); otherwise every cluster stays on its boot OPP.
229fn spawn_cpufreq_governor() {
230    if !ax_driver::cpufreq::governor_wanted() {
231        return;
232    }
233    info!("Initialize cpufreq ondemand governor...");
234    let _ = kernel_thread_builder(String::from("cpufreq-gov"))
235        .spawn(cpufreq_governor_loop)
236        .expect("failed to spawn kernel thread");
237}
238
239/// Periodic body of the DVFS governor task: sleep, sample every CPU's cumulative
240/// non-idle runtime, and let the driver scale each cluster to match load. The
241/// slow work (SCMI SMC + PMIC I2C/SPI voltage ramp) happens inside
242/// `governor_poll`, which is why this runs in a sleepable task rather than the
243/// scheduler tick.
244fn cpufreq_governor_loop() {
245    let period = core::time::Duration::from_millis(ax_driver::cpufreq::governor_period_ms());
246    loop {
247        sleep(period);
248        // RK3588 has 8 CPUs; an offline or topology-excluded core contributes
249        // zero runtime and therefore reads as idle.
250        let mut busy = [0u64; 8];
251        for (cpu, slot) in busy.iter_mut().enumerate() {
252            *slot = ax_runtime::task::sched::cpu_busy_runtime_ns(
253                ax_runtime::task::sched::CpuId::new(cpu as u32),
254            )
255            .unwrap_or(0);
256        }
257        ax_driver::cpufreq::governor_poll(&busy);
258    }
259}