1use alloc::{
2 string::{String, ToString},
3 sync::Arc,
4};
5
6use ax_fs_ng::vfs::current_fs_context;
7use ax_runtime::hal::cpu::user::UserContext;
8
9use crate::{
10 file::{FD_TABLE, FileTable, new_file_table_scope},
11 mm::{MmHandle, load_user_app, new_user_image_builder},
12 namespace::NsProxy,
13 pseudofs::{self, dev::tty},
14 sync::{Mutex, RwLock},
15 task::{
16 PidReservation, PidReservationKind, Process, ProcessData, ProcessDataInit, ProcessImage,
17 ROOT_PID_NS, Tgid, Thread, Tid, TidNumber, UserThreadOptions, kernel_thread_builder,
18 new_user_task, prepare_user_thread, sleep, spawn_alarm_task,
19 },
20 tracepoint::tracepoint_init,
21};
22
23pub fn init(args: &[String], envs: &[String]) {
25 crate::rdrive_osal::init();
27
28 crate::stop_machine::init();
29 crate::trap::init_handlers();
30 static_keys::global_init();
31 crate::cgroup::init();
32
33 tracepoint_init().expect("Failed to initialize tracepoints");
34
35 crate::ebpf::init_ebpf();
36 crate::perf::perf_event_init();
37 crate::kmod::init_kmod();
38
39 pseudofs::mount_all().expect("Failed to mount pseudofs");
40 crate::file::epoll::start_epoll_notify_worker();
41 spawn_alarm_task();
42 crate::mm::spawn_reclaimer_task();
43 if ax_driver::cpufreq::calibrate_wanted() {
47 run_opp_calibration();
48 } else {
49 spawn_cpufreq_governor();
50 }
51 ax_driver::cpufreq::log_frequency_readout();
53 pseudofs::usbfs::start_event_pump();
54
55 ax_alloc::register_page_reclaim_fn(ax_fs_ng::vfs::page_cache_reclaim);
56
57 let loc = current_fs_context()
58 .lock()
59 .resolve(&args[0])
60 .expect("Failed to resolve executable path");
61 let path = loc
62 .absolute_path()
63 .expect("Failed to get executable absolute path");
64 let name = loc.name().into_owned();
65
66 let mut image_builder =
67 new_user_image_builder().expect("Failed to create unpublished user address space");
68 let loaded_image = load_user_app(
69 &mut image_builder,
70 loc,
71 &args[0],
72 args,
73 envs,
74 &crate::task::Cred::root(),
75 )
76 .unwrap_or_else(|error| panic!("Failed to load user app: {error}"));
77 let prepared_image = image_builder
78 .finish(loaded_image)
79 .expect("loaded init image token no longer matches its address space");
80 let (uspace, entry_vaddr, ustack_top, auxv) = prepared_image.into_parts();
81
82 let uctx = UserContext::new(entry_vaddr.into(), ustack_top, 0);
83
84 const INIT_PID: u32 = 1;
93 let reservation = PidReservation::reserve(&ROOT_PID_NS, PidReservationKind::ProcessLeader)
94 .expect("failed to reserve init PID identity");
95 let pid = reservation
96 .number_in(&ROOT_PID_NS)
97 .expect("init PID reservation has no root binding")
98 .get();
99 assert_eq!(pid, INIT_PID);
100 let identity = reservation.identity();
101 let tid_lease = identity
102 .acquire_role::<Tid>()
103 .expect("failed to acquire init TID role");
104 let tgid_lease = identity
105 .acquire_role::<Tgid>()
106 .expect("failed to acquire init TGID role");
107 let proc = Process::new_init(identity.clone()).expect("failed to prepare init process");
108 proc.add_thread(TidNumber::try_from(pid).expect("init TID must be non-zero"));
109
110 if let Err(error) = tty::bind_console_to(&proc) {
111 warn!("Failed to bind console tty: {error:?}");
112 }
113
114 let proc = ProcessData::new(
115 proc,
116 identity.clone(),
117 tgid_lease,
118 ProcessDataInit::new(
119 ProcessImage::new(
120 path.to_string(),
121 Arc::new(args.to_vec()),
122 Arc::new(envs.to_vec()),
123 auxv,
124 "/".to_string(),
125 "/".to_string(),
126 ),
127 MmHandle::from_arc(Arc::new(Mutex::new(uspace)))
128 .expect("init MM identity must be unique"),
129 Arc::default(),
130 NsProxy::new_root(),
131 None,
132 TidNumber::try_from(pid).expect("init TID must be non-zero"),
133 ),
134 );
135 crate::cgroup::attach_initial_process(&identity)
137 .expect("Failed to attach init process to cgroup root");
138
139 let mut scope = scope_local::Scope::new();
140 let mut fd_table = FileTable::new();
141 crate::file::add_stdio(&mut fd_table).expect("Failed to add stdio");
142 *FD_TABLE.scope_mut(&mut scope) = new_file_table_scope(Arc::new(RwLock::new(fd_table)));
143
144 let thr = Thread::new(
145 identity.clone(),
146 tid_lease,
147 proc,
148 None,
149 starry_signal::SignalSet::default(),
150 scope,
151 )
152 .expect("failed to prepare init thread state");
153 let prepared_task = prepare_user_thread(
154 new_user_task(
155 uctx,
156 0,
157 TidNumber::try_from(pid).expect("init TID must be non-zero"),
158 ),
159 thr,
160 UserThreadOptions::new(&name).expect("failed to prepare init thread name"),
161 )
162 .expect("failed to prepare init task");
163 let staged_task = prepared_task.stage().expect("failed to stage init task");
164 let published_identity = reservation
165 .publish()
166 .expect("failed to publish init PID identity");
167 debug_assert!(Arc::ptr_eq(&published_identity, &identity));
168 staged_task.with_task(|task| task.as_thread().attach_pid_task(task));
169 tty::arm_console_irq();
170 let task = staged_task.activate();
171
172 let exit_code = task.join();
174 info!("Init process exited with code: {exit_code:?}");
175
176 let fs_context = current_fs_context();
177 let cx = fs_context.lock();
178 if let Err(err) = cx.root_dir().unmount_all() {
184 warn!("shutdown: unmount_all failed (best-effort): {err:?}");
185 }
186 cx.root_dir()
187 .filesystem()
188 .flush()
189 .expect("Failed to flush rootfs");
190}
191
192fn run_opp_calibration() {
199 info!("cpufreq: running OPP calibration sweep (governor disabled this boot)");
200 for &(cluster_idx, cpu) in &[(0usize, 0usize), (1, 4), (2, 6)] {
201 let mut affinity = ax_runtime::task::sched::CpuSet::empty(ax_runtime::hal::cpu_num());
202 let cpu_id =
203 u32::try_from(cpu).unwrap_or_else(|_| panic!("cpufreq CPU id {cpu} is out of range"));
204 assert!(
205 affinity.insert(ax_runtime::task::sched::CpuId::new(cpu_id)),
206 "cpufreq calibration CPU {cpu} is outside the runtime topology"
207 );
208 let task = kernel_thread_builder(String::from("cpufreq-cal"))
209 .affinity(affinity)
210 .spawn(move || ax_driver::cpufreq::calibrate_cluster(cluster_idx, cpu))
211 .expect("failed to spawn kernel thread");
212 let _exit_code = task.join().expect("failed to join kernel thread");
213 }
214 info!("cpufreq: OPP calibration sweep complete");
215}
216
217fn spawn_cpufreq_governor() {
230 if !ax_driver::cpufreq::governor_wanted() {
231 return;
232 }
233 info!("Initialize cpufreq ondemand governor...");
234 let _ = kernel_thread_builder(String::from("cpufreq-gov"))
235 .spawn(cpufreq_governor_loop)
236 .expect("failed to spawn kernel thread");
237}
238
239fn cpufreq_governor_loop() {
245 let period = core::time::Duration::from_millis(ax_driver::cpufreq::governor_period_ms());
246 loop {
247 sleep(period);
248 let mut busy = [0u64; 8];
251 for (cpu, slot) in busy.iter_mut().enumerate() {
252 *slot = ax_runtime::task::sched::cpu_busy_runtime_ns(
253 ax_runtime::task::sched::CpuId::new(cpu as u32),
254 )
255 .unwrap_or(0);
256 }
257 ax_driver::cpufreq::governor_poll(&busy);
258 }
259}