1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
use core::sync::atomic::{AtomicUsize, Ordering, fence};
use critical_section::RawRestoreState;
use super::pac::{MSTATUS_MIE, hart_id};
pub struct MPFSCriticalSection;
const LOCK_UNOWNED: usize = 0;
// Stores which hart (core) owns the lock: 0 = unowned, 1-4 = hart ID
static LOCK_OWNER: AtomicUsize = AtomicUsize::new(LOCK_UNOWNED);
#[repr(u8)]
enum RestoreState {
ReleaseLockLeaveInterruptsDisabled,
ReleaseLockRestoreInterrupts,
KeepLock,
}
unsafe impl critical_section::Impl for MPFSCriticalSection {
unsafe fn acquire() -> RawRestoreState {
unsafe {
let hart_id: usize = hart_id();
// Check if we already own the lock
if LOCK_OWNER.load(Ordering::Acquire) == hart_id {
// We already own the lock, so no need to lock again
// When we release this particular "lock", we won't do anything
// The initial lock-taker is responsible for actually releasing the lock
return RestoreState::KeepLock as RawRestoreState;
}
// First check if interrupts are enabled
let mut status: usize;
core::arch::asm!(
"csrr {}, mstatus",
out(reg) status
);
// When entering from an interrupt handler, interrupts are already disabled
// So we only want to restore them if they were enabled when we entered
let was_enabled = (status & MSTATUS_MIE as usize) != 0;
loop {
// Disable interrupts
core::arch::asm!(
"csrci mstatus, {}",
const MSTATUS_MIE,
options(nomem, nostack)
);
match LOCK_OWNER.compare_exchange(
LOCK_UNOWNED,
hart_id,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => break,
Err(_) => {
// Re-enable interrupts if they were enabled before
if was_enabled {
core::arch::asm!(
"csrsi mstatus, {}",
const MSTATUS_MIE,
options(nomem, nostack)
);
}
core::hint::spin_loop();
}
}
}
#[cfg(feature = "debug-logs")]
{
crate::print_unguarded!("hart {} acquired", hart_id);
if was_enabled {
crate::println_unguarded!(" (restore interrupts: true)\n");
} else {
crate::println_unguarded!(" (restore: false)\n");
}
}
if was_enabled {
// We need to restore interrupts when we release the lock
RestoreState::ReleaseLockRestoreInterrupts as RawRestoreState
} else {
// Interrupts were disabled when we entered, so we don't restore them when releasing
RestoreState::ReleaseLockLeaveInterruptsDisabled as RawRestoreState
}
}
}
unsafe fn release(restore_state: RawRestoreState) {
unsafe {
let state = restore_state as u8;
if state == RestoreState::KeepLock as u8 {
return;
}
#[cfg(feature = "debug-logs")]
{
crate::println_unguarded!("hart {} releasing\n", hart_id());
}
// Ensure all memory operations in the critical section are complete
// before releasing the lock
fence(Ordering::SeqCst);
// Release the lock
LOCK_OWNER.store(LOCK_UNOWNED, Ordering::Release);
if state == RestoreState::ReleaseLockRestoreInterrupts as u8 {
// Ensure lock release is visible before re-enabling interrupts
fence(Ordering::SeqCst);
// Re-enable interrupts
core::arch::asm!("csrsi mstatus, {}", const MSTATUS_MIE, options(nomem, nostack));
}
}
}
}