1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
/**
* Cem Runtime - Green Thread Scheduler
*
* This header defines the scheduler for lightweight concurrent execution
* (strands) in the Cem runtime. The scheduler provides cooperative multitasking
* with explicit yield points at I/O operations.
*
* Architecture:
* - Each strand has its own isolated stack (StackCell linked list)
* - Strands yield only at I/O operations (cooperative, not preemptive)
* - Simple FIFO ready queue for runnable strands
* - Fast context switching via custom assembly
* (cem_makecontext/cem_swapcontext)
*
* IMPLEMENTATION PHASES:
* =======================
* Phase 1: Infrastructure ✅ (Completed)
* - Scheduler initialization/shutdown
* - Ready queue (FIFO operations)
* - Strand state management structures
*
* Phase 2a: ucontext Context Switching ✅ (Completed)
* - Replaced jmp_buf with ucontext_t
* - Implemented strand_spawn() with makecontext()
* - Implemented strand_yield() with swapcontext()
* - Implemented scheduler_run() event loop
* - 64KB stacks per strand
* - Achieved: ~10,000 concurrent strands
*
* Phase 2b: Custom Assembly Context Switching (CURRENT)
* - Custom ARM64 and x86-64 implementations
* - Replace deprecated ucontext with cem_swapcontext/cem_makecontext
* - 10-20x faster context switches (~10-20ns vs ~500ns)
* - Add cleanup handler infrastructure to fix memory leaks
* - 64KB stacks per strand (will reduce to 8KB in Phase 2c)
* - Target: ~10,000 concurrent strands (same as 2a, but faster and leak-free)
*
* Phase 3: Dynamic Stack Growth (FUTURE)
* - Segmented stacks with growth on overflow
* - 2-4KB initial stacks
* - Target: 500,000+ concurrent strands
*
* See docs/SCHEDULER_IMPLEMENTATION.md for detailed roadmap
*/
// Cleanup Handlers
/**
* Cleanup handler function type
*
* Called when a strand terminates (either normally or abnormally).
* Used to free resources allocated during strand execution.
*
* @param arg - Arbitrary data passed to the cleanup function
*/
typedef void ;
/**
* Cleanup handler node
*
* Cleanup handlers are stored in a LIFO linked list (stack order).
* When a strand terminates, handlers are called in reverse order of
* registration (most recently registered first).
*/
typedef struct CleanupHandler CleanupHandler;
// Strand State
/**
* Strand execution states
*/
typedef enum StrandState;
/**
* Strand - A lightweight thread of execution
*
* Each strand maintains its own:
* - Execution context (cem_context_t for context switching)
* - Dynamic C stack (mmap'd with guard page, starts at 4KB, grows to 1MB max)
* - Cem stack (pointer to StackCell linked list)
* - Current state (ready/running/yielded/completed)
*
* Memory layout considerations:
* Phase 3 (Dynamic Stacks):
* - cem_context_t: 168 bytes (ARM64) or 64 bytes (x86-64)
* - C stack: 4KB initial, grows dynamically up to 1MB
* - StackCell pointer: 8 bytes
* - StackMetadata: ~64 bytes
* - Total per strand: ~4.3KB initially, grows as needed
* - With 10,000 strands: ~43MB (vs 640MB in Phase 2b!)
*/
typedef struct Strand Strand;
// Scheduler State
/**
* Global scheduler state
*
* The scheduler maintains:
* - Ready queue: FIFO queue of runnable strands
* - Blocked list: Strands waiting for I/O events
* - Current strand: The currently executing strand
* - Next strand ID: Counter for generating unique IDs
* - Scheduler context: The main scheduler's execution context
* - I/O multiplexing: kqueue (BSD/macOS) or epoll (Linux) for async I/O
*
* Note: This is a single-threaded scheduler. All fields are accessed
* from a single thread, so no locks are needed.
*/
typedef struct Scheduler Scheduler;
// Scheduler Operations
/**
* Initialize the global scheduler
* Must be called before any other scheduler operations
*/
void ;
/**
* Shutdown the scheduler and free all resources
*/
void ;
/**
* Create a new strand to execute the given function
*
* The function receives the strand's initial stack and returns
* the final stack state. The strand will be added to the ready queue.
*
* @param entry_func - Function to execute in the strand
* @param initial_stack - Initial stack state for the strand
* @return Strand ID
*/
uint64_t ;
/**
* Register a cleanup handler for the current strand
*
* The cleanup handler will be called when the strand terminates (either
* normally or abnormally). Handlers are called in LIFO order (most recently
* registered first).
*
* This is used to ensure resources are freed even if a strand is terminated
* while blocked on I/O or otherwise interrupted.
*
* IMPORTANT: This must only be called from within a strand.
*
* @param func - Cleanup function to call
* @param arg - Argument to pass to cleanup function
*/
void ;
/**
* Remove the most recently registered cleanup handler
*
* This is called when the resource has been successfully released and
* cleanup is no longer needed.
*
* IMPORTANT: This must only be called from within a strand.
*/
void ;
/**
* Update the argument of the most recently registered cleanup handler
*
* This is useful when a resource pointer changes (e.g., after realloc)
* but the cleanup function remains the same. This is safer than pop+push
* because it ensures the cleanup handler is never unregistered during the
* update.
*
* IMPORTANT: This must only be called from within a strand.
*
* @param new_arg - New argument to pass to the cleanup function
*/
void ;
/**
* Yield execution from the current strand back to the scheduler
*
* This saves the current execution context and transfers control
* to the scheduler, which will schedule the next ready strand.
* The current strand will be re-queued as READY.
*
* IMPORTANT: This must only be called from within a strand, not from
* the main scheduler loop.
*/
void ;
/**
* Block current strand on I/O read operation
*
* This saves the current execution context, registers the file descriptor
* for read events with kqueue, and transfers control back to the scheduler.
* When the FD becomes readable, the strand will be moved to the ready queue.
*
* @param fd - File descriptor to wait for (must be in non-blocking mode)
*/
void ;
/**
* Block current strand on I/O write operation
*
* This saves the current execution context, registers the file descriptor
* for write events with kqueue, and transfers control back to the scheduler.
* When the FD becomes writable, the strand will be moved to the ready queue.
*
* @param fd - File descriptor to wait for (must be in non-blocking mode)
*/
void ;
/**
* Run the scheduler until all strands complete
*
* This is the main scheduler loop:
* 1. Pop a strand from the ready queue
* 2. Run it until it yields or completes
* 3. If it yielded, re-queue it
* 4. Repeat until no strands remain
*
* @return Final stack state (from main strand, if any)
*/
StackCell *;
// Testing & Debug Operations
/**
* Synthetic yield for testing (Phase 1)
*
* This is a runtime function callable from Cem code to test
* the scheduler without implementing full I/O operations.
*
* Usage in Cem: `test_yield`
* Stack effect: ( -- )
*/
StackCell *;
/**
* Print scheduler state (for debugging)
*/
void ;
// Internal Functions (exposed for testing)
/**
* Enqueue a strand to the ready queue
*/
void ;
/**
* Dequeue a strand from the ready queue
* Returns NULL if queue is empty
*/
Strand *;
/**
* Check if ready queue is empty
*/
bool ;
// CEM_RUNTIME_SCHEDULER_H