nu-protocol 0.117.0

Nushell's internal protocols, including its abstract syntax tree
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
use std::{
    collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
    sync::{
        Arc, Mutex,
        mpsc::{Receiver, RecvTimeoutError, Sender, TryRecvError},
    },
};

#[cfg(not(target_family = "wasm"))]
use nu_utils::time::Instant;
#[cfg(not(target_family = "wasm"))]
use std::time::Duration;

use nu_system::{ForegroundChild, UnfreezeHandle, kill_by_pid};

use crate::{PipelineData, Signals, shell_error};

use crate::JobId;

#[derive(Debug)]
pub struct Jobs {
    next_job_id: usize,

    // this is the ID of the most recently added frozen job in the jobs table.
    // the methods of this struct must ensure the invariant of this always
    // being None or pointing to a valid job in the table
    last_frozen_job_id: Option<JobId>,
    jobs: HashMap<JobId, Job>,
}

impl Default for Jobs {
    fn default() -> Self {
        Self {
            next_job_id: 1,
            last_frozen_job_id: None,
            jobs: HashMap::default(),
        }
    }
}

impl Jobs {
    pub fn iter(&self) -> impl Iterator<Item = (JobId, &Job)> {
        self.jobs.iter().map(|(k, v)| (*k, v))
    }

    /// The id the next job added to the table will get. Ids only grow, so a job with a lower id
    /// was started before this call.
    pub fn next_id(&self) -> JobId {
        JobId::new(self.next_job_id)
    }

    /// Whether there are no tracked jobs (running or frozen).
    pub fn is_empty(&self) -> bool {
        self.jobs.is_empty()
    }

    pub fn lookup(&self, id: JobId) -> Option<&Job> {
        self.jobs.get(&id)
    }

    pub fn lookup_mut(&mut self, id: JobId) -> Option<&mut Job> {
        self.jobs.get_mut(&id)
    }

    pub fn remove_job(&mut self, id: JobId) -> Option<Job> {
        if self.last_frozen_job_id.is_some_and(|last| id == last) {
            self.last_frozen_job_id = None;
        }

        self.jobs.remove(&id)
    }

    fn assign_last_frozen_id_if_frozen(&mut self, id: JobId, job: &Job) {
        if let Job::Frozen(_) = job {
            self.last_frozen_job_id = Some(id);
        }
    }

    pub fn add_job(&mut self, job: Job) -> JobId {
        let this_id = self.next_id();

        self.assign_last_frozen_id_if_frozen(this_id, &job);

        self.jobs.insert(this_id, job);
        self.next_job_id += 1;

        this_id
    }

    pub fn most_recent_frozen_job_id(&mut self) -> Option<JobId> {
        self.last_frozen_job_id
    }

    // this is useful when you want to remove a job from the list and add it back later
    pub fn add_job_with_id(&mut self, id: JobId, job: Job) -> Result<(), &'static str> {
        self.assign_last_frozen_id_if_frozen(id, &job);

        if let Entry::Vacant(e) = self.jobs.entry(id) {
            e.insert(job);
            Ok(())
        } else {
            Err("job already exists")
        }
    }

    /// This function tries to forcefully kill a job from this job table,
    /// removes it from the job table. It always succeeds in removing the job
    /// from the table, but may fail in killing the job's active processes.
    pub fn kill_and_remove(&mut self, id: JobId) -> shell_error::io::Result<()> {
        if let Some(job) = self.jobs.get(&id) {
            let err = job.kill();

            self.remove_job(id);

            err?
        }

        Ok(())
    }

    /// This function tries to forcefully kill all the background jobs and
    /// removes all of them from the job table.
    ///
    /// It returns an error if any of the job killing attempts fails, but always
    /// succeeds in removing the jobs from the table.
    pub fn kill_all(&mut self) -> shell_error::io::Result<()> {
        self.last_frozen_job_id = None;

        let first_err = self
            .iter()
            .map(|(_, job)| job.kill().err())
            .fold(None, |acc, x| acc.or(x));

        self.jobs.clear();

        if let Some(err) = first_err {
            Err(err)
        } else {
            Ok(())
        }
    }
}

#[derive(Debug)]
pub enum Job {
    Thread(ThreadJob),
    Frozen(FrozenJob),
}

// A thread job represents a job that is currently executing as a background thread in nushell.
// This is an Arc-y type, cloning it does not uniquely clone the information of this particular
// job.

// Although rust's documentation does not document the acquire-release semantics of Mutex, this
// is a direct undocumentented requirement of its soundness, and is thus assumed by this
// implementaation.
// see issue https://github.com/rust-lang/rust/issues/126239.
#[derive(Clone, Debug)]
pub struct ThreadJob {
    signals: Signals,
    pids: Arc<Mutex<HashSet<u32>>>,
    description: Option<String>,
    auto_kill: bool,
    pub sender: Sender<Mail>,
}

impl ThreadJob {
    pub fn new(
        signals: Signals,
        description: Option<String>,
        auto_kill: bool,
        sender: Sender<Mail>,
    ) -> Self {
        ThreadJob {
            signals,
            pids: Arc::new(Mutex::new(HashSet::default())),
            sender,
            description,
            auto_kill,
        }
    }

    /// Spawn and register an external process while holding the PID lock.
    ///
    /// Shutdown must not finish between spawning the process and registering its PID: the
    /// shell could exit before the job thread gets a chance to kill the unregistered child.
    /// Returns `None` without spawning if the job has already been interrupted.
    pub fn try_spawn(
        &self,
        spawn: impl FnOnce() -> std::io::Result<ForegroundChild>,
    ) -> std::io::Result<Option<ForegroundChild>> {
        let mut pids = self.pids.lock().expect("PIDs lock was poisoned");
        if self.signals.interrupted() {
            return Ok(None);
        }

        let child = spawn()?;
        pids.insert(child.pid());
        Ok(Some(child))
    }

    /// Tries to add the provided pid to the active pid set of the current job.
    ///
    /// Returns true if the pid was added successfully, or false if the
    /// current job is interrupted.
    pub fn try_add_pid(&self, pid: u32) -> bool {
        let mut pids = self.pids.lock().expect("PIDs lock was poisoned");

        // note: this signals check must occur after the pids lock has been locked.
        if self.signals.interrupted() {
            false
        } else {
            pids.insert(pid);
            true
        }
    }

    pub fn collect_pids(&self) -> Vec<u32> {
        let lock = self.pids.lock().expect("PID lock was poisoned");

        lock.iter().copied().collect()
    }

    pub fn kill(&self) -> shell_error::io::Result<()> {
        // it's okay to make this interrupt outside of the mutex, since it has acquire-release
        // semantics.

        self.signals.trigger();

        let mut pids = self.pids.lock().expect("PIDs lock was poisoned");

        for pid in pids.iter() {
            kill_by_pid((*pid).into())?;
        }

        pids.clear();

        Ok(())
    }

    pub fn remove_pid(&self, pid: u32) {
        let mut pids = self.pids.lock().expect("PID lock was poisoned");

        pids.remove(&pid);
    }
}

impl Job {
    pub fn kill(&self) -> shell_error::io::Result<()> {
        match self {
            Job::Thread(thread_job) => thread_job.kill(),
            Job::Frozen(frozen_job) => frozen_job.kill(),
        }
    }

    pub fn description(&self) -> Option<&String> {
        match self {
            Job::Thread(thread_job) => thread_job.description.as_ref(),
            Job::Frozen(frozen_job) => frozen_job.description.as_ref(),
        }
    }

    pub fn is_auto_kill(&self) -> bool {
        match self {
            Job::Thread(job) => job.auto_kill,
            Job::Frozen(_) => false,
        }
    }

    pub fn assign_description(&mut self, description: Option<String>) {
        match self {
            Job::Thread(thread_job) => thread_job.description = description,
            Job::Frozen(frozen_job) => frozen_job.description = description,
        }
    }
}

#[derive(Debug)]
pub struct FrozenJob {
    pub unfreeze: UnfreezeHandle,
    pub description: Option<String>,
}

impl FrozenJob {
    pub fn kill(&self) -> shell_error::io::Result<()> {
        #[cfg(unix)]
        {
            Ok(kill_by_pid(self.unfreeze.pid() as i64)?)
        }

        // it doesn't happen outside unix.
        #[cfg(not(unix))]
        {
            Ok(())
        }
    }
}

/// Stores the information about the background job currently being executed by this thread, if any
#[derive(Clone, Debug)]
pub struct CurrentJob {
    pub id: JobId,

    /// The job the main thread spawned that this one descends from: `id` itself when the main
    /// thread spawned this job, the spawning job's `root` otherwise, and `JobId::ZERO` on the
    /// main thread.
    pub root: JobId,

    // The background thread job associated with this thread.
    // If None, it indicates this thread is currently the main job
    pub background_thread_job: Option<ThreadJob>,

    // note: although the mailbox is Mutex'd, it is only ever accessed
    // by the current job's threads
    pub mailbox: Arc<Mutex<Mailbox>>,
}

// The storage for unread messages
//
// Messages are initially sent over a mpsc channel,
// and may then be stored in a IgnoredMail struct when
// filtered out by a message tag.
#[derive(Debug)]
pub struct Mailbox {
    receiver: Receiver<Mail>,
    ignored_mail: IgnoredMail,
}

impl Mailbox {
    pub fn new(receiver: Receiver<Mail>) -> Self {
        Mailbox {
            receiver,
            ignored_mail: IgnoredMail::default(),
        }
    }

    #[cfg(not(target_family = "wasm"))]
    pub fn recv_timeout(
        &mut self,
        filter_tag: Option<FilterTag>,
        timeout: Duration,
    ) -> Result<PipelineData, RecvTimeoutError> {
        if let Some(value) = self.ignored_mail.pop(filter_tag) {
            Ok(value)
        } else {
            let mut waited_so_far = Duration::ZERO;
            let mut before = Instant::now();

            while waited_so_far < timeout {
                let (tag, value) = self
                    .receiver
                    .recv_timeout(timeout.checked_sub(waited_so_far).unwrap_or(Duration::ZERO))?;

                if filter_tag.is_none() || filter_tag == tag {
                    return Ok(value);
                } else {
                    self.ignored_mail.add((tag, value));
                    let now = Instant::now();
                    waited_so_far += now - before;
                    before = now;
                }
            }

            Err(RecvTimeoutError::Timeout)
        }
    }

    #[cfg(not(target_family = "wasm"))]
    pub fn try_recv(
        &mut self,
        filter_tag: Option<FilterTag>,
    ) -> Result<PipelineData, TryRecvError> {
        if let Some(value) = self.ignored_mail.pop(filter_tag) {
            Ok(value)
        } else {
            loop {
                let (tag, value) = self.receiver.try_recv()?;

                if filter_tag.is_none() || filter_tag == tag {
                    return Ok(value);
                } else {
                    self.ignored_mail.add((tag, value));
                }
            }
        }
    }

    pub fn clear(&mut self) {
        self.ignored_mail.clear();

        while self.receiver.try_recv().is_ok() {}
    }
}

// A data structure used to store messages which were received, but currently ignored by a tag filter
// messages are added and popped in a first-in-first-out matter.
#[derive(Default, Debug)]
struct IgnoredMail {
    next_id: usize,
    messages: BTreeMap<usize, Mail>,
    by_tag: HashMap<FilterTag, BTreeSet<usize>>,
}

pub type FilterTag = u64;
pub type Mail = (Option<FilterTag>, PipelineData);

impl IgnoredMail {
    pub fn add(&mut self, (tag, value): Mail) {
        let id = self.next_id;
        self.next_id += 1;

        self.messages.insert(id, (tag, value));

        if let Some(tag) = tag {
            self.by_tag.entry(tag).or_default().insert(id);
        }
    }

    pub fn pop(&mut self, tag: Option<FilterTag>) -> Option<PipelineData> {
        if let Some(tag) = tag {
            self.pop_oldest_with_tag(tag)
        } else {
            self.pop_oldest()
        }
    }

    pub fn clear(&mut self) {
        self.messages.clear();
        self.by_tag.clear();
    }

    fn pop_oldest(&mut self) -> Option<PipelineData> {
        let (id, (tag, value)) = self.messages.pop_first()?;

        if let Some(tag) = tag
            && let Entry::Occupied(mut occupied_entry) = self.by_tag.entry(tag)
        {
            occupied_entry.get_mut().remove(&id);
            if occupied_entry.get().is_empty() {
                occupied_entry.remove();
            }
        }

        Some(value)
    }

    fn pop_oldest_with_tag(&mut self, tag: FilterTag) -> Option<PipelineData> {
        let ids = self.by_tag.get_mut(&tag)?;

        let id = ids.pop_first()?;

        if ids.is_empty() {
            self.by_tag.remove(&tag);
        }

        Some(self.messages.remove(&id)?.1)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::{atomic::AtomicBool, mpsc};

    #[test]
    fn interrupted_job_does_not_spawn() {
        let (sender, _) = mpsc::channel();
        let signals = Signals::new(Arc::new(AtomicBool::new(false)));
        let job = ThreadJob::new(signals.clone(), None, false, sender);
        signals.trigger();

        let child = job.try_spawn(|| panic!("interrupted job must not spawn"));
        assert!(child.unwrap().is_none());
    }

    #[test]
    fn spawned_process_is_registered_before_returning() {
        let (sender, _) = mpsc::channel();
        let signals = Signals::new(Arc::new(AtomicBool::new(false)));
        let job = ThreadJob::new(signals, None, false, sender);
        // Listing this test binary's tests needs no platform-specific executable, and the
        // child terminates by itself if an assertion fails.
        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
        command
            .arg("--list")
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null());
        let mut child = job
            .try_spawn(|| {
                #[cfg(unix)]
                {
                    ForegroundChild::spawn(
                        command,
                        false,
                        true,
                        &Arc::new((
                            std::sync::atomic::AtomicU32::new(0),
                            std::sync::atomic::AtomicU32::new(0),
                        )),
                    )
                }
                #[cfg(not(unix))]
                {
                    ForegroundChild::spawn(command)
                }
            })
            .unwrap()
            .unwrap();

        assert!(job.collect_pids().contains(&child.pid()));
        child.wait().unwrap();
        job.remove_pid(child.pid());
        assert!(job.collect_pids().is_empty());
    }

    #[test]
    fn spawn_holds_pid_lock_until_registration() {
        let (sender, _) = mpsc::channel();
        let signals = Signals::new(Arc::new(AtomicBool::new(false)));
        let job = ThreadJob::new(signals, None, false, sender);
        let (started_tx, started_rx) = mpsc::channel();
        let (release_tx, release_rx) = mpsc::channel();

        std::thread::scope(|scope| {
            let job = &job;
            let spawning = scope.spawn(move || {
                job.try_spawn(|| {
                    started_tx.send(()).unwrap();
                    release_rx.recv_timeout(Duration::from_secs(30)).unwrap();
                    Err(std::io::Error::other("test spawn failure"))
                })
            });

            started_rx.recv_timeout(Duration::from_secs(30)).unwrap();
            // kill() takes this same lock after triggering the signals, so it cannot finish
            // while an external is being spawned but has not yet registered its PID.
            let locked = job.pids.try_lock().is_err();
            release_tx.send(()).unwrap();
            assert!(spawning.join().unwrap().is_err());
            assert!(locked, "shutdown can race with process creation");
        });

        job.kill().unwrap();
        assert!(job.collect_pids().is_empty());
    }
}