1use anyhow::{bail, Context, Result};
3use log::error;
4use serde::{Deserialize, Serialize};
5use std::collections::BTreeMap;
6use std::fmt::Write;
7use std::path::Path;
8use std::process::exit;
9use std::sync::Mutex;
10
11use super::{IoCostQoSOvr, JobSpec};
12use rd_agent_intf;
13use rd_util::*;
14
15pub const GITHUB_DOC_LINK: &'static str =
16 "https://github.com/facebookexperimental/resctl-demo/tree/main/resctl-bench/doc";
17
18lazy_static::lazy_static! {
19 static ref TOP_ARGS_STR: String = {
20 let dfl_args = Args::default();
21 format!(
22 "-r, --result=[RESULTFILE] 'Result json file'
23 -d, --dir=[TOPDIR] 'Top dir for bench files (dfl: {dfl_dir})'
24 -D, --dev=[DEVICE] 'Scratch device override (e.g. nvme0n1)'
25 -l, --linux=[PATH] 'Path to linux.tar, downloaded automatically if not specified'
26 -R, --rep-retention=[SECS] '1s report retention in seconds (dfl: {dfl_rep_ret:.1}h)'
27 -M, --mem-profile=[PROF|off] 'Memory profile in power-of-two gigabytes or \"off\" (dfl: {dfl_mem_prof})'
28 -m, --mem-avail=[SIZE] 'Amount of memory available for resctl-bench'
29 --mem-margin=[PCT] 'Memory margin for system.slice (dfl: {dfl_mem_margin}%)'
30 --systemd-timeout=[SECS] 'Systemd timeout (dfl: {dfl_systemd_timeout})'
31 --hashd-size=[SIZE] 'hashd memory footprint override'
32 --hashd-cpu-load=[keep|fake|real] 'hashd fake cpu load mode override'
33 --iocost-qos=[OVRS] 'iocost QoS overrides'
34 --swappiness=[OVR] 'swappiness override [0, 200]'
35 -a, --args=[FILE] 'Loads base command line arguments from FILE'
36 --iocost-from-sys 'Uses parameters from io.cost.{{model,qos}} instead of bench.json'
37 --keep-reports 'Prevents deleting expired report files'
38 --clear-reports 'Removes existing report files'
39 --force 'Ignore missing system requirements and proceed'
40 --force-shadow-inode-prot-test 'Force shadow inode protection test'
41 --skip-shadow-inode-prot-test 'Assume shadow inodes are protected without testing'
42 --test 'Test mode for development'
43 -v... 'Sets the level of verbosity'
44 --logfile=[FILE] 'Specify file to dump logs'",
45 dfl_dir = dfl_args.dir,
46 dfl_rep_ret = dfl_args.rep_retention,
47 dfl_mem_prof = dfl_args.mem_profile.unwrap(),
48 dfl_mem_margin = format_pct(dfl_args.mem_margin),
49 dfl_systemd_timeout = format_duration(dfl_args.systemd_timeout),
50 )
51 };
52 static ref HELP_BODY: Mutex<&'static str> = Mutex::new("");
53 static ref AFTER_HELP: Mutex<&'static str> = Mutex::new("");
54 static ref DOC_AFTER_HELP: Mutex<&'static str> = Mutex::new("");
55 static ref DOC_AFTER_HELP_FOOTER: String = format!(r#"
56The pages are in markdown. To convert, e.g., to pdf:
57
58 resctl-bench doc $SUBJECT | pandoc --toc --toc-depth=3 -o $SUBJECT.pdf
59
60The documentation can also be viewed at:
61
62 {}
63
64"#, GITHUB_DOC_LINK);
65}
66
67fn static_format_bench_list(header: &str, list: &[(String, String)], footer: &str) -> &'static str {
68 let mut buf = String::new();
69 let kind_width = list.iter().map(|pair| pair.0.len()).max().unwrap_or(0);
70 write!(buf, "{}", header).unwrap();
71 for pair in list.iter() {
72 writeln!(
73 buf,
74 " {:width$} {}",
75 &pair.0,
76 &pair.1,
77 width = kind_width
78 )
79 .unwrap();
80 }
81 write!(buf, "{}", footer).unwrap();
82 Box::leak(Box::new(buf))
83}
84
85pub fn set_bench_list(mut list: Vec<(String, String)>) {
86 *AFTER_HELP.lock().unwrap() = static_format_bench_list(
88 "BENCHMARKS: Use the \"doc\" subcommand for more info\n",
89 &list,
90 "",
91 );
92
93 list.insert(
95 0,
96 (
97 "common".to_string(),
98 "Overview, Common Concepts and Options".to_string(),
99 ),
100 );
101 list.push((
102 "shadow-inode".to_string(),
103 "Information on inode shadow entry protection".to_string(),
104 ));
105 *DOC_AFTER_HELP.lock().unwrap() =
106 static_format_bench_list("SUBJECTS:\n", &list, &DOC_AFTER_HELP_FOOTER);
107 list.remove(0);
108}
109
110#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
111pub enum Mode {
112 Run,
113 Study,
114 Solve,
115 Format,
116 Summary,
117 #[cfg(feature = "lambda")]
118 Lambda,
119 Upload,
120 Pack,
121 Merge,
122 Deps,
123 Doc,
124}
125
126#[derive(Debug, Clone, Serialize, Deserialize)]
127#[serde(default)]
128pub struct Args {
129 pub dir: String,
130 pub dev: Option<String>,
131 pub linux_tar: Option<String>,
132 pub rep_retention: u64,
133 pub systemd_timeout: f64,
134 pub hashd_size: Option<usize>,
135 pub hashd_fake_cpu_load: Option<bool>,
136 pub mem_profile: Option<u32>,
137 pub mem_avail: usize,
138 pub mem_margin: f64,
139 pub mode: Mode,
140 pub iocost_qos_ovr: IoCostQoSOvr,
141 pub swappiness_ovr: Option<u32>,
142 pub job_specs: Vec<JobSpec>,
143
144 #[serde(skip)]
145 pub result: String,
146 #[serde(skip)]
147 pub study_rep_d: String,
148 #[serde(skip)]
149 pub iocost_from_sys: bool,
150 #[serde(skip)]
151 pub keep_reports: bool,
152 #[serde(skip)]
153 pub clear_reports: bool,
154 #[serde(skip)]
155 pub force: bool,
156 #[serde(skip)]
157 pub force_shadow_inode_prot_test: bool,
158 #[serde(skip)]
159 pub skip_shadow_inode_prot_test: bool,
160 #[serde(skip)]
161 pub test: bool,
162 #[serde(skip)]
163 pub verbosity: u32,
164 #[serde(skip)]
165 pub logfile: Option<String>,
166 #[serde(skip)]
167 pub rstat: u32,
168 #[serde(skip)]
169 pub merge_srcs: Vec<String>,
170 #[serde(skip)]
171 pub merge_by_id: bool,
172 #[serde(skip)]
173 pub merge_ignore_versions: bool,
174 #[serde(skip)]
175 pub merge_ignore_sysreqs: bool,
176 #[serde(skip)]
177 pub merge_multiple: bool,
178 #[serde(skip)]
179 pub upload_email: Option<String>,
180 #[serde(skip)]
181 pub upload_github: Option<String>,
182 #[serde(skip)]
183 pub upload_url: Option<String>,
184 #[serde(skip)]
185 pub doc_subjects: Vec<String>,
186}
187
188impl Default for Args {
189 fn default() -> Self {
190 Self {
191 dir: rd_agent_intf::Args::default().dir.clone(),
192 dev: None,
193 linux_tar: None,
194 result: "".into(),
195 mode: Mode::Run,
196 iocost_qos_ovr: Default::default(),
197 swappiness_ovr: None,
198 job_specs: Default::default(),
199 study_rep_d: "".into(),
200 rep_retention: 7 * 24 * 3600,
201 systemd_timeout: 120.0,
202 hashd_size: None,
203 hashd_fake_cpu_load: None,
204 mem_profile: Some(Self::DFL_MEM_PROFILE),
205 mem_avail: 0,
206 mem_margin: rd_agent_intf::SliceConfig::DFL_MEM_MARGIN,
207 iocost_from_sys: false,
208 keep_reports: false,
209 clear_reports: false,
210 force: false,
211 force_shadow_inode_prot_test: false,
212 skip_shadow_inode_prot_test: false,
213 test: false,
214 verbosity: 0,
215 logfile: None,
216 rstat: 0,
217 merge_srcs: vec![],
218 merge_by_id: false,
219 merge_ignore_versions: false,
220 merge_ignore_sysreqs: false,
221 merge_multiple: false,
222 upload_email: None,
223 upload_github: None,
224 upload_url: None,
225 doc_subjects: vec![],
226 }
227 }
228}
229
230impl Args {
231 pub const RB_BENCH_FILENAME: &'static str = "rb-bench.json";
232 pub const DFL_MEM_PROFILE: u32 = 16;
233
234 pub fn set_help_body(help: &'static str) {
235 *HELP_BODY.lock().unwrap() = help;
236 }
237
238 pub fn demo_bench_knobs_path(&self) -> String {
239 self.dir.clone() + "/" + rd_agent_intf::BENCH_FILENAME
240 }
241
242 pub fn bench_knobs_path(&self) -> String {
243 self.dir.clone() + "/" + Self::RB_BENCH_FILENAME
244 }
245
246 pub fn parse_propset(input: &str) -> BTreeMap<String, String> {
247 let mut propset = BTreeMap::<String, String>::new();
248 for tok in input.split(',') {
249 if tok.len() == 0 {
250 continue;
251 }
252
253 let mut kv = tok.splitn(2, '=').collect::<Vec<&str>>();
255 while kv.len() < 2 {
256 kv.push("");
257 }
258
259 propset.insert(kv[0].into(), kv[1].into());
260 }
261 propset
262 }
263
264 pub fn parse_job_spec(spec: &str) -> Result<JobSpec> {
265 let mut groups = spec.split(':');
266
267 let kind = match groups.next() {
268 Some(v) => v,
269 None => bail!("invalid job type"),
270 };
271
272 let mut props = vec![];
273 let mut id = None;
274 let mut passive = None;
275
276 for group in groups {
277 let mut propset = Self::parse_propset(group);
278 id = propset.remove("id");
279 passive = propset.remove("passive");
280 props.push(propset);
281 }
282
283 if props.len() == 0 {
285 props.push(Default::default());
286 }
287
288 Ok(JobSpec::new(kind, id.as_deref(), passive.as_deref(), props))
289 }
290
291 fn parse_job_specs(subm: &clap::ArgMatches) -> Result<Vec<JobSpec>> {
292 let mut jobsets = BTreeMap::<usize, Vec<JobSpec>>::new();
293
294 match (subm.indices_of("spec"), subm.values_of("spec")) {
295 (Some(idxs), Some(specs)) => {
296 for (idx, spec) in idxs.zip(specs) {
297 match Self::parse_job_spec(spec) {
298 Ok(v) => {
299 jobsets.insert(idx, vec![v]);
300 }
301 Err(e) => bail!("spec {:?}: {}", spec, &e),
302 }
303 }
304 }
305 _ => {}
306 }
307
308 match (subm.indices_of("file"), subm.values_of("file")) {
309 (Some(idxs), Some(fnames)) => {
310 for (idx, fname) in idxs.zip(fnames) {
311 match Self::load(fname) {
312 Ok(v) => {
313 jobsets.insert(idx, v.job_specs);
314 }
315 Err(e) => bail!("file {:?}: {}", fname, &e),
316 }
317 }
318 }
319 _ => {}
320 }
321
322 let mut job_specs = Vec::new();
323 if jobsets.len() > 0 {
324 for jobset in jobsets.values_mut() {
325 job_specs.append(jobset);
326 }
327 }
328 Ok(job_specs)
329 }
330
331 fn process_subcommand(&mut self, mode: Mode, subm: &clap::ArgMatches) -> bool {
332 let mut updated = false;
333
334 if self.mode != mode {
335 self.job_specs = vec![];
336 self.mode = mode;
337 updated = true;
338 }
339
340 match mode {
341 Mode::Study => {
342 self.study_rep_d = match subm.value_of("reports") {
343 Some(v) => v.to_string(),
344 None => format!(
345 "{}-report.d",
346 Path::new(&self.result)
347 .file_stem()
348 .unwrap()
349 .to_string_lossy()
350 ),
351 }
352 }
353 Mode::Format => self.rstat = subm.occurrences_of("rstat") as u32,
354 _ => {}
355 }
356
357 match Self::parse_job_specs(subm) {
358 Ok(job_specs) => {
359 if job_specs.len() > 0 {
360 self.job_specs = job_specs;
361 updated = true;
362 }
363 match mode {
364 Mode::Run | Mode::Solve | Mode::Study => {
365 if self.job_specs.len() == 0 {
366 error!("{:?} requires job specs", &mode);
367 exit(1);
368 }
369 }
370 _ => {}
371 }
372 }
373 Err(e) => {
374 error!("{}", &e);
375 exit(1);
376 }
377 }
378
379 updated
380 }
381}
382
383impl JsonLoad for Args {}
384impl JsonSave for Args {}
385
386impl JsonArgs for Args {
387 fn match_cmdline() -> clap::ArgMatches<'static> {
388 let job_file_arg = clap::Arg::with_name("file")
389 .long("file")
390 .short("f")
391 .multiple(true)
392 .takes_value(true)
393 .number_of_values(1)
394 .help("Benchmark job file");
395 let job_spec_arg = clap::Arg::with_name("spec")
396 .multiple(true)
397 .help("Benchmark job spec - \"BENCH_TYPE[:KEY[=VAL][,KEY[=VAL]...]]...\"");
398
399 let mut app = clap::App::new("resctl-bench")
400 .version((*super::FULL_VERSION).as_str())
401 .author(clap::crate_authors!("\n"))
402 .about("Facebook Resource Control Benchmarks")
403 .setting(clap::AppSettings::UnifiedHelpMessage)
404 .setting(clap::AppSettings::DeriveDisplayOrder)
405 .before_help(*HELP_BODY.lock().unwrap())
406 .args_from_usage(&TOP_ARGS_STR)
407 .subcommand(
408 clap::SubCommand::with_name("run")
409 .about("Runs benchmarks")
410 .arg(job_file_arg.clone())
411 .arg(job_spec_arg.clone()),
412 )
413 .subcommand(
414 clap::SubCommand::with_name("study")
415 .about("Studies benchmark results, all benchmarks must be complete")
416 .arg(clap::Arg::with_name("reports")
417 .long("reports")
418 .short("R")
419 .takes_value(true)
420 .help("Study reports in the directory (default: RESULTFILE_BASENAME-report.d/)"),
421 )
422 .arg(job_file_arg.clone())
423 .arg(job_spec_arg.clone()),
424 )
425 .subcommand(
426 clap::SubCommand::with_name("solve")
427 .about("Solves benchmark results, optional phase to be used with merge")
428 .arg(job_file_arg.clone())
429 .arg(job_spec_arg.clone()),
430 )
431 .subcommand(
432 clap::SubCommand::with_name("format")
433 .about("Formats benchmark results")
434 .arg(
435 clap::Arg::with_name("rstat")
436 .long("rstat")
437 .short("R")
438 .multiple(true)
439 .help(
440 "Report extra resource stats if available (repeat for even more)",
441 ),
442 )
443 .arg(job_file_arg.clone())
444 .arg(job_spec_arg.clone()),
445 )
446 .subcommand(
447 clap::SubCommand::with_name("summary")
448 .about("Summarizes benchmark results")
449 .arg(job_file_arg.clone())
450 .arg(job_spec_arg.clone()),
451 )
452 .subcommand(clap::SubCommand::with_name("pack").about(
453 "Create a tarball containing the result file and the associated report files",
454 ))
455 .subcommand(
456 clap::SubCommand::with_name("merge")
457 .about("Merges result files from multiple runs on supported benchmarks")
458 .arg(
459 clap::Arg::with_name("SOURCEFILE")
460 .multiple(true)
461 .required(true)
462 .help("Result file to merge")
463 )
464 .arg(
465 clap::Arg::with_name("by-id")
466 .long("by-id")
467 .help("Don't ignore bench IDs when merging")
468 )
469 .arg(
470 clap::Arg::with_name("ignore-versions")
471 .long("ignore-versions")
472 .help("Ignore resctl-demo and bench versions when merging")
473 )
474 .arg(
475 clap::Arg::with_name("ignore-sysreqs")
476 .long("ignore-sysreqs")
477 .help("Accept results with missed sysreqs")
478 )
479 .arg(
480 clap::Arg::with_name("multiple")
481 .long("multiple")
482 .help("Allow more than one result per kind (and optionally id)")
483 )
484 )
485 .subcommand(
486 clap::App::new("upload")
487 .about("Upload results to community database")
488 .arg(
489 clap::Arg::with_name("upload-url")
490 .long("upload-url")
491 .takes_value(true)
492 .number_of_values(1)
493 .env("RESCTL_BENCH_UPLOAD_URL")
494 .required(true)
495 .help("The URL where the lambda function is accessible")
496 )
497 .arg(
498 clap::Arg::with_name("my-email")
499 .long("my-email")
500 .takes_value(true)
501 .number_of_values(1)
502 .help("Include your email address on your submission")
503 )
504 .arg(
505 clap::Arg::with_name("my-github")
506 .long("my-github")
507 .takes_value(true)
508 .number_of_values(1)
509 .help("Include your github username on your submission")
510 )
511 )
512 .subcommand(
513 clap::App::new("deps")
514 .about("Test all dependencies")
515 )
516 .subcommand(
517 clap::App::new("doc")
518 .about("Shows documentations")
519 .arg(
520 clap::Arg::with_name("SUBJECT")
521 .multiple(true)
522 .required(true)
523 .help("Documentation subject to show")
524 )
525 .after_help(*DOC_AFTER_HELP.lock().unwrap())
526 );
527
528 if cfg!(feature = "lambda") {
529 app = app.subcommand(
530 clap::SubCommand::with_name("lambda")
531 .about("AWS lambda function that handles automated submission of results"),
532 );
533 }
534
535 app.after_help(*AFTER_HELP.lock().unwrap()).get_matches()
536 }
537
538 fn verbosity(matches: &clap::ArgMatches) -> u32 {
539 matches.occurrences_of("v") as u32
540 }
541
542 fn log_file(matches: &clap::ArgMatches) -> String {
543 match matches.value_of("logfile") {
544 Some(v) => v.to_string(),
545 None => "".to_string(),
546 }
547 }
548
549 fn process_cmdline(&mut self, matches: &clap::ArgMatches) -> bool {
550 let dfl = Args::default();
551 let mut updated = false;
552
553 if let Some(v) = matches.value_of("dir") {
554 self.dir = if v.len() > 0 {
555 v.to_string()
556 } else {
557 dfl.dir.clone()
558 };
559 updated = true;
560 }
561 if let Some(v) = matches.value_of("dev") {
562 self.dev = if v.len() > 0 {
563 Some(v.to_string())
564 } else {
565 None
566 };
567 updated = true;
568 }
569 if let Some(v) = matches.value_of("linux") {
570 self.linux_tar = if v.len() > 0 {
571 Some(v.to_string())
572 } else {
573 None
574 };
575 updated = true;
576 }
577 if let Some(v) = matches.value_of("rep-retention") {
578 self.rep_retention = if v.len() > 0 {
579 v.parse::<u64>().unwrap()
580 } else {
581 dfl.rep_retention
582 };
583 updated = true;
584 }
585 if let Some(v) = matches.value_of("systemd-timeout") {
586 self.systemd_timeout = if v.len() > 0 {
587 parse_duration(v).unwrap().max(1.0)
588 } else {
589 dfl.systemd_timeout
590 };
591 updated = true;
592 }
593 if let Some(v) = matches.value_of("hashd-size") {
594 self.hashd_size = if v.len() > 0 {
595 Some((parse_size(v).unwrap() as usize).max(*PAGE_SIZE))
596 } else {
597 None
598 };
599 updated = true;
600 }
601 if let Some(v) = matches.value_of("hashd-cpu-load") {
602 self.hashd_fake_cpu_load = match v {
603 "keep" | "" => None,
604 "fake" => Some(true),
605 "real" => Some(false),
606 v => panic!("Invalid --hashd-cpu-load value {:?}", v),
607 };
608 updated = true;
609 }
610 if let Some(v) = matches.value_of("iocost-qos") {
611 self.iocost_qos_ovr = if v.len() > 0 {
612 let mut ovr = IoCostQoSOvr::default();
613 for (k, v) in Self::parse_propset(v).iter() {
614 ovr.parse(k, v)
615 .with_context(|| format!("Parsing iocost QoS override \"{}={}\"", k, v))
616 .unwrap();
617 }
618 ovr
619 } else {
620 Default::default()
621 };
622 updated = true;
623 }
624 if let Some(v) = matches.value_of("swappiness") {
625 self.swappiness_ovr = if v.len() > 0 {
626 let v = v.parse::<u32>().expect("Parsing swappiness");
627 if v > 200 {
628 panic!("Swappiness {} out of range", v);
629 }
630 Some(v)
631 } else {
632 None
633 };
634 }
635 if let Some(v) = matches.value_of("mem-profile") {
636 self.mem_profile = match v {
637 "off" => None,
638 v => Some(v.parse::<u32>().expect("Invalid mem-profile")),
639 };
640 updated = true;
641 }
642 if let Some(v) = matches.value_of("mem-avail") {
643 self.mem_avail = if v.len() > 0 {
644 parse_size(v).unwrap() as usize
645 } else {
646 0
647 };
648 updated = true;
649 }
650 if let Some(v) = matches.value_of("mem-margin") {
651 self.mem_margin = if v.len() > 0 {
652 parse_frac(v).unwrap()
653 } else {
654 dfl.mem_margin
655 };
656 updated = true;
657 }
658
659 self.result = matches.value_of("result").unwrap_or("").into();
660 self.iocost_from_sys = matches.is_present("iocost-from-sys");
661 self.keep_reports = matches.is_present("keep-reports");
662 self.clear_reports = matches.is_present("clear-reports");
663 self.force = matches.is_present("force");
664 self.force_shadow_inode_prot_test = matches.is_present("force-shadow-inode-prot-test");
665 self.skip_shadow_inode_prot_test = matches.is_present("skip-shadow-inode-prot-test");
666 self.test = matches.is_present("test");
667 self.verbosity = Self::verbosity(matches);
668 self.logfile = matches.value_of("logfile").map(|x| x.to_string());
669
670 updated |= match matches.subcommand() {
671 ("run", Some(subm)) => self.process_subcommand(Mode::Run, subm),
672 ("study", Some(subm)) => self.process_subcommand(Mode::Study, subm),
673 ("solve", Some(subm)) => self.process_subcommand(Mode::Solve, subm),
674 ("format", Some(subm)) => self.process_subcommand(Mode::Format, subm),
675 ("summary", Some(subm)) => self.process_subcommand(Mode::Summary, subm),
676 #[cfg(feature = "lambda")]
677 ("lambda", Some(subm)) => self.process_subcommand(Mode::Lambda, subm),
678 ("upload", Some(subm)) => {
679 self.mode = Mode::Upload;
680 self.upload_email = subm.value_of("my-email").map(|s| s.into());
681 self.upload_github = subm.value_of("my-github").map(|s| s.into());
682 self.upload_url = subm.value_of("upload-url").map(|s| s.into());
683 false
684 }
685 ("pack", Some(_subm)) => {
686 self.mode = Mode::Pack;
687 false
688 }
689 ("merge", Some(subm)) => {
690 self.mode = Mode::Merge;
691 self.merge_by_id = subm.is_present("by-id");
692 self.merge_ignore_versions = subm.is_present("ignore-versions");
693 self.merge_ignore_sysreqs = subm.is_present("ignore-sysreqs");
694 self.merge_multiple = subm.is_present("multiple");
695 self.merge_srcs = subm
696 .values_of("SOURCEFILE")
697 .unwrap()
698 .map(|x| x.to_string())
699 .collect();
700 false
701 }
702 ("deps", Some(_subm)) => {
703 self.mode = Mode::Deps;
704 false
705 }
706 ("doc", Some(subm)) => {
707 self.mode = Mode::Doc;
708 self.doc_subjects = subm
709 .values_of("SUBJECT")
710 .unwrap()
711 .map(|x| x.to_string())
712 .collect();
713 false
714 }
715 _ => false,
716 };
717
718 if self.mode != Mode::Doc && self.mode != Mode::Deps && self.result.len() == 0 {
719 error!("{:?} requires --result", &self.mode);
720 exit(1);
721 }
722
723 updated
724 }
725}