below_dump/
lib.rs

1// Copyright (c) Facebook, Inc. and its affiliates.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![deny(clippy::all)]
16#![allow(clippy::too_many_arguments)]
17
18use std::fs;
19use std::fs::File;
20use std::io;
21use std::io::Write;
22use std::path::PathBuf;
23use std::str::FromStr;
24use std::sync::mpsc::Receiver;
25use std::time::SystemTime;
26
27use anyhow::bail;
28use anyhow::Context;
29use anyhow::Error;
30use anyhow::Result;
31use common::cliutil;
32use common::util::get_belowrc_dump_section_key;
33use common::util::get_belowrc_filename;
34use common::util::timestamp_to_datetime;
35use model::Field;
36use model::FieldId;
37use model::Queriable;
38use serde_json::json;
39use serde_json::Value;
40use store::advance::new_advance_local;
41use store::advance::new_advance_remote;
42use store::Advance;
43use store::Direction;
44use tar::Archive;
45use tempfile::TempDir;
46use toml::value::Value as TValue;
47
48pub mod btrfs;
49pub mod cgroup;
50pub mod command;
51pub mod disk;
52pub mod ethtool;
53pub mod iface;
54pub mod network;
55pub mod print;
56pub mod process;
57pub mod system;
58pub mod tc;
59pub mod tmain;
60pub mod transport;
61
62#[cfg(test)]
63mod test;
64
65use command::expand_fields;
66pub use command::DumpCommand;
67use command::GeneralOpt;
68use command::OutputFormat;
69use render::HasRenderConfigForDump;
70use tmain::dump_timeseries;
71use tmain::Dumper;
72use tmain::IterExecResult;
73
74/// Fields available to all commands. Each enum represents some semantics and
75/// knows how to extract relevant data from a CommonFieldContext.
76#[derive(
77    Clone,
78    Debug,
79    PartialEq,
80    below_derive::EnumFromStr,
81    below_derive::EnumToString,
82    enum_iterator::Sequence
83)]
84pub enum CommonField {
85    Timestamp,
86    Datetime,
87}
88
89/// Context for initializing CommonFields.
90pub struct CommonFieldContext {
91    pub timestamp: i64,
92    pub hostname: String,
93}
94
95impl CommonField {
96    pub fn get_field(&self, ctx: &CommonFieldContext) -> Option<Field> {
97        match self {
98            Self::Timestamp => Field::from(ctx.timestamp),
99            Self::Datetime => Field::from(timestamp_to_datetime(&ctx.timestamp)),
100        }
101        .into()
102    }
103}
104
105/// Generic field for dumping different types of models. It's either a
106/// CommonField or a FieldId that extracts a Field from a given model. It
107/// represents a unified interface for dumpable items.
108#[derive(Clone, Debug, PartialEq)]
109pub enum DumpField<F: FieldId> {
110    Common(CommonField),
111    FieldId(F),
112}
113
114pub type CgroupField = DumpField<model::SingleCgroupModelFieldId>;
115pub type ProcessField = DumpField<model::SingleProcessModelFieldId>;
116pub type SystemField = DumpField<model::SystemModelFieldId>;
117pub type DiskField = DumpField<model::SingleDiskModelFieldId>;
118pub type BtrfsField = DumpField<model::BtrfsModelFieldId>;
119pub type NetworkField = DumpField<model::NetworkModelFieldId>;
120pub type IfaceField = DumpField<model::SingleNetModelFieldId>;
121// Essentially the same as NetworkField
122pub type TransportField = DumpField<model::NetworkModelFieldId>;
123pub type EthtoolQueueField = DumpField<model::SingleQueueModelFieldId>;
124pub type TcField = DumpField<model::SingleTcModelFieldId>;
125
126fn get_advance(
127    logger: slog::Logger,
128    dir: PathBuf,
129    host: Option<String>,
130    port: Option<u16>,
131    snapshot: Option<String>,
132    opts: &command::GeneralOpt,
133) -> Result<(SystemTime, SystemTime, Advance)> {
134    let (time_begin, time_end) = cliutil::system_time_range_from_date_and_adjuster(
135        opts.begin.as_str(),
136        opts.end.as_deref(),
137        opts.duration.as_deref(),
138        opts.yesterdays.as_deref(),
139    )?;
140
141    let mut advance = match (host, snapshot) {
142        (None, None) => new_advance_local(logger.clone(), dir, time_begin),
143        (Some(host), None) => new_advance_remote(logger.clone(), host, port, time_begin)?,
144        (None, Some(snapshot)) => {
145            let mut tarball =
146                Archive::new(fs::File::open(snapshot).context("Failed to open snapshot file")?);
147            let mut snapshot_dir = TempDir::with_prefix("snapshot_replay.")?.into_path();
148            tarball.unpack(&snapshot_dir)?;
149            // Find and append the name of the original snapshot directory
150            for path in fs::read_dir(&snapshot_dir)? {
151                snapshot_dir.push(path.unwrap().file_name());
152            }
153            new_advance_local(logger.clone(), snapshot_dir, time_begin)
154        }
155        (Some(_), Some(_)) => {
156            bail!("--host and --snapshot are incompatible options")
157        }
158    };
159
160    advance.initialize();
161
162    Ok((time_begin, time_end, advance))
163}
164
165/// Try to read $HOME/.config/below/belowrc file and generate a list of keys which will
166/// be used as fields. Any errors happen in this function will directly trigger a panic.
167pub fn parse_pattern<T: FromStr>(
168    filename: String,
169    pattern_key: String,
170    section_key: &str,
171) -> Option<Vec<T>> {
172    let dump_map = match std::fs::read_to_string(filename) {
173        Ok(belowrc_str) => match belowrc_str.parse::<TValue>() {
174            Ok(belowrc_val) => belowrc_val
175                .get(get_belowrc_dump_section_key())
176                .unwrap_or_else(|| {
177                    panic!(
178                        "Failed to get section key: [{}.{}]",
179                        get_belowrc_dump_section_key(),
180                        section_key
181                    )
182                })
183                .to_owned(),
184            Err(e) => panic!("Failed to parse belowrc file: {:#}", e),
185        },
186        Err(e) => panic!("Failed to parse belowrc file: {:#}", e),
187    };
188
189    Some(
190        dump_map
191            .get(section_key)
192            .unwrap_or_else(|| {
193                panic!(
194                    "Failed to get section key: [{}.{}]",
195                    get_belowrc_dump_section_key(),
196                    section_key
197                )
198            })
199            .get(&pattern_key)
200            .unwrap_or_else(|| panic!("Failed to get pattern key: {}", pattern_key))
201            .as_array()
202            .unwrap_or_else(|| panic!("Failed to parse pattern {} value to array.", pattern_key))
203            .iter()
204            .map(|field| {
205                T::from_str(
206                    field.as_str().unwrap_or_else(|| {
207                        panic!("Failed to parse field key {} into string", field)
208                    }),
209                )
210                .map_err(|_| format!("Failed to parse field key: {}", field))
211                .unwrap()
212            })
213            .collect(),
214    )
215}
216
217pub fn run(
218    logger: slog::Logger,
219    errs: Receiver<Error>,
220    dir: PathBuf,
221    host: Option<String>,
222    port: Option<u16>,
223    snapshot: Option<String>,
224    cmd: DumpCommand,
225) -> Result<()> {
226    let filename = get_belowrc_filename();
227
228    match cmd {
229        DumpCommand::System {
230            fields,
231            opts,
232            pattern,
233        } => {
234            let (time_begin, time_end, advance) =
235                get_advance(logger, dir, host, port, snapshot, &opts)?;
236            let default = opts.everything || opts.default;
237            let detail = opts.everything || opts.detail;
238            let fields = if let Some(pattern_key) = pattern {
239                parse_pattern(filename, pattern_key, "system")
240            } else {
241                fields
242            };
243            let fields = expand_fields(
244                match fields.as_ref() {
245                    Some(fields) if !default => fields,
246                    _ => command::DEFAULT_SYSTEM_FIELDS,
247                },
248                detail,
249            );
250            let system = system::System::new(&opts, fields);
251            let mut output: Box<dyn Write> = match opts.output.as_ref() {
252                Some(file_path) => Box::new(File::create(file_path)?),
253                None => Box::new(io::stdout()),
254            };
255            dump_timeseries(
256                advance,
257                time_begin,
258                time_end,
259                &system,
260                output.as_mut(),
261                opts.output_format,
262                opts.br,
263                errs,
264            )
265        }
266        DumpCommand::Disk {
267            fields,
268            opts,
269            select,
270            pattern,
271        } => {
272            let (time_begin, time_end, advance) =
273                get_advance(logger, dir, host, port, snapshot, &opts)?;
274            let default = opts.everything || opts.default;
275            let detail = opts.everything || opts.detail;
276            let fields = if let Some(pattern_key) = pattern {
277                parse_pattern(filename, pattern_key, "disk")
278            } else {
279                fields
280            };
281            let fields = expand_fields(
282                match fields.as_ref() {
283                    Some(fields) if !default => fields,
284                    _ => command::DEFAULT_DISK_FIELDS,
285                },
286                detail,
287            );
288            let disk = disk::Disk::new(&opts, select, fields);
289            let mut output: Box<dyn Write> = match opts.output.as_ref() {
290                Some(file_path) => Box::new(File::create(file_path)?),
291                None => Box::new(io::stdout()),
292            };
293            dump_timeseries(
294                advance,
295                time_begin,
296                time_end,
297                &disk,
298                output.as_mut(),
299                opts.output_format,
300                opts.br,
301                errs,
302            )
303        }
304        DumpCommand::Btrfs {
305            fields,
306            opts,
307            select,
308            pattern,
309        } => {
310            let (time_begin, time_end, advance) =
311                get_advance(logger, dir, host, port, snapshot, &opts)?;
312            let default = opts.everything || opts.default;
313            let detail = opts.everything || opts.detail;
314            let fields = if let Some(pattern_key) = pattern {
315                parse_pattern(filename, pattern_key, "btrfs")
316            } else {
317                fields
318            };
319            let fields = expand_fields(
320                match fields.as_ref() {
321                    Some(fields) if !default => fields,
322                    _ => command::DEFAULT_BTRFS_FIELDS,
323                },
324                detail,
325            );
326            let btrfs = btrfs::Btrfs::new(&opts, select, fields);
327            let mut output: Box<dyn Write> = match opts.output.as_ref() {
328                Some(file_path) => Box::new(File::create(file_path)?),
329                None => Box::new(io::stdout()),
330            };
331            dump_timeseries(
332                advance,
333                time_begin,
334                time_end,
335                &btrfs,
336                output.as_mut(),
337                opts.output_format,
338                opts.br,
339                errs,
340            )
341        }
342        DumpCommand::Process {
343            fields,
344            opts,
345            select,
346            pattern,
347        } => {
348            let (time_begin, time_end, advance) =
349                get_advance(logger, dir, host, port, snapshot, &opts)?;
350            let default = opts.everything || opts.default;
351            let detail = opts.everything || opts.detail;
352            let fields = if let Some(pattern_key) = pattern {
353                parse_pattern(filename, pattern_key, "process")
354            } else {
355                fields
356            };
357            let fields = expand_fields(
358                match fields.as_ref() {
359                    Some(fields) if !default => fields,
360                    _ => command::DEFAULT_PROCESS_FIELDS,
361                },
362                detail,
363            );
364            let process = process::Process::new(&opts, select, fields);
365            let mut output: Box<dyn Write> = match opts.output.as_ref() {
366                Some(file_path) => Box::new(File::create(file_path)?),
367                None => Box::new(io::stdout()),
368            };
369            dump_timeseries(
370                advance,
371                time_begin,
372                time_end,
373                &process,
374                output.as_mut(),
375                opts.output_format,
376                opts.br,
377                errs,
378            )
379        }
380        DumpCommand::Cgroup {
381            fields,
382            opts,
383            select,
384            pattern,
385        } => {
386            let (time_begin, time_end, advance) =
387                get_advance(logger, dir, host, port, snapshot, &opts)?;
388            let default = opts.everything || opts.default;
389            let detail = opts.everything || opts.detail;
390            let fields = if let Some(pattern_key) = pattern {
391                parse_pattern(filename, pattern_key, "cgroup")
392            } else {
393                fields
394            };
395            let fields = expand_fields(
396                match fields.as_ref() {
397                    Some(fields) if !default => fields,
398                    _ => command::DEFAULT_CGROUP_FIELDS,
399                },
400                detail,
401            );
402            let cgroup = cgroup::Cgroup::new(&opts, select, fields);
403            let mut output: Box<dyn Write> = match opts.output.as_ref() {
404                Some(file_path) => Box::new(File::create(file_path)?),
405                None => Box::new(io::stdout()),
406            };
407            dump_timeseries(
408                advance,
409                time_begin,
410                time_end,
411                &cgroup,
412                output.as_mut(),
413                opts.output_format,
414                opts.br,
415                errs,
416            )
417        }
418        DumpCommand::Iface {
419            fields,
420            opts,
421            select,
422            pattern,
423        } => {
424            let (time_begin, time_end, advance) =
425                get_advance(logger, dir, host, port, snapshot, &opts)?;
426            let default = opts.everything || opts.default;
427            let detail = opts.everything || opts.detail;
428            let fields = if let Some(pattern_key) = pattern {
429                parse_pattern(filename, pattern_key, "iface")
430            } else {
431                fields
432            };
433            let fields = expand_fields(
434                match fields.as_ref() {
435                    Some(fields) if !default => fields,
436                    _ => command::DEFAULT_IFACE_FIELDS,
437                },
438                detail,
439            );
440            let iface = iface::Iface::new(&opts, select, fields);
441            let mut output: Box<dyn Write> = match opts.output.as_ref() {
442                Some(file_path) => Box::new(File::create(file_path)?),
443                None => Box::new(io::stdout()),
444            };
445            dump_timeseries(
446                advance,
447                time_begin,
448                time_end,
449                &iface,
450                output.as_mut(),
451                opts.output_format,
452                opts.br,
453                errs,
454            )
455        }
456        DumpCommand::Network {
457            fields,
458            opts,
459            pattern,
460        } => {
461            let (time_begin, time_end, advance) =
462                get_advance(logger, dir, host, port, snapshot, &opts)?;
463            let default = opts.everything || opts.default;
464            let detail = opts.everything || opts.detail;
465            let fields = if let Some(pattern_key) = pattern {
466                parse_pattern(filename, pattern_key, "network")
467            } else {
468                fields
469            };
470            let fields = expand_fields(
471                match fields.as_ref() {
472                    Some(fields) if !default => fields,
473                    _ => command::DEFAULT_NETWORK_FIELDS,
474                },
475                detail,
476            );
477            let network = network::Network::new(&opts, fields);
478            let mut output: Box<dyn Write> = match opts.output.as_ref() {
479                Some(file_path) => Box::new(File::create(file_path)?),
480                None => Box::new(io::stdout()),
481            };
482            dump_timeseries(
483                advance,
484                time_begin,
485                time_end,
486                &network,
487                output.as_mut(),
488                opts.output_format,
489                opts.br,
490                errs,
491            )
492        }
493        DumpCommand::Transport {
494            fields,
495            opts,
496            pattern,
497        } => {
498            let (time_begin, time_end, advance) =
499                get_advance(logger, dir, host, port, snapshot, &opts)?;
500            let default = opts.everything || opts.default;
501            let detail = opts.everything || opts.detail;
502            let fields = if let Some(pattern_key) = pattern {
503                parse_pattern(filename, pattern_key, "transport")
504            } else {
505                fields
506            };
507            let fields = expand_fields(
508                match fields.as_ref() {
509                    Some(fields) if !default => fields,
510                    _ => command::DEFAULT_TRANSPORT_FIELDS,
511                },
512                detail,
513            );
514            let transport = transport::Transport::new(&opts, fields);
515            let mut output: Box<dyn Write> = match opts.output.as_ref() {
516                Some(file_path) => Box::new(File::create(file_path)?),
517                None => Box::new(io::stdout()),
518            };
519            dump_timeseries(
520                advance,
521                time_begin,
522                time_end,
523                &transport,
524                output.as_mut(),
525                opts.output_format,
526                opts.br,
527                errs,
528            )
529        }
530        DumpCommand::EthtoolQueue {
531            fields,
532            opts,
533            pattern,
534        } => {
535            let (time_begin, time_end, advance) =
536                get_advance(logger, dir, host, port, snapshot, &opts)?;
537            let default = opts.everything || opts.default;
538            let detail = opts.everything || opts.detail;
539            let fields = if let Some(pattern_key) = pattern {
540                parse_pattern(filename, pattern_key, "ethtool_queue")
541            } else {
542                fields
543            };
544            let fields = expand_fields(
545                match fields.as_ref() {
546                    Some(fields) if !default => fields,
547                    _ => command::DEFAULT_ETHTOOL_QUEUE_FIELDS,
548                },
549                detail,
550            );
551            let ethtool = ethtool::EthtoolQueue::new(&opts, fields);
552            let mut output: Box<dyn Write> = match opts.output.as_ref() {
553                Some(file_path) => Box::new(File::create(file_path)?),
554                None => Box::new(io::stdout()),
555            };
556            dump_timeseries(
557                advance,
558                time_begin,
559                time_end,
560                &ethtool,
561                output.as_mut(),
562                opts.output_format,
563                opts.br,
564                errs,
565            )
566        }
567        DumpCommand::Tc {
568            fields,
569            opts,
570            pattern,
571        } => {
572            let (time_begin, time_end, advance) =
573                get_advance(logger, dir, host, port, snapshot, &opts)?;
574            let detail = opts.everything || opts.detail;
575            let fields = if let Some(pattern_key) = pattern {
576                parse_pattern(filename, pattern_key, "tc")
577            } else {
578                fields
579            };
580            let fields = expand_fields(
581                match fields.as_ref() {
582                    Some(fields) => fields,
583                    _ => command::DEFAULT_TC_FIELDS,
584                },
585                detail,
586            );
587            let tc = tc::Tc::new(&opts, fields);
588            let mut output: Box<dyn Write> = match opts.output.as_ref() {
589                Some(file_path) => Box::new(File::create(file_path)?),
590                None => Box::new(io::stdout()),
591            };
592            dump_timeseries(
593                advance,
594                time_begin,
595                time_end,
596                &tc,
597                output.as_mut(),
598                opts.output_format,
599                opts.br,
600                errs,
601            )
602        }
603    }
604}
below_dump/lib.rs

below_dump/
lib.rs
