webgraph_cli/to/
bvgraph.rs

1/*
2 * SPDX-FileCopyrightText: 2023 Inria
3 * SPDX-FileCopyrightText: 2023 Tommaso Fontana
4 *
5 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6 */
7
8use crate::create_parent_dir;
9use crate::*;
10use anyhow::Result;
11use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
12use dsi_bitstream::prelude::*;
13
14use mmap_rs::MmapFlags;
15use std::path::PathBuf;
16use tempfile::Builder;
17use webgraph::prelude::*;
18
19#[derive(Parser, Debug)]
20#[command(name = "bvgraph", about = "Recompresses a BvGraph, possibly applying a permutation to its node identifiers.", long_about = None)]
21pub struct CliArgs {
22    /// The basename of the source graph.
23    pub src: PathBuf,
24    /// The basename of the destination graph.
25    pub dst: PathBuf,
26
27    #[clap(flatten)]
28    pub num_threads: NumThreadsArg,
29
30    #[arg(long)]
31    /// The path to an optional permutation in binary big-endian format to be applied to the graph.
32    pub permutation: Option<PathBuf>,
33
34    #[clap(flatten)]
35    pub memory_usage: MemoryUsageArg,
36
37    #[clap(flatten)]
38    pub ca: CompressArgs,
39}
40
41pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {
42    create_parent_dir(&args.dst)?;
43
44    let permutation = if let Some(path) = args.permutation.as_ref() {
45        Some(JavaPermutation::mmap(path, MmapFlags::RANDOM_ACCESS)?)
46    } else {
47        None
48    };
49
50    let target_endianness = args.ca.endianness.clone();
51    match get_endianness(&args.src)?.as_str() {
52        #[cfg(feature = "be_bins")]
53        BE::NAME => compress::<BE>(global_args, args, target_endianness, permutation),
54        #[cfg(feature = "le_bins")]
55        LE::NAME => compress::<LE>(global_args, args, target_endianness, permutation),
56        e => panic!("Unknown endianness: {}", e),
57    }
58}
59
60pub fn compress<E: Endianness>(
61    _global_args: GlobalArgs,
62    args: CliArgs,
63    target_endianness: Option<String>,
64    permutation: Option<JavaPermutation>,
65) -> Result<()>
66where
67    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
68    for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Send + Sync + Clone,
69{
70    let dir = Builder::new().prefix("to_bvgraph_").tempdir()?;
71
72    let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);
73    let chunk_size = args.ca.chunk_size;
74    let bvgraphz = args.ca.bvgraphz;
75    let mut builder = BvCompConfig::new(&args.dst)
76        .with_comp_flags(args.ca.into())
77        .with_tmp_dir(&dir);
78
79    if bvgraphz {
80        builder = builder.with_chunk_size(chunk_size);
81    }
82
83    if args.src.with_extension(EF_EXTENSION).exists() {
84        let graph = BvGraph::with_basename(&args.src).endianness::<E>().load()?;
85
86        if let Some(permutation) = permutation {
87            let memory_usage = args.memory_usage.memory_usage;
88            thread_pool.install(|| {
89                log::info!("Permuting graph with memory usage {}", memory_usage);
90                let start = std::time::Instant::now();
91                let sorted =
92                    webgraph::transform::permute_split(&graph, &permutation, memory_usage)?;
93                log::info!(
94                    "Permuted the graph. It took {:.3} seconds",
95                    start.elapsed().as_secs_f64()
96                );
97                builder.par_comp_lenders_endianness(
98                    &sorted,
99                    sorted.num_nodes(),
100                    &target_endianness.unwrap_or_else(|| BE::NAME.into()),
101                )
102            })?;
103        } else {
104            thread_pool.install(|| {
105                builder.par_comp_lenders_endianness(
106                    &graph,
107                    graph.num_nodes(),
108                    &target_endianness.unwrap_or_else(|| BE::NAME.into()),
109                )
110            })?;
111        }
112    } else {
113        log::warn!(
114            "The .ef file does not exist. The graph will be read sequentially which will result in slower compression. If you can, run `webgraph build ef` before recompressing."
115        );
116        let seq_graph = BvGraphSeq::with_basename(&args.src)
117            .endianness::<E>()
118            .load()?;
119
120        if let Some(permutation) = permutation {
121            let memory_usage = args.memory_usage.memory_usage;
122
123            log::info!("Permuting graph with memory usage {}", memory_usage);
124            let start = std::time::Instant::now();
125            let permuted = webgraph::transform::permute(&seq_graph, &permutation, memory_usage)?;
126            log::info!(
127                "Permuted the graph. It took {:.3} seconds",
128                start.elapsed().as_secs_f64()
129            );
130
131            thread_pool.install(|| {
132                builder.par_comp_lenders_endianness(
133                    &permuted,
134                    permuted.num_nodes(),
135                    &target_endianness.unwrap_or_else(|| BE::NAME.into()),
136                )
137            })?;
138        } else {
139            thread_pool.install(|| {
140                builder.par_comp_lenders_endianness(
141                    &seq_graph,
142                    seq_graph.num_nodes(),
143                    &target_endianness.unwrap_or_else(|| BE::NAME.into()),
144                )
145            })?;
146        }
147    }
148    Ok(())
149}
webgraph_cli/to/bvgraph.rs

webgraph_cli/to/
bvgraph.rs
