webgraph_cli/to/
bvgraph.rs

1/*
2 * SPDX-FileCopyrightText: 2023 Inria
3 * SPDX-FileCopyrightText: 2023 Tommaso Fontana
4 *
5 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6 */
7
8use crate::create_parent_dir;
9use crate::*;
10use anyhow::Result;
11use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
12use dsi_bitstream::prelude::*;
13
14use mmap_rs::MmapFlags;
15use std::path::PathBuf;
16use tempfile::Builder;
17use webgraph::prelude::*;
18
19#[derive(Parser, Debug)]
20#[command(name = "bvgraph", about = "Recompresses a BvGraph, possibly applying a permutation to its node identifiers.", long_about = None)]
21pub struct CliArgs {
22    /// The basename of the source graph.
23    pub src: PathBuf,
24    /// The basename of the destination graph.
25    pub dst: PathBuf,
26
27    #[clap(flatten)]
28    pub num_threads: NumThreadsArg,
29
30    #[arg(long)]
31    /// The path to an optional permutation in binary big-endian format to be applied to the graph.
32    pub permutation: Option<PathBuf>,
33
34    #[clap(flatten)]
35    pub memory_usage: MemoryUsageArg,
36
37    #[clap(flatten)]
38    pub ca: CompressArgs,
39}
40
41pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {
42    create_parent_dir(&args.dst)?;
43
44    let permutation = if let Some(path) = args.permutation.as_ref() {
45        Some(JavaPermutation::mmap(path, MmapFlags::RANDOM_ACCESS)?)
46    } else {
47        None
48    };
49
50    let target_endianness = args.ca.endianness.clone();
51    match get_endianness(&args.src)?.as_str() {
52        #[cfg(feature = "be_bins")]
53        BE::NAME => compress::<BE>(global_args, args, target_endianness, permutation)?,
54        #[cfg(feature = "le_bins")]
55        LE::NAME => compress::<LE>(global_args, args, target_endianness, permutation)?,
56        e => panic!("Unknown endianness: {}", e),
57    };
58    Ok(())
59}
60
61pub fn compress<E: Endianness>(
62    _global_args: GlobalArgs,
63    args: CliArgs,
64    target_endianness: Option<String>,
65    permutation: Option<JavaPermutation>,
66) -> Result<()>
67where
68    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
69    for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Send + Sync + Clone,
70{
71    let dir = Builder::new().prefix("to_bvgraph_").tempdir()?;
72
73    let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);
74
75    if args.src.with_extension(EF_EXTENSION).exists() {
76        let graph = BvGraph::with_basename(&args.src).endianness::<E>().load()?;
77
78        if let Some(permutation) = permutation {
79            let memory_usage = args.memory_usage.memory_usage;
80
81            log::info!("Permuting graph with memory usage {}", memory_usage);
82            let start = std::time::Instant::now();
83            let sorted = webgraph::transform::permute_split(
84                &graph,
85                &permutation,
86                memory_usage,
87                &thread_pool,
88            )?;
89            log::info!(
90                "Permuted the graph. It took {:.3} seconds",
91                start.elapsed().as_secs_f64()
92            );
93            BvComp::parallel_endianness(
94                args.dst,
95                &sorted,
96                sorted.num_nodes(),
97                args.ca.into(),
98                &thread_pool,
99                dir,
100                &target_endianness.unwrap_or_else(|| E::NAME.into()),
101            )?;
102        } else {
103            BvComp::parallel_endianness(
104                args.dst,
105                &graph,
106                graph.num_nodes(),
107                args.ca.into(),
108                &thread_pool,
109                dir,
110                &target_endianness.unwrap_or_else(|| E::NAME.into()),
111            )?;
112        }
113    } else {
114        log::warn!(
115            "The .ef file does not exist. The graph will be sequentially which will result in slower compression. If you can, run `build_ef` before recompressing."
116        );
117        let seq_graph = BvGraphSeq::with_basename(&args.src)
118            .endianness::<E>()
119            .load()?;
120
121        if let Some(permutation) = permutation {
122            let memory_usage = args.memory_usage.memory_usage;
123
124            log::info!("Permuting graph with memory usage {}", memory_usage);
125            let start = std::time::Instant::now();
126            let permuted = webgraph::transform::permute(&seq_graph, &permutation, memory_usage)?;
127            log::info!(
128                "Permuted the graph. It took {:.3} seconds",
129                start.elapsed().as_secs_f64()
130            );
131
132            BvComp::parallel_endianness(
133                args.dst,
134                &permuted,
135                permuted.num_nodes(),
136                args.ca.into(),
137                &thread_pool,
138                dir,
139                &target_endianness.unwrap_or_else(|| E::NAME.into()),
140            )?;
141        } else {
142            BvComp::parallel_endianness(
143                args.dst,
144                &seq_graph,
145                seq_graph.num_nodes(),
146                args.ca.into(),
147                &thread_pool,
148                dir,
149                &target_endianness.unwrap_or_else(|| E::NAME.into()),
150            )?;
151        }
152    }
153    Ok(())
154}
webgraph_cli/to/bvgraph.rs

webgraph_cli/to/
bvgraph.rs
