xet-data 1.5.2

Data processing pipeline for chunking, deduplication, and file reconstruction; used in the Hugging Face Xet client tools. Intended to be used through the API in the hf-xet package.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

use std::io::Write;
use std::path::Path;
use std::process::Command;

use anyhow::Result;
use tempfile::TempDir;
use tracing::info;

/// Set this to true to see the output of the tests on success.
const DEBUG: bool = false;

struct IntegrationTest {
    test_script: String,
    arguments: Vec<String>,
    assets: Vec<(String, &'static [u8])>,
}

impl IntegrationTest {
    fn new(test_script: &str) -> Self {
        Self {
            test_script: test_script.to_owned(),
            arguments: Vec::new(),
            assets: Vec::new(),
        }
    }

    #[allow(unused)]
    fn add_arguments(&mut self, args: &[&str]) {
        self.arguments.extend(args.iter().map(|s| s.to_string()))
    }

    #[allow(unused)]
    fn add_asset(&mut self, name: &str, arg: &'static [u8]) {
        self.assets.push((name.to_owned(), arg));
    }

    fn run(&self) -> Result<()> {
        // Create a temporary directory
        let tmp_repo_dest = TempDir::new().unwrap();
        let tmp_path_path = tmp_repo_dest.path().to_path_buf();

        std::fs::write(tmp_path_path.join("test_script.sh"), &self.test_script).unwrap();

        std::fs::write(tmp_path_path.join("initialize.sh"), include_str!("integration_tests/initialize.sh")).unwrap();

        // Write the assets into the tmp path
        for (name, data) in self.assets.iter() {
            std::fs::write(tmp_path_path.join(name), data)?;
        }

        let mut cmd = Command::new("bash");
        cmd.args(["-e", "-x", "test_script.sh"]);
        cmd.args(&self.arguments[..]);
        cmd.current_dir(tmp_path_path.clone());

        // Add in the path of the test bin executable

        let test_bin_path = env!("CARGO_BIN_EXE_x");
        let buildpath = Path::new(&test_bin_path).parent().unwrap();
        info!("Adding {:?} to path.", &buildpath);
        cmd.env("PATH", format!("{}:{}", &buildpath.to_str().unwrap(), &std::env::var("PATH").unwrap()));

        // Now, to prevent ~/.gitconfig to be read, we need to reset the home directory; otherwise
        // these tests will not be run in an isolated environment.
        //
        // NOTE: this is not a problem with git version 2.32 or later.  There, GIT_CONFIG_GLOBAL
        // works and the scripts take advantage of it.  However, outside of that, this is needed
        // to avoid issues with a lesser git.
        cmd.env("HOME", tmp_path_path.as_os_str());

        // Now, run the script.
        let out = cmd.output()?;
        let status = out.status;

        if status.success() {
            if DEBUG {
                // Just dump things to the output
                eprintln!("Test succeeded, STDOUT:");
                std::io::stdout().write_all(&out.stdout).unwrap();
                eprintln!("STDERR:");
                std::io::stderr().write_all(&out.stderr).unwrap();
            }
            Ok(())
        } else {
            eprintln!("Test failed, STDOUT:");
            std::io::stderr().write_all(&out.stderr).unwrap();
            // Parse output for error string:
            let stderr_out = std::str::from_utf8(&out.stderr)?;

            eprintln!("STDERR:\n{}", &stderr_out);

            let error_re = regex::Regex::new("ERROR:>>>>>(.*)<<<<<").unwrap();

            let captures = error_re.captures(stderr_out);

            if let Some(captured_text) = captures {
                Err(std::io::Error::new(
                    std::io::ErrorKind::Other,
                    format!("Test failed: {}", captured_text.get(1).unwrap().as_str()),
                )
                .into())
            } else {
                Err(std::io::Error::new(std::io::ErrorKind::Other, "Test failed: Unknown Error.").into())
            }
        }
    }
}

#[cfg(all(test, unix))]
mod git_integration_tests {
    use anyhow::Result;

    use super::*;
    #[test]
    #[cfg_attr(feature = "smoke-test", ignore)]
    fn test_basic_read() -> Result<()> {
        IntegrationTest::new(include_str!("integration_tests/test_basic_clean_smudge.sh")).run()
    }
}