hdfs-native 0.13.5

Native HDFS client implementation in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

#[cfg(feature = "integration-test")]
mod common;

#[cfg(feature = "integration-test")]
mod test {
    use hdfs_native::{
        Client, ClientBuilder, Result, WriteOptions,
        minidfs::{DfsFeatures, MiniDfs},
    };
    use serial_test::serial;
    use std::collections::HashSet;

    async fn touch(client: &Client, path: &str) {
        client
            .create(path, WriteOptions::default())
            .await
            .unwrap()
            .close()
            .await
            .unwrap();
    }

    #[tokio::test]
    #[serial]
    async fn test_viewfs() {
        let features = HashSet::from([DfsFeatures::ViewFS]);
        let _ = env_logger::builder().is_test(true).try_init();

        // VIEWFS feature creates one mount with a fallback
        // /mount1 resolves to hdfs://ns0/nested
        // fallback resolves to hdfs://ns1/nested
        let _dfs = MiniDfs::with_features(&features);
        let viewfs = Client::default();
        let hdfs1 = ClientBuilder::new().with_url("hdfs://ns0").build().unwrap();
        let hdfs2 = ClientBuilder::new().with_url("hdfs://ns1").build().unwrap();

        touch(&viewfs, "/mount1/file1").await;
        touch(&viewfs, "/root/file2").await;

        test_file_info(&viewfs, &hdfs1, &hdfs2).await.unwrap();
        test_listing(&viewfs).await.unwrap();
        test_rename(&viewfs, &hdfs1, &hdfs2).await.unwrap();
    }

    async fn test_file_info(viewfs: &Client, hdfs1: &Client, hdfs2: &Client) -> Result<()> {
        assert_eq!(
            viewfs.get_file_info("/mount1/file1").await?.path,
            "/mount1/file1"
        );
        assert_eq!(
            viewfs.get_file_info("/root/file2").await?.path,
            "/root/file2"
        );
        assert_eq!(
            hdfs1.get_file_info("/nested/file1").await?.path,
            "/nested/file1"
        );
        assert_eq!(
            hdfs2.get_file_info("/nested/root/file2").await?.path,
            "/nested/root/file2"
        );

        Ok(())
    }

    async fn test_listing(viewfs: &Client) -> Result<()> {
        let mut statuses = viewfs.list_status("/", true).await?;
        assert_eq!(statuses.len(), 2);
        statuses.sort_by_key(|x| x.path.clone());

        // TODO: Should /mount1 show up here if it doesn't exist in the fallback system?
        assert_eq!(statuses[0].path, "/root");
        assert_eq!(statuses[1].path, "/root/file2");

        let statuses = viewfs.list_status("/mount1", true).await?;
        assert_eq!(statuses.len(), 1);
        assert_eq!(statuses[0].path, "/mount1/file1");
        Ok(())
    }

    async fn test_rename(viewfs: &Client, hdfs1: &Client, hdfs2: &Client) -> Result<()> {
        viewfs
            .rename("/mount1/file1", "/mount1/file3", false)
            .await
            .unwrap();
        assert!(hdfs1.get_file_info("/nested/file3").await.is_ok());
        assert!(hdfs1.get_file_info("/nested/file1").await.is_err());
        viewfs
            .rename("/mount1/file3", "/mount1/file1", false)
            .await
            .unwrap();

        viewfs
            .rename("/root/file2", "/root/file4", false)
            .await
            .unwrap();
        assert!(hdfs2.get_file_info("/nested/root/file4").await.is_ok());
        assert!(hdfs2.get_file_info("/nested/root/file2").await.is_err());
        viewfs
            .rename("/root/file4", "/root/file3", false)
            .await
            .unwrap();

        assert!(
            viewfs
                .rename("/mount1/file1", "/root/file1", false)
                .await
                .is_err()
        );

        Ok(())
    }
}