quickwit-core 0.3.0

Core crate of quickwit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

// Copyright (C) 2021 Quickwit, Inc.
//
// Quickwit is offered under the AGPL v3.0 and as commercial software.
// For commercial licensing, contact us at hello@quickwit.io.
//
// AGPL:
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

#![allow(clippy::bool_assert_comparison)]

//! `quickwit-core` provides IndexService that let you manage your indexes:
//! - `create` for creating a new index;
//! - `get_index` for getting an index;
//! - `get_indexes` for getting all indexes registered in the metastore;
//! - `reset_index` for indexing new-line delimited json documents;
//! - `delete_index` for deleting an index;
//! - `garbage_collect_index` for garbage collecting dangling files.

mod index;

pub use index::{clear_cache_directory, get_cache_directory_path, IndexService, IndexServiceError};

#[cfg(test)]
mod tests {
    use std::path::Path;

    use quickwit_common::uri::Uri;
    use quickwit_config::{IndexConfig, IndexingSettings, SearchSettings};
    use quickwit_indexing::{FileEntry, TestSandbox};
    use quickwit_metastore::quickwit_metastore_uri_resolver;
    use quickwit_storage::StorageUriResolver;

    use crate::IndexService;

    const METASTORE_URI: &str = "ram://quickwit-test-indexes";

    #[tokio::test]
    async fn test_file_entry_from_split_and_index_delete() -> anyhow::Result<()> {
        quickwit_common::setup_logging_for_tests();
        let index_id = "test-index";
        let doc_mapping_yaml = r#"
            field_mappings:
              - name: title
                type: text
              - name: body
                type: text
              - name: url
                type: text
        "#;
        let test_sandbox =
            TestSandbox::create(index_id, doc_mapping_yaml, "{}", &["title", "body"]).await?;
        test_sandbox.add_documents(vec![
            serde_json::json!({"title": "snoopy", "body": "Snoopy is an anthropomorphic beagle[5] in the comic strip...", "url": "http://snoopy"}),
        ]).await?;
        let splits = test_sandbox
            .metastore()
            .list_all_splits(index_id)
            .await?
            .into_iter()
            .map(|metadata| metadata.split_metadata)
            .collect::<Vec<_>>();
        let file_entries: Vec<FileEntry> = splits.iter().map(FileEntry::from).collect();
        assert_eq!(file_entries.len(), 1);
        for file_entry in file_entries {
            let split_num_bytes = test_sandbox
                .storage()
                .file_num_bytes(Path::new(file_entry.file_name.as_str()))
                .await?;
            assert_eq!(split_num_bytes, file_entry.file_size_in_bytes);
        }
        // Now delete the index.
        let index_service = IndexService::new(
            test_sandbox.metastore(),
            StorageUriResolver::for_test(),
            Uri::new("file:///quickwit".to_string()),
        );
        let deleted_file_entries = index_service.delete_index(index_id, false).await?;
        assert_eq!(deleted_file_entries.len(), 1);
        Ok(())
    }

    #[tokio::test]
    async fn test_create_index_without_index_uri() -> anyhow::Result<()> {
        quickwit_common::setup_logging_for_tests();
        let index_id = "test-index-no-index-uri";
        let doc_mapping_yaml = r#"
            field_mappings:
              - name: title
                type: text
        "#;
        let index_config = IndexConfig {
            version: 0,
            index_id: index_id.to_string(),
            index_uri: None,
            doc_mapping: serde_yaml::from_str(doc_mapping_yaml)?,
            indexing_settings: IndexingSettings::default(),
            search_settings: SearchSettings::default(),
            sources: Vec::new(),
        };
        let metastore = quickwit_metastore_uri_resolver()
            .resolve(METASTORE_URI)
            .await?;
        let storage_resolver = StorageUriResolver::for_test();
        let index_service = IndexService::new(
            metastore,
            storage_resolver,
            Uri::new("ram://test-storage-indexes".to_string()),
        );
        let index_metadata = index_service.create_index(index_config).await?;
        assert_eq!(index_metadata.index_id, index_id);
        assert_eq!(
            index_metadata.index_uri,
            "ram://test-storage-indexes/test-index-no-index-uri"
        );
        Ok(())
    }
}