seekstorm 3.0.0

Search engine library & multi-tenancy server
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168

#![crate_name = "seekstorm_server"]
#![doc(html_logo_url = "http://seekstorm.com/assets/logo.svg")]

//! # `seekstorm_server`
//! **SeekStorm** is an open-source, sub-millisecond full-text search library & multi-tenancy server implemented in Rust.<br>
//! **SeekStorm server** is a standalone search server to be accessed via HTTP.
//! * it is both accessible via RESTful API endpoints and via embedded web UI
//! * supports multi-tenancy: multiple users, each with multiple indices
//! * API-key management
//! * rate-limiting
//! ### Command line parameters
//! ```rust
//! * index_path   (default = "/seekstorm_index" in current directory)
//! * local_ip     (default = 0.0.0.0)
//! * local_port   (default = 80)
//! ./seekstorm_server.exe local_ip="127.0.0.1" local_port=80 index_path="c:/seekstorm_index"
//! ```
//! &#x26A0; **WARNING**: make sure to set the MASTER_KEY_SECRET environment variable to a secret,
//! otherwise your generated API keys will be compromised.
//! ### Console commands
//! ```
//! quit to exit
//! ```
//! ### REST API endpoints
//! Use VSC extension "Rest client" to execute API calls, inspect responses and generate code snippets in your language:  
//! [**interactive API endpoint examples**](https://github.com/SeekStorm/SeekStorm/blob/master/src/seekstorm_server/test_api.rest)
//! ### create api key
//! ```
//! curl --request POST --url http://127.0.0.1:80/api/v1/apikey --header 'apikey: A6xnQhbz4Vx2HuGl4lXwZ5U2I8iziLRFnhP5eNfIRvQ=' --header 'content-type: application/json' --data '{"indices_max": 10,"indices_size_max":100000,"documents_max":10000000,"operations_max":10000000,"rate_limit": 100000}'
//! ```
//! ### delete api key
//! ```
//! curl --request DELETE --url http://127.0.0.1/api/v1/apikey --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json' --header 'user-agent: vscode-restclient'
//! ```
//! ---
//! ### create index
//! ```
//! curl --request POST --url http://127.0.0.1:80/api/v1/index --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json' --data '{"schema":[{"field_type": "Text","store": true,"field": "title","index_lexical": true,"boost":10.0},{"field_type": "Text","store": true,"field": "body","index_lexical": true},{"field_type": "String","store": true,"field": "url","index_lexical": false}],"index_name": "test_index","similarity": "Bm25fProximity","tokenizer": "UnicodeAlphanumeric"}'
//! ```
//! ### get index
//! ```
//! curl --request GET --url http://127.0.0.1/api/v1/index/0 --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json'
//! ```
//! ### delete index
//! ```
//! curl --request DELETE --url http://127.0.0.1/api/v1/index/0 --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json'
//! ```
//! ### commit index
//! ```
//! curl --request PATCH --url http://127.0.0.1/api/v1/index/0 --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json'
//! ```
//! ---
//! ### index document(s)
//! single document
//! ```
//! curl --request POST --url http://127.0.0.1:80/api/v1/index/0/doc --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json' --data '{"title":"title1 test","body":"body1","url":"url1"}'
//! ```
//! multiple documents
//! ```
//! curl --request POST --url http://127.0.0.1:80/api/v1/index/0/doc --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json' --data '[{"title":"title2","body":"body2 test","url":"url2"},{"title":"title3 test","body":"body3 test","url":"url3"}]'
//! ```
//! ### delete document(s)
//! single document (by document id in URL parameter)
//! ```
//! curl --request DELETE --url http://127.0.0.1/api/v1/index/0/doc/0 --header 'apikey: VF9vcy0U1l19VQWYEKD+eeyX+RxuOLsj19EPuIWGc6A=' --header 'content-type: application/json'
//! ```
//! single document (by document id in JSON parameter)
//! ```
//! curl --request DELETE --url http://127.0.0.1/api/v1/index/0/doc --header 'apikey: VF9vcy0U1l19VQWYEKD+eeyX+RxuOLsj19EPuIWGc6A=' --header 'content-type: application/json' --data 0
//! ```
//! multiple documents (by document id's in JSON parameter)
//! ```
//! curl --request DELETE --url http://127.0.0.1/api/v1/index/0/doc --header 'apikey: VF9vcy0U1l19VQWYEKD+eeyX+RxuOLsj19EPuIWGc6A=' --header 'content-type: application/json' --data '[0,1]'
//! ```
//! multiple documents (by query in JSON parameter)
//! ```
//! curl --request DELETE --url http://127.0.0.1/api/v1/index/0/doc --header 'apikey: VF9vcy0U1l19VQWYEKD+eeyX+RxuOLsj19EPuIWGc6A=' --header 'content-type: application/json' --data '{"query":"test","offset":0,"length":10,"realtime": true,"field_filter": ["title", "body"]}'
//! ```
//! ### update document(s)
//! single document
//! ```
//! curl --request PATCH --url http://127.0.0.1/api/v1/index/0/doc --header 'apikey: VF9vcy0U1l19VQWYEKD+eeyX+RxuOLsj19EPuIWGc6A=' --header 'content-type: application/json' --data '[0,{"title":"title1 test","body":"body1","url":"url1"}]'
//! ```
//! multiple documents
//! ```
//! curl --request PATCH --url http://127.0.0.1/api/v1/index/0/doc --header 'apikey: VF9vcy0U1l19VQWYEKD+eeyX+RxuOLsj19EPuIWGc6A=' --header 'content-type: application/json' --data '[[1,{"title":"title1 test","body":"body1","url":"url1"}],[2,{"title":"title3 test","body":"body3 test","url":"url3"}]]'
//! ```
//! ### get document
//! without highlight
//! ```
//! curl --request GET --url http://127.0.0.1/api/v1/index/0/doc/0 --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json'
//! ```
//! with highlight
//! ```
//! curl --request GET --url http://127.0.0.1/api/v1/index/0/doc/0 --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json' --data '{"query_terms": ["test"],"fields": ["title", "body"],"highlights": [{ "field": "title", "fragment_number": 0, "fragment_size": 1000, "highlight_markup": true},{ "field": "body", "fragment_number": 2, "fragment_size": 160, "highlight_markup": true},{ "field": "body", "name": "body2", "fragment_number": 0, "fragment_size": 4000, "highlight_markup": true}]}'
//! ```
//! ---
//! ### query index (GET)
//!  with URL parameter
//! ```
//! curl --request GET --url 'http://127.0.0.1/api/v1/index/0/query?query=test&offset=0&length=10' --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json'
//! ```
//! with JSON parameter
//! ```
//! curl --request GET --url http://127.0.0.1/api/v1/index/0/query --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json' --data '{"query":"test","offset":0,"length":10,"realtime": true,"field_filter": ["title", "body"]}'
//! ```
//! ### query index (POST)
//! ```
//! curl --request POST --url http://127.0.0.1/api/v1/index/0/query --header 'apikey: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' --header 'content-type: application/json' --data '{"query":"test","offset":0,"length":10,"realtime": true,"field_filter": ["title", "body"]}'
//! ```
//! ---
//! ## Open embedded Web UI in browser
//! <a href="http://127.0.0.1">http://127.0.0.1</a>
//! To use the embedded Web UI for a selected index you need to change the API_KEY and index_id (in QUERY_URL) in master.js
//! **before** building the seekstorm_server (html/css/js are embedded ressources).

use std::collections::HashMap;
use std::env;
use std::error::Error;
use std::str;
use std::sync::LazyLock;
#[cfg(target_os = "windows")]
use winapi::um::{
    processthreadsapi::{GetCurrentProcess, SetPriorityClass},
    winbase::ABOVE_NORMAL_PRIORITY_CLASS,
};

use crate::server::initialize;

/// &#x26A0; **WARNING**: make sure to set the MASTER_KEY_SECRET environment variable to a secret, otherwise your generated API keys will be compromised.
#[doc(hidden)]
pub static MASTER_KEY_SECRET: LazyLock<String> =
    LazyLock::new(|| env::var("MASTER_KEY_SECRET").unwrap_or("1234".to_string()));

#[doc(hidden)]
mod api_endpoints;
#[doc(hidden)]
mod http_server;
#[doc(hidden)]
mod multi_tenancy;
#[doc(hidden)]
mod server;
#[doc(hidden)]
pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
#[doc(hidden)]
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
    #[cfg(target_os = "windows")]
    unsafe {
        let process = GetCurrentProcess();
        SetPriorityClass(process, ABOVE_NORMAL_PRIORITY_CLASS);
    }

    let args: Vec<String> = env::args().collect();
    let mut params = HashMap::new();
    if args.len() > 1 {
        for s in args {
            let split: Vec<&str> = s.split('=').collect();
            if split.len() == 2 {
                params.insert(split[0].trim().to_owned(), split[1].trim().to_owned());
            }
        }
    }

    initialize(params).await;

    Ok(())
}