assemblyline_models/datastore/
service.rs

1use std::collections::HashMap;
2
3use serde::{Serialize, Deserialize};
4use serde_with::{SerializeDisplay, DeserializeFromStr};
5use struct_metadata::Described;
6
7use crate::types::classification::{unrestricted_classification, unrestricted_classification_string};
8use crate::types::{ClassificationString, JsonMap, NonZeroInteger, ServiceName, Text};
9use crate::{ElasticMeta, Readable};
10
11/// Environment Variable Model
12#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
13#[metadata_type(ElasticMeta)]
14#[metadata(index=false, store=false)]
15pub struct EnvironmentVariable {
16    /// Name of Environment Variable
17    pub name: String,
18    /// Value of Environment Variable
19    pub value: String,
20}
21
22
23/// Docker Container Configuration
24#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Debug)]
25#[metadata_type(ElasticMeta)]
26#[metadata(index=false, store=false)]
27pub struct DockerConfig {
28    /// Does the container have internet-access?
29    #[serde(default)]
30    pub allow_internet_access: bool,
31    /// Command to run when container starts up.
32    #[serde(default)]
33    pub command: Option<Vec<String>>,
34    /// CPU allocation
35    #[serde(default="default_cpu_cores")]
36    pub cpu_cores: f32,
37    /// Additional environemnt variables for the container
38    #[serde(default)]
39    pub environment: Vec<EnvironmentVariable>,
40    /// Complete name of the Docker image with tag, may include registry
41    pub image: String,
42    /// The username to use when pulling the image
43    #[serde(default)]
44    pub registry_username: Option<String>,
45    /// The password or token to use when pulling the image
46    #[serde(default)]
47    pub registry_password: Option<String>,
48    /// The type of container registry
49    #[serde(default="default_registry_type")]
50    pub registry_type: RegistryType,
51    /// What ports of container to expose?
52    #[serde(default)]
53    pub ports: Vec<String>,
54    /// Container RAM limit
55    #[serde(default="default_ram_mb")]
56    pub ram_mb: i32,
57    /// Container RAM request
58    #[serde(default="default_ram_mb_min")]
59    pub ram_mb_min: i32,
60    /// Service account to use for pods in kubernetes
61    #[serde(default)]
62    pub service_account: Option<String>,
63    /// Additional container labels.
64    #[serde(default)]
65    pub labels: Vec<EnvironmentVariable>,
66}
67
68fn default_cpu_cores() -> f32 { 1.0 }
69fn default_registry_type() -> RegistryType { RegistryType::Docker }
70fn default_ram_mb() -> i32 { 512 }
71fn default_ram_mb_min() -> i32 { 256 }
72
73#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
74#[metadata_type(ElasticMeta)]
75#[strum(serialize_all = "lowercase")]
76pub enum RegistryType {
77    Docker,
78    Harbor,
79}
80
81/// Container's Persistent Volume Configuration
82#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
83#[metadata_type(ElasticMeta)]
84#[metadata(index=false, store=false)]
85pub struct PersistentVolume {
86    /// Path into the container to mount volume
87    pub mount_path: String,
88    /// The amount of storage allocated for volume
89    pub capacity: String,
90    /// Storage class used to create volume
91    pub storage_class: String,
92    /// Access mode for volume
93    #[serde(default="default_access_mode")]
94    pub access_mode: AccessMode,
95}
96
97fn default_access_mode() -> AccessMode { AccessMode::ReadWriteOnce }
98
99#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
100#[metadata_type(ElasticMeta)]
101pub enum AccessMode {
102    ReadWriteOnce, ReadWriteMany
103}
104
105/// Container's Dependency Configuration
106#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Debug)]
107#[metadata_type(ElasticMeta)]
108#[metadata(index=false, store=false)]
109pub struct DependencyConfig {
110    /// Docker container configuration for dependency
111    pub container: DockerConfig,
112    /// Volume configuration for dependency
113    #[serde(default)]
114    pub volumes: HashMap<String, PersistentVolume>,
115    /// Should this dependency run as other core components?
116    #[serde(default)]
117    pub run_as_core: bool,
118}
119
120#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default, Described)]
121#[metadata_type(ElasticMeta)]
122#[serde(rename_all="UPPERCASE")]
123pub enum FetchMethods {
124    #[default]
125    Get,
126    Post,
127    Git,
128}
129
130
131/// Update Source Configuration
132#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
133#[metadata_type(ElasticMeta)]
134#[metadata(index=false, store=false)]
135pub struct UpdateSource {
136    /// Is this source active for periodic fetching?
137    #[serde(default="default_enabled")]
138    pub enabled: bool,
139    /// Name of source
140    pub name: String,
141    /// Password used to authenticate with source
142    #[serde(default)]
143    pub password: Option<String>,
144    /// Pattern used to find files of interest from source
145    #[serde(default)]
146    pub pattern: Option<String>,
147    /// Private key used to authenticate with source
148    #[serde(default)]
149    pub private_key: Option<String>,
150    /// CA cert for source
151    #[serde(default)]
152    pub ca_cert: Option<String>,
153    /// Ignore SSL errors when reaching out to source?
154    #[serde(default)]
155    pub ssl_ignore_errors: bool,
156    /// Proxy server for source
157    #[serde(default)]
158    pub proxy: Option<String>,
159    /// URI to source
160    pub uri: String,
161    /// Username used to authenticate with source
162    #[serde(default)]
163    pub username: Option<String>,
164    /// Headers
165    #[serde(default)]
166    pub headers: Vec<EnvironmentVariable>,
167    /// Default classification used in absence of one defined in files from source
168    #[serde(default="unrestricted_classification_string")]
169    pub default_classification: ClassificationString,
170    /// Use managed identity for authentication with Azure DevOps
171    #[serde(default)]
172    pub use_managed_identity: bool,
173    /// Branch to checkout from Git repository.
174    #[serde(default)]
175    pub git_branch: Option<String>,
176    /// Synchronize signatures with remote source. Allows system to auto-disable signatures no longer found in source.
177    #[serde(default)]
178    pub sync: bool,
179    /// Fetch method to be used with source
180    #[serde(default)]
181    pub fetch_method: FetchMethods,
182    /// Should the source's classfication override the signature's self-defined classification, if any?
183    #[serde(default)]
184    pub override_classification: bool,
185    /// Processing configuration for source
186    #[serde(default)]
187    pub configuration: HashMap<String, serde_json::Value>,
188    /// Data that's sent in a POST request (`fetch_method="POST"`)
189    #[serde(default)]
190    pub data: Option<Text>,
191    /// Update check interval, in seconds, for this source
192    #[serde(default)]
193    #[metadata(mapping="integer")]
194    pub update_interval: Option<NonZeroInteger>,
195    /// Ignore source caching and forcefully fetch from source
196    #[serde(default)]
197    pub ignore_cache: bool,
198}
199
200fn default_enabled() -> bool { true }
201
202/// Update Configuration for Signatures
203#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
204#[metadata_type(ElasticMeta)]
205#[metadata(index=false, store=false)]
206pub struct UpdateConfig {
207    /// Does the updater produce signatures?
208    #[metadata(index=true)]
209    #[serde(default)]
210    pub generates_signatures: bool,
211    /// List of external sources
212    #[serde(default)]
213    pub sources: Vec<UpdateSource>,
214    /// Update check interval, in seconds
215    pub update_interval_seconds: i32,
216    /// Should the service wait for updates first?
217    #[serde(default)]
218    pub wait_for_update: bool,
219    /// Delimiter used when given a list of signatures
220    #[serde(default="default_signature_delimiter")]
221    pub signature_delimiter: SignatureDelimiter,
222    /// Custom delimiter definition
223    #[serde(default)]
224    pub custom_delimiter: Option<String>,
225    /// Default pattern used for matching files
226    #[serde(default="default_default_pattern")]
227    pub default_pattern: Text,
228}
229
230fn default_signature_delimiter() -> SignatureDelimiter { SignatureDelimiter::DoubleNewLine }
231fn default_default_pattern() -> Text { ".*".into() }
232
233#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
234#[metadata_type(ElasticMeta)]
235#[strum(serialize_all = "snake_case")]
236pub enum SignatureDelimiter {
237    NewLine,
238    DoubleNewLine,
239    Pipe,
240    Comma,
241    Space,
242    None,
243    File,
244    Custom,
245}
246
247impl SignatureDelimiter {
248    pub fn token(&self) -> String {
249        match self {
250            SignatureDelimiter::NewLine => "\n".to_owned(),
251            SignatureDelimiter::DoubleNewLine => "\n\n".to_owned(),
252            SignatureDelimiter::Pipe => "|".to_owned(),
253            SignatureDelimiter::Comma => ",".to_owned(),
254            SignatureDelimiter::Space => " ".to_owned(),
255            SignatureDelimiter::None => "".to_owned(),
256            SignatureDelimiter::File => "".to_owned(),
257            SignatureDelimiter::Custom => "".to_owned(),
258        }
259    }
260}
261
262/// Submission Parameters for Service
263#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
264#[metadata_type(ElasticMeta)]
265#[metadata(index=false, store=false)]
266pub struct SubmissionParams {
267    /// Default value (must match value in `value` field)
268    pub default: serde_json::Value,
269    /// Name of parameter
270    pub name: String,
271    /// Type of parameter
272    #[serde(rename="type")]
273    pub param_type: ParamKinds,
274    /// Default value (must match value in `default` field)
275    pub value: serde_json::Value,
276    /// List of values if `type: list`
277    #[serde(default)]
278    pub list: Vec<serde_json::Value>,
279    /// Should this parameter be hidden?
280    #[serde(default)]
281    pub hide: bool,
282}
283
284#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
285#[metadata_type(ElasticMeta)]
286#[strum(serialize_all = "lowercase")]
287pub enum ParamKinds {
288    Str,
289    Int,
290    List,
291    Bool,
292}
293
294/// Service Configuration
295#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Debug)]
296#[metadata_type(ElasticMeta)]
297#[metadata(index=true, store=false)]
298pub struct Service {
299    /// Regex to accept files as identified by Assemblyline
300    /// Regexes applied to assemblyline style file type string
301    #[metadata(store=true)]
302    #[serde(default="default_service_accepts")]
303    pub accepts: String,
304    /// Regex to reject files as identified by Assemblyline
305    /// Regexes applied to assemblyline style file type string
306    #[metadata(store=true)]
307    #[serde(default="default_service_rejects")]
308    pub rejects: Option<String>,
309    /// Should the service be auto-updated?
310    #[serde(default)]
311    pub auto_update: Option<bool>,
312    /// Which category does this service belong to?
313    #[metadata(store=true, copyto="__text__")]
314    #[serde(default="default_category")]
315    pub category: ServiceName,
316    /// Classification of the service
317    #[serde(default="unrestricted_classification")]
318    pub classification: String,
319    /// Service Configuration
320    #[metadata(index=false, store=false)]
321    #[serde(default)]
322    pub config: JsonMap,
323    /// Description of service
324    #[metadata(store=true, copyto="__text__")]
325    #[serde(default="default_description")]
326    pub description: Text,
327    /// Default classification assigned to service results
328    #[serde(default="unrestricted_classification")]
329    pub default_result_classification: String,
330    /// Is the service enabled
331    #[metadata(store=true)]
332    #[serde(default)]
333    pub enabled: bool,
334    /// Does this service perform analysis outside of Assemblyline?
335    #[serde(default)]
336    pub is_external: bool,
337    /// How many licences is the service allowed to use?
338    #[serde(default)]
339    #[metadata(mapping="integer")]
340    pub licence_count: u32,
341    /// The minimum number of service instances. Overrides Scaler's min_instances configuration.
342    #[serde(default)]
343    #[metadata(mapping="integer")]
344    pub min_instances: Option<u32>,
345    /// If more than this many jobs are queued for this service drop those over this limit. 0 is unlimited.
346    #[serde(default)]
347    #[metadata(mapping="integer")]
348    pub max_queue_length: u32,
349
350    /// Does this service use tags from other services for analysis?
351    #[serde(default)]
352    pub uses_tags: bool,
353    /// Does this service use scores of tags from other services for analysis?
354    #[serde(default)]
355    pub uses_tag_scores: bool,
356    /// Does this service use temp data from other services for analysis?
357    #[serde(default)]
358    pub uses_temp_submission_data: bool,
359    /// Does this service use submission metadata for analysis?
360    #[serde(default)]
361    pub uses_metadata: bool,
362    /// This service watches these temporary keys for changes when partial results are produced.
363    #[serde(default)]
364    pub monitored_keys: Vec<String>,
365
366    /// Name of service
367    #[metadata(store=true, copyto="__text__")]
368    pub name: ServiceName,
369    /// Version of service
370    #[metadata(store=true)]
371    pub version: String,
372
373    /// Should the service be able to talk to core infrastructure or just service-server for tasking?
374    #[serde(default)]
375    pub privileged: bool,
376    /// Should the result cache be disabled for this service?
377    #[serde(default)]
378    pub disable_cache: bool,
379
380    /// Which execution stage does this service run in?
381    #[metadata(store=true, copyto="__text__")]
382    #[serde(default="default_stage")]
383    pub stage: String,
384    /// Submission parameters of service
385    #[metadata(index=false)]
386    #[serde(default)]
387    pub submission_params: Vec<SubmissionParams>,
388    /// Service task timeout, in seconds
389    #[serde(default="default_timeout")]
390    pub timeout: i32,
391
392    /// Docker configuration for service
393    pub docker_config: DockerConfig,
394    /// Dependency configuration for service
395    #[serde(default)]
396    pub dependencies: HashMap<String, DependencyConfig>,
397
398    /// What channel to watch for service updates?
399    #[serde(default="default_update_channel")]
400    pub update_channel: ChannelKinds,
401    /// Update configuration for fetching external resources
402    pub update_config: Option<UpdateConfig>,
403
404    /// List of service names/categories where recursion is prevented.
405    #[serde(default)]
406    pub recursion_prevention: Vec<ServiceName>,
407}
408
409fn default_category() -> ServiceName { ServiceName::from_string("Static Analysis".to_owned()) }
410fn default_description() -> Text { Text("NA".to_owned()) }
411fn default_stage() -> String { "CORE".to_owned() }
412fn default_timeout() -> i32 { 60 }
413fn default_update_channel() -> ChannelKinds { ChannelKinds::Stable }
414
415impl Service {
416    pub fn key(&self) -> String {
417        format!("{}_{}", self.name, self.version)
418    }
419}
420
421#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
422#[metadata_type(ElasticMeta)]
423#[strum(serialize_all = "lowercase")]
424pub enum ChannelKinds {
425    Stable,
426    RC,
427    Beta,
428    Dev,
429}
430
431fn default_service_accepts() -> String { ".*".to_string() }
432fn default_service_rejects() -> Option<String> { Some("empty|metadata/.*".to_string()) }
433
434impl Readable for Service {
435    fn set_from_archive(&mut self, _from_archive: bool) {}
436}
assemblyline_models/datastore/service.rs

assemblyline_models/datastore/
service.rs
