assemblyline_models/datastore/
service.rs

1use std::collections::HashMap;
2
3use serde::{Serialize, Deserialize};
4use serde_with::{SerializeDisplay, DeserializeFromStr};
5use struct_metadata::Described;
6
7use crate::types::classification::{unrestricted_classification, unrestricted_classification_string};
8use crate::types::{ClassificationString, JsonMap, NonZeroInteger, ServiceName, Text};
9use crate::{ElasticMeta, Readable};
10
11/// Environment Variable Model
12#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
13#[metadata_type(ElasticMeta)]
14#[metadata(index=false, store=false)]
15pub struct EnvironmentVariable {
16    /// Name of Environment Variable
17    pub name: String,
18    /// Value of Environment Variable
19    pub value: String,
20}
21
22
23/// Docker Container Configuration
24#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Debug)]
25#[metadata_type(ElasticMeta)]
26#[metadata(index=false, store=false)]
27pub struct DockerConfig {
28    /// Does the container have internet-access?
29    #[serde(default)]
30    pub allow_internet_access: bool,
31    /// Command to run when container starts up.
32    #[serde(default)]
33    pub command: Option<Vec<String>>,
34    /// CPU allocation
35    #[serde(default="default_cpu_cores")]
36    pub cpu_cores: f32,
37    /// Additional environemnt variables for the container
38    #[serde(default)]
39    pub environment: Vec<EnvironmentVariable>,
40    /// Complete name of the Docker image with tag, may include registry
41    pub image: String,
42    /// The username to use when pulling the image
43    #[serde(default)]
44    pub registry_username: Option<String>,
45    /// The password or token to use when pulling the image
46    #[serde(default)]
47    pub registry_password: Option<String>,
48    /// The type of container registry
49    #[serde(default="default_registry_type")]
50    pub registry_type: RegistryType,
51    /// What ports of container to expose?
52    #[serde(default)]
53    pub ports: Vec<String>,
54    /// Container RAM limit
55    #[serde(default="default_ram_mb")]
56    pub ram_mb: i32,
57    /// Container RAM request
58    #[serde(default="default_ram_mb_min")]
59    pub ram_mb_min: i32,
60    /// Service account to use for pods in kubernetes
61    #[serde(default)]
62    pub service_account: Option<String>,
63    /// Additional container labels.
64    #[serde(default)]
65    pub labels: Vec<EnvironmentVariable>,
66}
67
68fn default_cpu_cores() -> f32 { 1.0 }
69fn default_registry_type() -> RegistryType { RegistryType::Docker }
70fn default_ram_mb() -> i32 { 512 }
71fn default_ram_mb_min() -> i32 { 256 }
72
73#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
74#[metadata_type(ElasticMeta)]
75#[strum(serialize_all = "lowercase")]
76pub enum RegistryType {
77    Docker,
78    Harbor,
79}
80
81/// Container's Persistent Volume Configuration
82#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
83#[metadata_type(ElasticMeta)]
84#[metadata(index=false, store=false)]
85pub struct PersistentVolume {
86    /// Path into the container to mount volume
87    pub mount_path: String,
88    /// The amount of storage allocated for volume
89    pub capacity: String,
90    /// Storage class used to create volume
91    pub storage_class: String,
92    /// Access mode for volume
93    #[serde(default="default_access_mode")]
94    pub access_mode: AccessMode,
95}
96
97fn default_access_mode() -> AccessMode { AccessMode::ReadWriteOnce }
98
99#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
100#[metadata_type(ElasticMeta)]
101pub enum AccessMode {
102    ReadWriteOnce, ReadWriteMany
103}
104
105/// Container's Dependency Configuration
106#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Debug)]
107#[metadata_type(ElasticMeta)]
108#[metadata(index=false, store=false)]
109pub struct DependencyConfig {
110    /// Docker container configuration for dependency
111    pub container: DockerConfig,
112    /// Volume configuration for dependency
113    #[serde(default)]
114    pub volumes: HashMap<String, PersistentVolume>,
115    /// Should this dependency run as other core components?
116    #[serde(default)]
117    pub run_as_core: bool,
118}
119
120#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default, Described)]
121#[metadata_type(ElasticMeta)]
122#[serde(rename_all="UPPERCASE")]
123pub enum FetchMethods {
124    #[default]
125    Get,
126    Post,
127    Git,
128} 
129
130
131/// Update Source Configuration
132#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
133#[metadata_type(ElasticMeta)]
134#[metadata(index=false, store=false)]
135pub struct UpdateSource {
136    /// Is this source active for periodic fetching?
137    #[serde(default="default_enabled")]
138    pub enabled: bool,
139    /// Name of source
140    pub name: String,
141    /// Password used to authenticate with source
142    #[serde(default)]
143    pub password: Option<String>,
144    /// Pattern used to find files of interest from source
145    #[serde(default)]
146    pub pattern: Option<String>,
147    /// Private key used to authenticate with source
148    #[serde(default)]
149    pub private_key: Option<String>,
150    /// CA cert for source
151    #[serde(default)]
152    pub ca_cert: Option<String>,
153    /// Ignore SSL errors when reaching out to source?
154    #[serde(default)]
155    pub ssl_ignore_errors: bool,
156    /// Proxy server for source
157    #[serde(default)]
158    pub proxy: Option<String>,
159    /// URI to source
160    pub uri: String,
161    /// Username used to authenticate with source
162    #[serde(default)]
163    pub username: Option<String>,
164    /// Headers
165    #[serde(default)]
166    pub headers: Vec<EnvironmentVariable>,
167    /// Default classification used in absence of one defined in files from source
168    #[serde(default="unrestricted_classification_string")]
169    pub default_classification: ClassificationString,
170    /// Use managed identity for authentication with Azure DevOps
171    #[serde(default)]
172    pub use_managed_identity: bool,
173    /// Branch to checkout from Git repository.
174    #[serde(default)]
175    pub git_branch: Option<String>,
176    /// Synchronize signatures with remote source. Allows system to auto-disable signatures no longer found in source.
177    #[serde(default)]
178    pub sync: bool,
179    /// Fetch method to be used with source
180    #[serde(default)]
181    pub fetch_method: FetchMethods,
182    /// Should the source's classfication override the signature's self-defined classification, if any?
183    #[serde(default)]
184    pub override_classification: bool,
185    /// Processing configuration for source
186    #[serde(default)]
187    pub configuration: HashMap<String, serde_json::Value>,
188    /// Data that's sent in a POST request (`fetch_method="POST"`)
189    #[serde(default)]
190    pub data: Option<Text>,
191    /// Update check interval, in seconds, for this source
192    #[serde(default)]
193    #[metadata(mapping="integer")]
194    update_interval: Option<NonZeroInteger>,
195    /// Ignore source caching and forcefully fetch from source
196    #[serde(default)]
197    pub ignore_cache: bool,
198}
199
200fn default_enabled() -> bool { true }
201
202/// Update Configuration for Signatures
203#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
204#[metadata_type(ElasticMeta)]
205#[metadata(index=false, store=false)]
206pub struct UpdateConfig {
207    /// Does the updater produce signatures?
208    #[metadata(index=true)]
209    #[serde(default)]
210    pub generates_signatures: bool,
211    /// List of external sources
212    #[serde(default)]
213    pub sources: Vec<UpdateSource>,
214    /// Update check interval, in seconds
215    pub update_interval_seconds: i32,
216    /// Should the service wait for updates first?
217    #[serde(default)]
218    pub wait_for_update: bool,
219    /// Delimiter used when given a list of signatures
220    #[serde(default="default_signature_delimiter")]
221    pub signature_delimiter: SignatureDelimiter,
222    /// Custom delimiter definition
223    pub custom_delimiter: Option<String>,
224    /// Default pattern used for matching files
225    #[serde(default="default_default_pattern")]
226    pub default_pattern: Text,
227}
228
229fn default_signature_delimiter() -> SignatureDelimiter { SignatureDelimiter::DoubleNewLine }
230fn default_default_pattern() -> Text { ".*".into() }
231
232#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
233#[metadata_type(ElasticMeta)]
234#[strum(serialize_all = "snake_case")]
235pub enum SignatureDelimiter {
236    NewLine,
237    DoubleNewLine,
238    Pipe,
239    Comma,
240    Space,
241    None,
242    File,
243    Custom,
244}
245
246impl SignatureDelimiter {
247    pub fn token(&self) -> String {
248        match self {
249            SignatureDelimiter::NewLine => "\n".to_owned(),
250            SignatureDelimiter::DoubleNewLine => "\n\n".to_owned(),
251            SignatureDelimiter::Pipe => "|".to_owned(),
252            SignatureDelimiter::Comma => ",".to_owned(),
253            SignatureDelimiter::Space => " ".to_owned(),
254            SignatureDelimiter::None => "".to_owned(),
255            SignatureDelimiter::File => "".to_owned(),
256            SignatureDelimiter::Custom => "".to_owned(),
257        }
258    }
259}
260
261/// Submission Parameters for Service
262#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Eq, Debug)]
263#[metadata_type(ElasticMeta)]
264#[metadata(index=false, store=false)]
265pub struct SubmissionParams {
266    /// Default value (must match value in `value` field)
267    pub default: serde_json::Value,
268    /// Name of parameter
269    pub name: String,
270    /// Type of parameter
271    #[serde(rename="type")]
272    pub param_type: ParamKinds,
273    /// Default value (must match value in `default` field)
274    pub value: serde_json::Value,
275    /// List of values if `type: list`
276    #[serde(default)]
277    pub list: Vec<serde_json::Value>,
278    /// Should this parameter be hidden?
279    #[serde(default)]
280    pub hide: bool,
281}
282
283#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
284#[metadata_type(ElasticMeta)]
285#[strum(serialize_all = "lowercase")]
286pub enum ParamKinds {
287    Str,
288    Int,
289    List,
290    Bool,
291}
292
293/// Service Configuration
294#[derive(Serialize, Deserialize, Clone, Described, PartialEq, Debug)]
295#[metadata_type(ElasticMeta)]
296#[metadata(index=true, store=false)]
297pub struct Service {
298    /// Regex to accept files as identified by Assemblyline
299    /// Regexes applied to assemblyline style file type string
300    #[metadata(store=true)]
301    #[serde(default="default_service_accepts")]
302    pub accepts: String,
303    /// Regex to reject files as identified by Assemblyline
304    /// Regexes applied to assemblyline style file type string
305    #[metadata(store=true)]
306    #[serde(default="default_service_rejects")]
307    pub rejects: Option<String>,
308    /// Should the service be auto-updated?
309    #[serde(default)]
310    pub auto_update: bool,
311    /// Which category does this service belong to?
312    #[metadata(store=true, copyto="__text__")]
313    #[serde(default="default_category")]
314    pub category: ServiceName,
315    /// Classification of the service
316    #[serde(default="unrestricted_classification")]
317    pub classification: String,
318    /// Service Configuration
319    #[metadata(index=false, store=false)]
320    #[serde(default)]
321    pub config: JsonMap,
322    /// Description of service
323    #[metadata(store=true, copyto="__text__")]
324    #[serde(default="default_description")]
325    pub description: Text,
326    /// Default classification assigned to service results
327    #[serde(default="unrestricted_classification")]
328    pub default_result_classification: String,
329    /// Is the service enabled
330    #[metadata(store=true)]
331    #[serde(default)]
332    pub enabled: bool,
333    /// Does this service perform analysis outside of Assemblyline?
334    #[serde(default)]
335    pub is_external: bool,
336    /// How many licences is the service allowed to use?
337    #[serde(default)]
338    #[metadata(mapping="integer")]
339    pub licence_count: u32,
340    /// The minimum number of service instances. Overrides Scaler's min_instances configuration.
341    #[serde(default)]
342    #[metadata(mapping="integer")]
343    pub min_instances: Option<u32>,
344    /// If more than this many jobs are queued for this service drop those over this limit. 0 is unlimited.
345    #[serde(default)]
346    #[metadata(mapping="integer")]
347    pub max_queue_length: u32,
348
349    /// Does this service use tags from other services for analysis?
350    #[serde(default)]
351    pub uses_tags: bool,
352    /// Does this service use scores of tags from other services for analysis?
353    #[serde(default)]
354    pub uses_tag_scores: bool,
355    /// Does this service use temp data from other services for analysis?
356    #[serde(default)]
357    pub uses_temp_submission_data: bool,
358    /// Does this service use submission metadata for analysis?
359    #[serde(default)]
360    pub uses_metadata: bool,
361    /// This service watches these temporary keys for changes when partial results are produced.
362    #[serde(default)]
363    pub monitored_keys: Vec<String>,
364
365    /// Name of service
366    #[metadata(store=true, copyto="__text__")]
367    pub name: ServiceName,
368    /// Version of service
369    #[metadata(store=true)]
370    pub version: String,
371
372    /// Should the service be able to talk to core infrastructure or just service-server for tasking?
373    #[serde(default)]
374    pub privileged: bool,
375    /// Should the result cache be disabled for this service?
376    #[serde(default)]
377    pub disable_cache: bool,
378
379    /// Which execution stage does this service run in?
380    #[metadata(store=true, copyto="__text__")]
381    #[serde(default="default_stage")]
382    pub stage: String,
383    /// Submission parameters of service
384    #[metadata(index=false)]
385    #[serde(default)]
386    pub submission_params: Vec<SubmissionParams>,
387    /// Service task timeout, in seconds
388    #[serde(default="default_timeout")]
389    pub timeout: i32,
390
391    /// Docker configuration for service
392    pub docker_config: DockerConfig,
393    /// Dependency configuration for service
394    #[serde(default)]
395    pub dependencies: HashMap<String, DependencyConfig>,
396
397    /// What channel to watch for service updates?
398    #[serde(default="default_update_channel")]
399    pub update_channel: ChannelKinds,
400    /// Update configuration for fetching external resources
401    pub update_config: Option<UpdateConfig>,
402    
403    /// List of service names/categories where recursion is prevented.
404    #[serde(default)]
405    pub recursion_prevention: Vec<ServiceName>,
406}
407
408fn default_category() -> ServiceName { ServiceName::from_string("Static Analysis".to_owned()) }
409fn default_description() -> Text { Text("NA".to_owned()) }
410fn default_stage() -> String { "CORE".to_owned() }
411fn default_timeout() -> i32 { 60 }
412fn default_update_channel() -> ChannelKinds { ChannelKinds::Stable }
413
414impl Service {
415    pub fn key(&self) -> String {
416        format!("{}_{}", self.name, self.version)
417    }
418}
419
420#[derive(SerializeDisplay, DeserializeFromStr, strum::Display, strum::EnumString, Described, PartialEq, Eq, Debug, Clone, Copy)]
421#[metadata_type(ElasticMeta)]
422#[strum(serialize_all = "lowercase")]
423pub enum ChannelKinds {
424    Stable,
425    RC,
426    Beta,
427    Dev,
428}
429
430fn default_service_accepts() -> String { ".*".to_string() }
431fn default_service_rejects() -> Option<String> { Some("empty|metadata/.*".to_string()) }
432
433impl Readable for Service {
434    fn set_from_archive(&mut self, _from_archive: bool) {}
435}
assemblyline_models/datastore/service.rs

assemblyline_models/datastore/
service.rs
