floe-core 0.3.2

Core library for Floe, a YAML-driven technical ingestion tool.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use serde::Serialize;
use std::collections::HashMap;

#[derive(Debug, Serialize)]
pub struct CommonManifest {
    pub schema: &'static str,
    pub generated_at_ts_ms: u64,
    pub floe_version: &'static str,
    pub spec_version: String,
    pub manifest_id: String,
    pub config_uri: String,
    pub config_checksum: Option<String>,
    pub report_base_uri: String,
    pub domains: Vec<ManifestDomain>,
    pub execution: ManifestExecution,
    pub runners: ManifestRunners,
    pub entities: Vec<ManifestEntity>,
}

#[derive(Debug, Serialize)]
pub struct ManifestDomain {
    pub name: String,
    pub incoming_dir: String,
}

#[derive(Debug, Serialize)]
pub struct ManifestExecution {
    pub entrypoint: &'static str,
    pub base_args: Vec<&'static str>,
    pub per_entity_args: Vec<&'static str>,
    pub log_format: &'static str,
    pub result_contract: ManifestResultContract,
    pub defaults: ManifestExecutionDefaults,
}

#[derive(Debug, Serialize)]
pub struct ManifestResultContract {
    pub run_finished_event: bool,
    pub summary_uri_field: &'static str,
    pub exit_codes: HashMap<&'static str, &'static str>,
}

#[derive(Debug, Serialize)]
pub struct ManifestExecutionDefaults {
    pub env: HashMap<String, String>,
    pub workdir: Option<String>,
}

#[derive(Debug, Serialize)]
pub struct ManifestRunners {
    pub default: &'static str,
    pub definitions: HashMap<&'static str, ManifestRunnerDefinition>,
}

#[derive(Debug, Serialize)]
pub struct ManifestRunnerDefinition {
    #[serde(rename = "type")]
    pub runner_type: &'static str,
    pub image: Option<String>,
    pub namespace: Option<String>,
    pub service_account: Option<String>,
    pub resources: Option<ManifestRunnerResources>,
    pub env: Option<HashMap<String, String>>,
}

#[derive(Debug, Serialize)]
pub struct ManifestRunnerResources {
    pub cpu: Option<String>,
    pub memory_mb: Option<u64>,
}

#[derive(Debug, Serialize)]
pub struct ManifestEntity {
    pub name: String,
    pub domain: Option<String>,
    pub group_name: String,
    pub asset_key: Vec<String>,
    pub source_format: String,
    pub accepted_sink_uri: String,
    pub rejected_sink_uri: Option<String>,
    pub tags: Option<HashMap<String, String>>,
    pub source: ManifestSource,
    pub sinks: ManifestSinks,
    pub runner: Option<String>,
}

#[derive(Debug, Serialize)]
pub struct ManifestSource {
    pub format: String,
    pub storage: String,
    pub uri: String,
    pub path: String,
    pub resolved: bool,
    pub cast_mode: Option<String>,
    pub options: Option<serde_json::Value>,
}

#[derive(Debug, Serialize)]
pub struct ManifestSinks {
    pub accepted: ManifestSinkTarget,
    pub rejected: Option<ManifestSinkTarget>,
    pub archive: Option<ManifestArchiveTarget>,
}

#[derive(Debug, Serialize)]
pub struct ManifestSinkTarget {
    pub format: String,
    pub storage: String,
    pub uri: String,
    pub path: String,
    pub resolved: bool,
}

#[derive(Debug, Serialize)]
pub struct ManifestArchiveTarget {
    pub storage: String,
    pub uri: String,
    pub path: String,
    pub resolved: bool,
}