floe-core 0.3.7

Core library for Floe, a YAML-driven technical ingestion tool.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

use serde::Serialize;
use std::collections::HashMap;

#[derive(Debug, Serialize)]
pub struct CommonManifest {
    pub schema: &'static str,
    pub generated_at_ts_ms: u64,
    pub floe_version: &'static str,
    pub spec_version: String,
    pub manifest_id: String,
    pub config_uri: String,
    pub config_checksum: Option<String>,
    pub report_base_uri: String,
    pub domains: Vec<ManifestDomain>,
    pub execution: ManifestExecution,
    pub runners: ManifestRunners,
    pub entities: Vec<ManifestEntity>,
}

#[derive(Debug, Serialize)]
pub struct ManifestDomain {
    pub name: String,
    pub incoming_dir: String,
}

#[derive(Debug, Serialize)]
pub struct ManifestExecution {
    pub entrypoint: &'static str,
    pub base_args: Vec<&'static str>,
    pub per_entity_args: Vec<&'static str>,
    pub log_format: &'static str,
    pub result_contract: ManifestResultContract,
    pub defaults: ManifestExecutionDefaults,
}

#[derive(Debug, Serialize)]
pub struct ManifestResultContract {
    pub run_finished_event: bool,
    pub summary_uri_field: &'static str,
    pub exit_codes: HashMap<&'static str, &'static str>,
}

#[derive(Debug, Serialize)]
pub struct ManifestExecutionDefaults {
    pub env: HashMap<String, String>,
    pub workdir: Option<String>,
}

#[derive(Debug, Serialize)]
pub struct ManifestRunners {
    pub default: &'static str,
    pub definitions: HashMap<&'static str, ManifestRunnerDefinition>,
}

#[derive(Debug, Serialize)]
pub struct ManifestRunnerDefinition {
    #[serde(rename = "type")]
    pub runner_type: &'static str,
    pub command: Option<String>,
    pub args: Option<Vec<String>>,
    pub timeout_seconds: Option<u64>,
    pub ttl_seconds_after_finished: Option<u64>,
    pub poll_interval_seconds: Option<u64>,
    pub secrets: Option<Vec<String>>,
    pub image: Option<String>,
    pub namespace: Option<String>,
    pub service_account: Option<String>,
    pub resources: Option<ManifestRunnerResources>,
    pub env: Option<HashMap<String, String>>,
    pub workspace_url: Option<String>,
    pub existing_cluster_id: Option<String>,
    pub config_uri: Option<String>,
    pub python_file_uri: Option<String>,
    pub job_name: Option<String>,
    pub auth: Option<ManifestRunnerAuth>,
    pub env_parameters: Option<HashMap<String, String>>,
}

#[derive(Debug, Serialize)]
pub struct ManifestRunnerAuth {
    pub service_principal_oauth_ref: Option<String>,
}

#[derive(Debug, Serialize)]
pub struct ManifestRunnerResources {
    pub cpu: Option<String>,
    pub memory_mb: Option<u64>,
}

#[derive(Debug, Serialize)]
pub struct ManifestEntity {
    pub name: String,
    pub domain: Option<String>,
    pub group_name: String,
    pub asset_key: Vec<String>,
    pub source_format: String,
    pub accepted_sink_uri: String,
    pub rejected_sink_uri: Option<String>,
    pub tags: Option<HashMap<String, String>>,
    pub source: ManifestSource,
    pub sinks: ManifestSinks,
    pub runner: Option<String>,
}

#[derive(Debug, Serialize)]
pub struct ManifestSource {
    pub format: String,
    pub storage: String,
    pub uri: String,
    pub path: String,
    pub resolved: bool,
    pub cast_mode: Option<String>,
    pub options: Option<serde_json::Value>,
}

#[derive(Debug, Serialize)]
pub struct ManifestSinks {
    pub accepted: ManifestSinkTarget,
    pub rejected: Option<ManifestSinkTarget>,
    pub archive: Option<ManifestArchiveTarget>,
}

#[derive(Debug, Serialize)]
pub struct ManifestSinkTarget {
    pub format: String,
    pub storage: String,
    pub uri: String,
    pub path: String,
    pub resolved: bool,
}

#[derive(Debug, Serialize)]
pub struct ManifestArchiveTarget {
    pub storage: String,
    pub uri: String,
    pub path: String,
    pub resolved: bool,
}