otlp2pipeline 0.4.0

OTLP ingestion worker for Cloudflare Pipelines and AWS
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

use anyhow::Result;

use super::cli::AwsCli;
use super::context::S3_TABLES_ROLE_NAME;
use super::helpers::{load_config, resolve_env_name, resolve_region, stack_name};
use super::schema::TABLES;
use crate::cli::PlanArgs;

pub fn execute_plan(args: PlanArgs) -> Result<()> {
    let config = load_config()?;
    let env_name = resolve_env_name(args.env)?;
    let region = resolve_region(args.region, &config);
    let stack = stack_name(&env_name);

    let cli = AwsCli::new(&region);
    let account = cli.sts().get_caller_identity()?;

    eprintln!("==> AWS Deployment Plan (Dry Run)\n");
    eprintln!("Account: {}", account.account_id);
    eprintln!("Region:  {}", region);
    eprintln!("Stack:   {}", stack);
    eprintln!();

    // Check current state
    eprintln!("==> Current State\n");

    // IAM Role
    let role_exists = cli.iam().role_exists(S3_TABLES_ROLE_NAME)?;
    eprintln!(
        "IAM Role ({}): {}",
        S3_TABLES_ROLE_NAME,
        if role_exists {
            "exists"
        } else {
            "will be created"
        }
    );

    // Glue catalog
    let catalog_exists = cli.glue().catalog_exists("s3tablescatalog")?;
    eprintln!(
        "Glue Catalog (s3tablescatalog): {}",
        if catalog_exists {
            "exists"
        } else {
            "will be created"
        }
    );

    // Stack
    let stack_exists = cli.cloudformation().describe_stack(&stack)?.is_some();
    eprintln!(
        "CloudFormation Stack ({}): {}",
        stack,
        if stack_exists {
            "exists (will update)"
        } else {
            "will be created"
        }
    );

    eprintln!();
    eprintln!("==> Resources to be Created/Updated\n");

    eprintln!("Phase 0: S3 Tables + LakeFormation");
    eprintln!("  - IAM Role: {}", S3_TABLES_ROLE_NAME);
    eprintln!("  - LakeFormation Admin: {}", account.caller_arn);
    eprintln!(
        "  - LakeFormation Resource: arn:aws:s3tables:{}:{}:bucket/*",
        region, account.account_id
    );
    eprintln!("  - Glue Catalog: s3tablescatalog");
    eprintln!();

    eprintln!("Phase 1: CloudFormation Stack");
    eprintln!("  - Stack: {}", stack);
    eprintln!("  - Table Bucket: {}", stack);
    eprintln!("  - Namespace: default");
    eprintln!(
        "  - Error Bucket: {}-errors-{}-{}",
        stack, account.account_id, region
    );
    eprintln!("  - Firehose IAM Role");
    eprintln!("  - Lambda IAM Role");
    eprintln!("  - CloudWatch Log Group");
    eprintln!();

    eprintln!("Phase 2: Athena Table Creation");
    for table in TABLES {
        eprintln!("  - Table: {} (partitioned by day(timestamp))", table);
    }
    eprintln!();

    eprintln!("Phase 3: LakeFormation Permissions");
    eprintln!("  - DESCRIBE on database 'default'");
    for table in TABLES {
        eprintln!("  - ALL on table '{}'", table);
    }
    eprintln!();

    eprintln!("Phase 4: Firehose Streams");
    for table in TABLES {
        let stream_name = format!("{}-{}", stack, table);
        let exists = cli.firehose().stream_exists(&stream_name)?;
        eprintln!(
            "  - {} (AppendOnly): {}",
            stream_name,
            if exists { "exists" } else { "will be created" }
        );
    }
    eprintln!();

    eprintln!("Phase 5: Lambda (optional --local)");
    let function_name = format!("{}-ingest", stack);
    let function_exists = cli.lambda().function_exists(&function_name)?;
    eprintln!(
        "  - Function: {} {}",
        function_name,
        if function_exists {
            "(exists)"
        } else {
            "(will be created with --local)"
        }
    );
    eprintln!();

    eprintln!("==> To deploy, run:");
    eprintln!(
        "  otlp2pipeline aws create --env {} --region {}",
        env_name, region
    );
    eprintln!();
    eprintln!("To deploy with local Lambda build:");
    eprintln!(
        "  otlp2pipeline aws create --env {} --region {} --local",
        env_name, region
    );

    Ok(())
}