melodium 0.10.1

Flow-oriented language & tool, focusing on treatments applied on data, allowing high scalability and massive parallelization safely
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134


/*
  Example of advanced CI/CD implementation.

  This file illustrate treatments doing some advanced interaction between runners, commands, and data.
*/

use cicd/runners::CicdDispatchEngine
use cicd/runners::CicdRunnerEngine
use cicd/runners::setupRunner
use cicd/runners::stopRunner
use cicd/steps::stepOn
use cicd/steps::stepOnWithInput
use process/command::|command
use process/command::|raw_commands
use process/environment::Environment
use process/environment::|environment
use std/engine/util::startup
use std/data/string_map::|map
use std/ops/option::|wrap
use work/resources::|container
use work/resources/arch::|amd64
use work/resources::|volume
use work/resources::|mount

/**
  `main` treatment, used as `advanced` entrypoint as defined in `Compo.toml`.

  - `api_token`: service key provided to access Mélodium work distribution API;
  - `work_location`: location where dispatched runners take place (default throught API, can be set to `"compose"` for local run);
  - `output_directory`: directory where output files (including logs) are wrote;
  - `repository_clone_url`: URL to process Git clone;
  - `repository_clone_ref`: Git reference (sha or tag) to clone.

  All the other parameters are present to handle integration with CI services.
*/
treatment main(const api_token: string, const work_location: string = "api", repository_clone_url: string, repository_clone_ref: string, ci_service_token: string = "", ci_service_project: string = "", ci_service_ref: string = "", ci_service_sha: string = "", ci_service_pipeline: string = "", on_github: bool = false, on_gitlab: bool = false)
  // Model used to dispatch work among Mélodium engines.
  model cicd: CicdDispatchEngine(api_token=api_token, location=work_location)
{
    startup()

    build[cicd=cicd](repository_clone_url=repository_clone_url, repository_clone_ref=repository_clone_ref)
    test[cicd=cicd]()

    startup.trigger -> build.trigger,data -> test.data
    startup.trigger -----------------------> test.trigger
}

/**
  Build step, cloning and listing the repository root content.
*/
treatment build[cicd: CicdDispatchEngine](repository_clone_url: string, repository_clone_ref: string)
  model runner: CicdRunnerEngine()
  input trigger: Block<void>
  output data:   Stream<byte>
{
    setupRunner[
        dispatcher=cicd,
        runner=runner,
        logger=logger
    ](
        name="build",
        cpu=10,
        memory=500,
        storage=1000,
        volumes=[|volume("build-result", 30)],
        containers=[|container("ubuntu", 1000, 10, 8000, |amd64(), [|mount("build-result", "/mounted/result")], "ubuntu:noble", _)]
    )
    gitClone: stepOn[runner=runner](
        name="git",
        executor_name="ubuntu",
        environment=|wrap<Environment>(|environment(|map([]), "/root", false, false)),
        commands=[
            |command("apt-get", ["update"]),
            |command("apt-get", ["install", "-y", "git"]),
            |command("git", ["config", "--global", "url.https://.insteadOf", "git://"]),
            |command("git", ["clone", "--branch", repository_clone_ref, "--depth", "1", repository_clone_url, "project"])
        ]
    )
    makeList: stepOn[runner=runner](
        name="list",
        executor_name="ubuntu",
        environment=|wrap<Environment>(|environment(|map([]), "/root/project", false, false)),
        commands=|raw_commands([
            "sh -c \"ls -l --almost-all | tee files-list.txt\"",
            "mv files-list.txt /mounted/result/"
        ]),
        out_filesystem="build-result",
        out_file="files-list.txt"
    )
    stopRunner[runner=runner]()

    Self.trigger -> setupRunner.trigger,ready -> gitClone.trigger,completed -> makeList.trigger,finished -> stopRunner.trigger
                                                                               makeList.data -------------> Self.data
}

/**
  Test step, insuring listing is not empty.
*/
treatment test[cicd: CicdDispatchEngine]()
  model runner: CicdRunnerEngine()
  input trigger: Block<void>
  input data: Stream<byte>
  output finished: Block<void>
{
    setupRunner[
        dispatcher=cicd,
        runner=runner,
        logger=logger
    ](
        name="test",
        cpu=10,
        memory=500,
        storage=1000,
        volumes=[|volume("tested-data", 30)],
        containers=[|container("alpine", 500, 10, 1000, |amd64(), [|mount("tested-data", "/mounted/data")], "alpine:3", _)]
    )
    makeTest: stepOnWithInput[runner=runner](
        name="test",
        executor_name="alpine",
        in_filesystem="tested-data",
        in_file="list.txt",
        commands=[
            |command("test", ["-s", "/mounted/data/list.txt"]),
            |command("cat", ["/mounted/data/list.txt"])
        ]
    )
    stopRunner[runner=runner]()

    Self.trigger -> setupRunner.trigger,ready -> makeTest.trigger
    Self.data ---------------------------------> makeTest.data,finished -> stopRunner.trigger
                                                 makeTest.finished ------> Self.finished
}