torc 0.23.0

Workflow management system
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

# Sample Data Processing Workflow
# This demonstrates a complete workflow specification in YAML format

name: sample_data_processing_workflow
description: A sample workflow that demonstrates data processing with multiple jobs

# Job definitions - these define the computational tasks
jobs:
  - name: data_download
    command: wget https://example.com/data.csv -O input_data.csv
    invocation_script: null
    cancel_on_blocking_job_failure: false
    resource_requirements: small_job
    depends_on: null  # This job has no dependencies
    input_files: null
    output_files:
      - raw_data
    input_user_data: null
    output_user_data:
      - download_metadata
    scheduler: default_scheduler

  - name: data_validation
    command: python validate_data.py input_data.csv
    # Multi-line invocation script with environment setup
    invocation_script: |
      #!/bin/bash
      set -e
      export PYTHONPATH=/opt/validation:$PYTHONPATH
    cancel_on_blocking_job_failure: true
    resource_requirements: small_job
    depends_on:
      - data_download  # Wait for download to complete
    input_files:
      - raw_data
      - validation_script
    output_files:
      - validated_data
    input_user_data:
      - download_metadata
    output_user_data:
      - validation_results
    scheduler: default_scheduler

  - name: data_analysis
    command: python analyze_data.py validated_data.csv --output results.json
    invocation_script: null
    cancel_on_blocking_job_failure: true
    resource_requirements: large_job  # Requires more resources
    depends_on:
      - data_validation  # Wait for validation
    input_files:
      - validated_data
      - analysis_script
    output_files:
      - analysis_results
    input_user_data:
      - validation_results
    output_user_data:
      - final_analysis
    scheduler: gpu_scheduler  # Use GPU scheduler for analysis

# File definitions - these represent data files used by jobs
files:
  - name: raw_data
    path: /data/input/raw_data.csv

  - name: validated_data
    path: /data/processed/validated_data.csv

  - name: analysis_results
    path: /data/output/results.json

  - name: validation_script
    path: /scripts/validate_data.py

  - name: analysis_script
    path: /scripts/analyze_data.py

# User data - structured metadata passed between jobs
user_data:
  - name: download_metadata
    data:
      source_url: https://example.com/data.csv
      download_timestamp: "2024-01-15T10:30:00Z"
      file_size_bytes: 1048576
    is_ephemeral: true  # This data is temporary

  - name: validation_results
    data:
      validation_rules:
        - no_nulls
        - valid_dates
        - numeric_ranges
      passed: true
      row_count: 10000
    is_ephemeral: false  # Keep this data for audit trail

  - name: final_analysis
    data:
      analysis_type: statistical_summary
      confidence_level: 0.95
    is_ephemeral: false

# Resource requirements - define compute resources needed
resource_requirements:
  - name: small_job
    num_cpus: 1
    num_gpus: 0
    num_nodes: 1
    memory: 2g
    runtime: PT30M  # ISO 8601 duration: 30 minutes

  - name: large_job
    num_cpus: 4
    num_gpus: 1
    num_nodes: 1
    memory: 16g
    runtime: PT2H  # ISO 8601 duration: 2 hours

# Slurm scheduler configurations
slurm_schedulers:
  - name: default_scheduler
    account: project_account
    gres: null
    mem: 8G
    nodes: 1
    ntasks_per_node: 1
    partition: general
    qos: normal
    tmp: 10G
    walltime: "01:00:00"
    extra: "--constraint=haswell"  # Additional Slurm options

  - name: gpu_scheduler
    account: gpu_project
    gres: "gpu:1"  # Request 1 GPU
    mem: 32G
    nodes: 1
    ntasks_per_node: 1
    partition: gpu
    qos: high
    tmp: 50G
    walltime: "04:00:00"
    extra: "--constraint=v100"  # Specifically request V100 GPUs