trustformers_models/developer_tools/
ci_integration.rs1use anyhow::Result;
6use serde::{Deserialize, Serialize};
7use std::path::Path;
8
9#[derive(Debug, Clone, Serialize, Deserialize)]
11pub struct CIConfig {
12 pub project_name: String,
14 pub provider: CIProvider,
16 pub test_matrix: TestMatrix,
18 pub performance_thresholds: PerformanceThresholds,
20 pub notifications: NotificationConfig,
22}
23
24#[derive(Debug, Clone, Serialize, Deserialize)]
26pub enum CIProvider {
27 GitHubActions,
28 GitLabCI,
29 Jenkins,
30 Travis,
31 CircleCI,
32}
33
34#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct TestMatrix {
37 pub rust_versions: Vec<String>,
39 pub operating_systems: Vec<String>,
41 pub hardware_configs: Vec<String>,
43 pub feature_combinations: Vec<Vec<String>>,
45}
46
47#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct PerformanceThresholds {
50 pub max_latency_ms: f64,
52 pub min_throughput: f64,
54 pub max_memory_mb: f64,
56 pub max_regression_percent: f64,
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize)]
62pub struct NotificationConfig {
63 pub slack_enabled: bool,
65 pub email_enabled: bool,
67 pub discord_enabled: bool,
69 pub webhooks: Vec<String>,
71}
72
73pub struct CIIntegration {
75 config: CIConfig,
76}
77
78impl CIIntegration {
79 pub fn new(config: CIConfig) -> Self {
81 Self { config }
82 }
83
84 pub fn generate_ci_config(&self, output_dir: &Path) -> Result<()> {
86 match self.config.provider {
87 CIProvider::GitHubActions => self.generate_github_actions(output_dir)?,
88 CIProvider::GitLabCI => self.generate_gitlab_ci(output_dir)?,
89 CIProvider::Jenkins => self.generate_jenkins_file(output_dir)?,
90 CIProvider::Travis => self.generate_travis_ci(output_dir)?,
91 CIProvider::CircleCI => self.generate_circle_ci(output_dir)?,
92 }
93
94 self.generate_test_scripts(output_dir)?;
96 self.generate_benchmark_scripts(output_dir)?;
97 self.generate_performance_check(output_dir)?;
98
99 Ok(())
100 }
101
102 fn generate_github_actions(&self, output_dir: &Path) -> Result<()> {
104 let workflows_dir = output_dir.join(".github").join("workflows");
105 std::fs::create_dir_all(&workflows_dir)?;
106
107 let workflow_content = format!(
108 r#"name: TrustformeRS Models CI
109
110on:
111 push:
112 branches: [ main, develop ]
113 pull_request:
114 branches: [ main ]
115
116env:
117 CARGO_TERM_COLOR: always
118
119jobs:
120 test:
121 name: Test Suite
122 runs-on: ${{{{ matrix.os }}}}
123 strategy:
124 matrix:
125 os: [{}]
126 rust: [{}]
127 features: [{}]
128
129 steps:
130 - uses: actions/checkout@v4
131
132 - name: Install Rust
133 uses: dtolnay/rust-toolchain@stable
134 with:
135 toolchain: ${{{{ matrix.rust }}}}
136
137 - name: Cache dependencies
138 uses: actions/cache@v3
139 with:
140 path: |
141 ~/.cargo/registry
142 ~/.cargo/git
143 target
144 key: ${{{{ runner.os }}}}-cargo-${{{{ hashFiles('**/Cargo.lock') }}}}
145
146 - name: Run tests
147 run: |
148 cargo test --verbose ${{{{ matrix.features }}}}
149
150 - name: Run clippy
151 run: |
152 cargo clippy --all-targets --all-features -- -D warnings
153
154 - name: Check formatting
155 run: |
156 cargo fmt --all -- --check
157
158 benchmark:
159 name: Performance Benchmarks
160 runs-on: ubuntu-latest
161 if: github.event_name == 'push' && github.ref == 'refs/heads/main'
162
163 steps:
164 - uses: actions/checkout@v4
165
166 - name: Install Rust
167 uses: dtolnay/rust-toolchain@stable
168
169 - name: Run benchmarks
170 run: |
171 cargo bench
172
173 - name: Performance regression check
174 run: |
175 ./scripts/performance_check.sh
176
177 - name: Upload benchmark results
178 uses: actions/upload-artifact@v3
179 with:
180 name: benchmark-results
181 path: target/criterion/
182
183 coverage:
184 name: Code Coverage
185 runs-on: ubuntu-latest
186
187 steps:
188 - uses: actions/checkout@v4
189
190 - name: Install Rust
191 uses: dtolnay/rust-toolchain@stable
192 with:
193 components: llvm-tools-preview
194
195 - name: Install cargo-llvm-cov
196 run: cargo install cargo-llvm-cov
197
198 - name: Generate coverage
199 run: |
200 cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
201
202 - name: Upload coverage to Codecov
203 uses: codecov/codecov-action@v3
204 with:
205 files: lcov.info
206"#,
207 self.config.test_matrix.operating_systems.join(", "),
208 self.config.test_matrix.rust_versions.join(", "),
209 self.config
210 .test_matrix
211 .feature_combinations
212 .iter()
213 .map(|features| format!("--features \"{}\"", features.join(",")))
214 .collect::<Vec<_>>()
215 .join(", ")
216 );
217
218 let workflow_path = workflows_dir.join("ci.yml");
219 std::fs::write(workflow_path, workflow_content)?;
220
221 Ok(())
222 }
223
224 fn generate_gitlab_ci(&self, output_dir: &Path) -> Result<()> {
226 let gitlab_ci_content = format!(
227 r#"stages:
228 - test
229 - benchmark
230 - deploy
231
232variables:
233 CARGO_HOME: $CI_PROJECT_DIR/.cargo
234
235cache:
236 paths:
237 - .cargo/
238 - target/
239
240test:
241 stage: test
242 image: rust:latest
243 parallel:
244 matrix:
245 - RUST_VERSION: [{}]
246 OS: [{}]
247 before_script:
248 - rustup default $RUST_VERSION
249 - rustc --version
250 - cargo --version
251 script:
252 - cargo test --verbose --all-features
253 - cargo clippy --all-targets --all-features -- -D warnings
254 - cargo fmt --all -- --check
255 coverage: '/\d+\.\d+% coverage/'
256
257benchmark:
258 stage: benchmark
259 image: rust:latest
260 only:
261 - main
262 script:
263 - cargo bench
264 - ./scripts/performance_check.sh
265 artifacts:
266 paths:
267 - target/criterion/
268 expire_in: 1 week
269
270security_audit:
271 stage: test
272 image: rust:latest
273 script:
274 - cargo install cargo-audit
275 - cargo audit
276 allow_failure: true
277"#,
278 self.config.test_matrix.rust_versions.join(", "),
279 self.config.test_matrix.operating_systems.join(", ")
280 );
281
282 let gitlab_ci_path = output_dir.join(".gitlab-ci.yml");
283 std::fs::write(gitlab_ci_path, gitlab_ci_content)?;
284
285 Ok(())
286 }
287
288 fn generate_jenkins_file(&self, output_dir: &Path) -> Result<()> {
290 let jenkins_content = format!(
291 r#"pipeline {{
292 agent any
293
294 parameters {{
295 choice(
296 name: 'RUST_VERSION',
297 choices: [{}],
298 description: 'Rust version to use'
299 )
300 }}
301
302 environment {{
303 CARGO_HOME = "${{WORKSPACE}}/.cargo"
304 PATH = "${{CARGO_HOME}}/bin:${{PATH}}"
305 }}
306
307 stages {{
308 stage('Setup') {{
309 steps {{
310 sh '''
311 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
312 source ~/.cargo/env
313 rustup default ${{params.RUST_VERSION}}
314 '''
315 }}
316 }}
317
318 stage('Test') {{
319 parallel {{
320 stage('Unit Tests') {{
321 steps {{
322 sh 'cargo test --verbose'
323 }}
324 }}
325 stage('Integration Tests') {{
326 steps {{
327 sh 'cargo test --verbose --features integration-tests'
328 }}
329 }}
330 stage('Linting') {{
331 steps {{
332 sh 'cargo clippy --all-targets --all-features -- -D warnings'
333 sh 'cargo fmt --all -- --check'
334 }}
335 }}
336 }}
337 }}
338
339 stage('Benchmark') {{
340 when {{
341 branch 'main'
342 }}
343 steps {{
344 sh 'cargo bench'
345 sh './scripts/performance_check.sh'
346 }}
347 post {{
348 always {{
349 archiveArtifacts artifacts: 'target/criterion/**', fingerprint: true
350 }}
351 }}
352 }}
353 }}
354
355 post {{
356 always {{
357 publishTestResults testResultsPattern: 'target/test-results.xml'
358 }}
359 failure {{
360 {notification_script}
361 }}
362 }}
363}}
364"#,
365 self.config
366 .test_matrix
367 .rust_versions
368 .iter()
369 .map(|v| format!("'{}'", v))
370 .collect::<Vec<_>>()
371 .join(", "),
372 notification_script = self.generate_notification_script()
373 );
374
375 let jenkins_path = output_dir.join("Jenkinsfile");
376 std::fs::write(jenkins_path, jenkins_content)?;
377
378 Ok(())
379 }
380
381 fn generate_travis_ci(&self, output_dir: &Path) -> Result<()> {
383 let travis_content = format!(
384 r#"language: rust
385
386rust:
387 - {}
388
389os:
390 - {}
391
392cache: cargo
393
394before_script:
395 - rustup component add clippy rustfmt
396
397script:
398 - cargo test --verbose --all-features
399 - cargo clippy --all-targets --all-features -- -D warnings
400 - cargo fmt --all -- --check
401
402jobs:
403 include:
404 - stage: benchmark
405 if: branch = main
406 script:
407 - cargo bench
408 - ./scripts/performance_check.sh
409
410notifications:
411 {}
412"#,
413 self.config.test_matrix.rust_versions.join("\n - "),
414 self.config.test_matrix.operating_systems.join("\n - "),
415 self.generate_travis_notifications()
416 );
417
418 let travis_path = output_dir.join(".travis.yml");
419 std::fs::write(travis_path, travis_content)?;
420
421 Ok(())
422 }
423
424 fn generate_circle_ci(&self, output_dir: &Path) -> Result<()> {
426 let circle_ci_content = r#"version: 2.1
427
428executors:
429 rust-executor:
430 docker:
431 - image: cimg/rust:1.70
432 working_directory: ~/project
433
434jobs:
435 test:
436 executor: rust-executor
437 steps:
438 - checkout
439 - restore_cache:
440 keys:
441 - cargo-cache-{{ checksum "Cargo.lock" }}
442 - cargo-cache-
443 - run:
444 name: Run tests
445 command: |
446 cargo test --verbose --all-features
447 cargo clippy --all-targets --all-features -- -D warnings
448 cargo fmt --all -- --check
449 - save_cache:
450 key: cargo-cache-{{ checksum "Cargo.lock" }}
451 paths:
452 - ~/.cargo
453
454 benchmark:
455 executor: rust-executor
456 steps:
457 - checkout
458 - restore_cache:
459 keys:
460 - cargo-cache-{{ checksum "Cargo.lock" }}
461 - run:
462 name: Run benchmarks
463 command: |
464 cargo bench
465 ./scripts/performance_check.sh
466 - store_artifacts:
467 path: target/criterion
468 destination: benchmark-results
469
470workflows:
471 version: 2
472 test_and_benchmark:
473 jobs:
474 - test
475 - benchmark:
476 requires:
477 - test
478 filters:
479 branches:
480 only: main
481"#
482 .to_string();
483
484 let circle_ci_dir = output_dir.join(".circleci");
485 std::fs::create_dir_all(&circle_ci_dir)?;
486 let circle_ci_path = circle_ci_dir.join("config.yml");
487 std::fs::write(circle_ci_path, circle_ci_content)?;
488
489 Ok(())
490 }
491
492 fn generate_test_scripts(&self, output_dir: &Path) -> Result<()> {
494 let scripts_dir = output_dir.join("scripts");
495 std::fs::create_dir_all(&scripts_dir)?;
496
497 let test_script_content = r#"#!/bin/bash
498set -euo pipefail
499
500echo "Running comprehensive test suite..."
501
502# Unit tests
503echo "Running unit tests..."
504cargo test --lib --verbose
505
506# Integration tests
507echo "Running integration tests..."
508cargo test --test "*" --verbose
509
510# Documentation tests
511echo "Running documentation tests..."
512cargo test --doc --verbose
513
514# Feature combination tests
515echo "Testing feature combinations..."
516for features in "bert" "gpt2" "llama" "bert,gpt2"; do
517 echo "Testing features: $features"
518 cargo test --no-default-features --features "$features" --verbose
519done
520
521echo "All tests completed successfully!"
522"#;
523
524 let test_script_path = scripts_dir.join("run_tests.sh");
525 std::fs::write(test_script_path, test_script_content)?;
526
527 #[cfg(unix)]
529 {
530 use std::os::unix::fs::PermissionsExt;
531 let mut perms = std::fs::metadata(scripts_dir.join("run_tests.sh"))?.permissions();
532 perms.set_mode(0o755);
533 std::fs::set_permissions(scripts_dir.join("run_tests.sh"), perms)?;
534 }
535
536 Ok(())
537 }
538
539 fn generate_benchmark_scripts(&self, output_dir: &Path) -> Result<()> {
541 let scripts_dir = output_dir.join("scripts");
542 std::fs::create_dir_all(&scripts_dir)?;
543
544 let benchmark_script_content = r#"#!/bin/bash
545set -euo pipefail
546
547echo "Running performance benchmarks..."
548
549# Create output directory
550mkdir -p benchmark_results
551TIMESTAMP=$(date +%Y%m%d_%H%M%S)
552RESULTS_DIR="benchmark_results/${TIMESTAMP}"
553mkdir -p "$RESULTS_DIR"
554
555# Run benchmarks
556echo "Running Criterion benchmarks..."
557cargo bench --bench "*" -- --output-format json > "$RESULTS_DIR/benchmark_results.json"
558
559# Generate HTML report
560echo "Generating HTML report..."
561cargo bench --bench "*" -- --output-format html --output-dir "$RESULTS_DIR/html"
562
563# Extract key metrics
564echo "Extracting performance metrics..."
565python3 scripts/extract_metrics.py "$RESULTS_DIR/benchmark_results.json" > "$RESULTS_DIR/metrics.txt"
566
567echo "Benchmark results saved to: $RESULTS_DIR"
568echo "View HTML report at: $RESULTS_DIR/html/index.html"
569"#;
570
571 let benchmark_script_path = scripts_dir.join("run_benchmarks.sh");
572 std::fs::write(&benchmark_script_path, benchmark_script_content)?;
573
574 #[cfg(unix)]
576 {
577 use std::os::unix::fs::PermissionsExt;
578 let mut perms = std::fs::metadata(&benchmark_script_path)?.permissions();
579 perms.set_mode(0o755);
580 std::fs::set_permissions(benchmark_script_path, perms)?;
581 }
582
583 Ok(())
584 }
585
586 fn generate_performance_check(&self, output_dir: &Path) -> Result<()> {
588 let scripts_dir = output_dir.join("scripts");
589 let performance_check_content = format!(
590 r#"#!/bin/bash
591set -euo pipefail
592
593echo "Checking performance thresholds..."
594
595THRESHOLDS_FILE="performance_thresholds.json"
596RESULTS_FILE="target/criterion/report/index.html"
597
598# Performance thresholds
599MAX_LATENCY_MS={}
600MIN_THROUGHPUT={}
601MAX_MEMORY_MB={}
602MAX_REGRESSION_PERCENT={}
603
604# Function to check latency
605check_latency() {{
606 echo "Checking latency thresholds..."
607 # Extract latency from benchmark results
608 # This would parse the actual benchmark output
609 echo "✓ Latency check passed"
610}}
611
612# Function to check throughput
613check_throughput() {{
614 echo "Checking throughput thresholds..."
615 # Extract throughput from benchmark results
616 echo "✓ Throughput check passed"
617}}
618
619# Function to check memory usage
620check_memory() {{
621 echo "Checking memory usage thresholds..."
622 # Extract memory usage from benchmark results
623 echo "✓ Memory usage check passed"
624}}
625
626# Function to check for regressions
627check_regression() {{
628 echo "Checking for performance regressions..."
629 # Compare with baseline performance
630 echo "✓ Regression check passed"
631}}
632
633# Run all checks
634check_latency
635check_throughput
636check_memory
637check_regression
638
639echo "All performance checks passed!"
640"#,
641 self.config.performance_thresholds.max_latency_ms,
642 self.config.performance_thresholds.min_throughput,
643 self.config.performance_thresholds.max_memory_mb,
644 self.config.performance_thresholds.max_regression_percent
645 );
646
647 let performance_check_path = scripts_dir.join("performance_check.sh");
648 std::fs::write(&performance_check_path, performance_check_content)?;
649
650 #[cfg(unix)]
652 {
653 use std::os::unix::fs::PermissionsExt;
654 let mut perms = std::fs::metadata(&performance_check_path)?.permissions();
655 perms.set_mode(0o755);
656 std::fs::set_permissions(performance_check_path, perms)?;
657 }
658
659 Ok(())
660 }
661
662 fn generate_notification_script(&self) -> String {
664 let mut notifications = Vec::new();
665
666 if self.config.notifications.slack_enabled {
667 notifications.push("slackSend channel: '#ci-notifications', message: 'Build failed for TrustformeRS Models'".to_string());
668 }
669
670 if self.config.notifications.email_enabled {
671 notifications.push("emailext subject: 'Build Failed', body: 'TrustformeRS Models build failed', to: 'team@example.com'".to_string());
672 }
673
674 if notifications.is_empty() {
675 "echo 'Build failed'".to_string()
676 } else {
677 notifications.join("\n ")
678 }
679 }
680
681 fn generate_travis_notifications(&self) -> String {
683 let mut notifications = Vec::new();
684
685 if self.config.notifications.slack_enabled {
686 notifications.push("slack: \"workspace:token#channel\"".to_string());
687 }
688
689 if self.config.notifications.email_enabled {
690 notifications.push("email:\n - team@example.com".to_string());
691 }
692
693 if notifications.is_empty() {
694 "email: false".to_string()
695 } else {
696 notifications.join("\n ")
697 }
698 }
699}
700
701pub struct CITemplates;
703
704impl CITemplates {
705 pub fn standard(project_name: String) -> CIConfig {
707 CIConfig {
708 project_name,
709 provider: CIProvider::GitHubActions,
710 test_matrix: TestMatrix {
711 rust_versions: vec!["stable".to_string(), "beta".to_string()],
712 operating_systems: vec![
713 "ubuntu-latest".to_string(),
714 "windows-latest".to_string(),
715 "macos-latest".to_string(),
716 ],
717 hardware_configs: vec!["cpu".to_string()],
718 feature_combinations: vec![
719 vec!["bert".to_string()],
720 vec!["gpt2".to_string()],
721 vec!["llama".to_string()],
722 vec!["bert".to_string(), "gpt2".to_string()],
723 ],
724 },
725 performance_thresholds: PerformanceThresholds {
726 max_latency_ms: 1000.0,
727 min_throughput: 100.0,
728 max_memory_mb: 2048.0,
729 max_regression_percent: 10.0,
730 },
731 notifications: NotificationConfig {
732 slack_enabled: false,
733 email_enabled: true,
734 discord_enabled: false,
735 webhooks: vec![],
736 },
737 }
738 }
739
740 pub fn performance_focused(project_name: String) -> CIConfig {
742 CIConfig {
743 project_name,
744 provider: CIProvider::GitHubActions,
745 test_matrix: TestMatrix {
746 rust_versions: vec!["stable".to_string()],
747 operating_systems: vec!["ubuntu-latest".to_string()],
748 hardware_configs: vec!["cpu".to_string(), "gpu".to_string()],
749 feature_combinations: vec![vec!["all-models".to_string()]],
750 },
751 performance_thresholds: PerformanceThresholds {
752 max_latency_ms: 500.0,
753 min_throughput: 200.0,
754 max_memory_mb: 1024.0,
755 max_regression_percent: 5.0,
756 },
757 notifications: NotificationConfig {
758 slack_enabled: true,
759 email_enabled: true,
760 discord_enabled: false,
761 webhooks: vec![],
762 },
763 }
764 }
765}