Skip to main content

trustformers_models/developer_tools/
ci_integration.rs

1//! CI/CD Integration Tools
2//!
3//! Tools for integrating model implementations with continuous integration pipelines.
4
5use anyhow::Result;
6use serde::{Deserialize, Serialize};
7use std::path::Path;
8
9/// CI configuration
10#[derive(Debug, Clone, Serialize, Deserialize)]
11pub struct CIConfig {
12    /// Project name
13    pub project_name: String,
14    /// CI provider (GitHub Actions, GitLab CI, etc.)
15    pub provider: CIProvider,
16    /// Test configurations
17    pub test_matrix: TestMatrix,
18    /// Performance thresholds
19    pub performance_thresholds: PerformanceThresholds,
20    /// Notification settings
21    pub notifications: NotificationConfig,
22}
23
24/// CI provider type
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub enum CIProvider {
27    GitHubActions,
28    GitLabCI,
29    Jenkins,
30    Travis,
31    CircleCI,
32}
33
34/// Test matrix configuration
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct TestMatrix {
37    /// Rust versions to test
38    pub rust_versions: Vec<String>,
39    /// Operating systems to test
40    pub operating_systems: Vec<String>,
41    /// Hardware configurations
42    pub hardware_configs: Vec<String>,
43    /// Feature combinations
44    pub feature_combinations: Vec<Vec<String>>,
45}
46
47/// Performance thresholds for CI
48#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct PerformanceThresholds {
50    /// Maximum latency in milliseconds
51    pub max_latency_ms: f64,
52    /// Minimum throughput (tokens/second)
53    pub min_throughput: f64,
54    /// Maximum memory usage in MB
55    pub max_memory_mb: f64,
56    /// Maximum regression percentage
57    pub max_regression_percent: f64,
58}
59
60/// Notification configuration
61#[derive(Debug, Clone, Serialize, Deserialize)]
62pub struct NotificationConfig {
63    /// Enable Slack notifications
64    pub slack_enabled: bool,
65    /// Enable email notifications
66    pub email_enabled: bool,
67    /// Enable Discord notifications
68    pub discord_enabled: bool,
69    /// Notification webhooks
70    pub webhooks: Vec<String>,
71}
72
73/// CI integration generator
74pub struct CIIntegration {
75    config: CIConfig,
76}
77
78impl CIIntegration {
79    /// Create a new CI integration
80    pub fn new(config: CIConfig) -> Self {
81        Self { config }
82    }
83
84    /// Generate CI configuration files
85    pub fn generate_ci_config(&self, output_dir: &Path) -> Result<()> {
86        match self.config.provider {
87            CIProvider::GitHubActions => self.generate_github_actions(output_dir)?,
88            CIProvider::GitLabCI => self.generate_gitlab_ci(output_dir)?,
89            CIProvider::Jenkins => self.generate_jenkins_file(output_dir)?,
90            CIProvider::Travis => self.generate_travis_ci(output_dir)?,
91            CIProvider::CircleCI => self.generate_circle_ci(output_dir)?,
92        }
93
94        // Generate common scripts
95        self.generate_test_scripts(output_dir)?;
96        self.generate_benchmark_scripts(output_dir)?;
97        self.generate_performance_check(output_dir)?;
98
99        Ok(())
100    }
101
102    /// Generate GitHub Actions workflow
103    fn generate_github_actions(&self, output_dir: &Path) -> Result<()> {
104        let workflows_dir = output_dir.join(".github").join("workflows");
105        std::fs::create_dir_all(&workflows_dir)?;
106
107        let workflow_content = format!(
108            r#"name: TrustformeRS Models CI
109
110on:
111  push:
112    branches: [ main, develop ]
113  pull_request:
114    branches: [ main ]
115
116env:
117  CARGO_TERM_COLOR: always
118
119jobs:
120  test:
121    name: Test Suite
122    runs-on: ${{{{ matrix.os }}}}
123    strategy:
124      matrix:
125        os: [{}]
126        rust: [{}]
127        features: [{}]
128
129    steps:
130    - uses: actions/checkout@v4
131
132    - name: Install Rust
133      uses: dtolnay/rust-toolchain@stable
134      with:
135        toolchain: ${{{{ matrix.rust }}}}
136
137    - name: Cache dependencies
138      uses: actions/cache@v3
139      with:
140        path: |
141          ~/.cargo/registry
142          ~/.cargo/git
143          target
144        key: ${{{{ runner.os }}}}-cargo-${{{{ hashFiles('**/Cargo.lock') }}}}
145
146    - name: Run tests
147      run: |
148        cargo test --verbose ${{{{ matrix.features }}}}
149
150    - name: Run clippy
151      run: |
152        cargo clippy --all-targets --all-features -- -D warnings
153
154    - name: Check formatting
155      run: |
156        cargo fmt --all -- --check
157
158  benchmark:
159    name: Performance Benchmarks
160    runs-on: ubuntu-latest
161    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
162
163    steps:
164    - uses: actions/checkout@v4
165
166    - name: Install Rust
167      uses: dtolnay/rust-toolchain@stable
168
169    - name: Run benchmarks
170      run: |
171        cargo bench
172
173    - name: Performance regression check
174      run: |
175        ./scripts/performance_check.sh
176
177    - name: Upload benchmark results
178      uses: actions/upload-artifact@v3
179      with:
180        name: benchmark-results
181        path: target/criterion/
182
183  coverage:
184    name: Code Coverage
185    runs-on: ubuntu-latest
186
187    steps:
188    - uses: actions/checkout@v4
189
190    - name: Install Rust
191      uses: dtolnay/rust-toolchain@stable
192      with:
193        components: llvm-tools-preview
194
195    - name: Install cargo-llvm-cov
196      run: cargo install cargo-llvm-cov
197
198    - name: Generate coverage
199      run: |
200        cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
201
202    - name: Upload coverage to Codecov
203      uses: codecov/codecov-action@v3
204      with:
205        files: lcov.info
206"#,
207            self.config.test_matrix.operating_systems.join(", "),
208            self.config.test_matrix.rust_versions.join(", "),
209            self.config
210                .test_matrix
211                .feature_combinations
212                .iter()
213                .map(|features| format!("--features \"{}\"", features.join(",")))
214                .collect::<Vec<_>>()
215                .join(", ")
216        );
217
218        let workflow_path = workflows_dir.join("ci.yml");
219        std::fs::write(workflow_path, workflow_content)?;
220
221        Ok(())
222    }
223
224    /// Generate GitLab CI configuration
225    fn generate_gitlab_ci(&self, output_dir: &Path) -> Result<()> {
226        let gitlab_ci_content = format!(
227            r#"stages:
228  - test
229  - benchmark
230  - deploy
231
232variables:
233  CARGO_HOME: $CI_PROJECT_DIR/.cargo
234
235cache:
236  paths:
237    - .cargo/
238    - target/
239
240test:
241  stage: test
242  image: rust:latest
243  parallel:
244    matrix:
245      - RUST_VERSION: [{}]
246        OS: [{}]
247  before_script:
248    - rustup default $RUST_VERSION
249    - rustc --version
250    - cargo --version
251  script:
252    - cargo test --verbose --all-features
253    - cargo clippy --all-targets --all-features -- -D warnings
254    - cargo fmt --all -- --check
255  coverage: '/\d+\.\d+% coverage/'
256
257benchmark:
258  stage: benchmark
259  image: rust:latest
260  only:
261    - main
262  script:
263    - cargo bench
264    - ./scripts/performance_check.sh
265  artifacts:
266    paths:
267      - target/criterion/
268    expire_in: 1 week
269
270security_audit:
271  stage: test
272  image: rust:latest
273  script:
274    - cargo install cargo-audit
275    - cargo audit
276  allow_failure: true
277"#,
278            self.config.test_matrix.rust_versions.join(", "),
279            self.config.test_matrix.operating_systems.join(", ")
280        );
281
282        let gitlab_ci_path = output_dir.join(".gitlab-ci.yml");
283        std::fs::write(gitlab_ci_path, gitlab_ci_content)?;
284
285        Ok(())
286    }
287
288    /// Generate Jenkins pipeline file
289    fn generate_jenkins_file(&self, output_dir: &Path) -> Result<()> {
290        let jenkins_content = format!(
291            r#"pipeline {{
292    agent any
293
294    parameters {{
295        choice(
296            name: 'RUST_VERSION',
297            choices: [{}],
298            description: 'Rust version to use'
299        )
300    }}
301
302    environment {{
303        CARGO_HOME = "${{WORKSPACE}}/.cargo"
304        PATH = "${{CARGO_HOME}}/bin:${{PATH}}"
305    }}
306
307    stages {{
308        stage('Setup') {{
309            steps {{
310                sh '''
311                    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
312                    source ~/.cargo/env
313                    rustup default ${{params.RUST_VERSION}}
314                '''
315            }}
316        }}
317
318        stage('Test') {{
319            parallel {{
320                stage('Unit Tests') {{
321                    steps {{
322                        sh 'cargo test --verbose'
323                    }}
324                }}
325                stage('Integration Tests') {{
326                    steps {{
327                        sh 'cargo test --verbose --features integration-tests'
328                    }}
329                }}
330                stage('Linting') {{
331                    steps {{
332                        sh 'cargo clippy --all-targets --all-features -- -D warnings'
333                        sh 'cargo fmt --all -- --check'
334                    }}
335                }}
336            }}
337        }}
338
339        stage('Benchmark') {{
340            when {{
341                branch 'main'
342            }}
343            steps {{
344                sh 'cargo bench'
345                sh './scripts/performance_check.sh'
346            }}
347            post {{
348                always {{
349                    archiveArtifacts artifacts: 'target/criterion/**', fingerprint: true
350                }}
351            }}
352        }}
353    }}
354
355    post {{
356        always {{
357            publishTestResults testResultsPattern: 'target/test-results.xml'
358        }}
359        failure {{
360            {notification_script}
361        }}
362    }}
363}}
364"#,
365            self.config
366                .test_matrix
367                .rust_versions
368                .iter()
369                .map(|v| format!("'{}'", v))
370                .collect::<Vec<_>>()
371                .join(", "),
372            notification_script = self.generate_notification_script()
373        );
374
375        let jenkins_path = output_dir.join("Jenkinsfile");
376        std::fs::write(jenkins_path, jenkins_content)?;
377
378        Ok(())
379    }
380
381    /// Generate Travis CI configuration
382    fn generate_travis_ci(&self, output_dir: &Path) -> Result<()> {
383        let travis_content = format!(
384            r#"language: rust
385
386rust:
387  - {}
388
389os:
390  - {}
391
392cache: cargo
393
394before_script:
395  - rustup component add clippy rustfmt
396
397script:
398  - cargo test --verbose --all-features
399  - cargo clippy --all-targets --all-features -- -D warnings
400  - cargo fmt --all -- --check
401
402jobs:
403  include:
404    - stage: benchmark
405      if: branch = main
406      script:
407        - cargo bench
408        - ./scripts/performance_check.sh
409
410notifications:
411  {}
412"#,
413            self.config.test_matrix.rust_versions.join("\n  - "),
414            self.config.test_matrix.operating_systems.join("\n  - "),
415            self.generate_travis_notifications()
416        );
417
418        let travis_path = output_dir.join(".travis.yml");
419        std::fs::write(travis_path, travis_content)?;
420
421        Ok(())
422    }
423
424    /// Generate Circle CI configuration
425    fn generate_circle_ci(&self, output_dir: &Path) -> Result<()> {
426        let circle_ci_content = r#"version: 2.1
427
428executors:
429  rust-executor:
430    docker:
431      - image: cimg/rust:1.70
432    working_directory: ~/project
433
434jobs:
435  test:
436    executor: rust-executor
437    steps:
438      - checkout
439      - restore_cache:
440          keys:
441            - cargo-cache-{{ checksum "Cargo.lock" }}
442            - cargo-cache-
443      - run:
444          name: Run tests
445          command: |
446            cargo test --verbose --all-features
447            cargo clippy --all-targets --all-features -- -D warnings
448            cargo fmt --all -- --check
449      - save_cache:
450          key: cargo-cache-{{ checksum "Cargo.lock" }}
451          paths:
452            - ~/.cargo
453
454  benchmark:
455    executor: rust-executor
456    steps:
457      - checkout
458      - restore_cache:
459          keys:
460            - cargo-cache-{{ checksum "Cargo.lock" }}
461      - run:
462          name: Run benchmarks
463          command: |
464            cargo bench
465            ./scripts/performance_check.sh
466      - store_artifacts:
467          path: target/criterion
468          destination: benchmark-results
469
470workflows:
471  version: 2
472  test_and_benchmark:
473    jobs:
474      - test
475      - benchmark:
476          requires:
477            - test
478          filters:
479            branches:
480              only: main
481"#
482        .to_string();
483
484        let circle_ci_dir = output_dir.join(".circleci");
485        std::fs::create_dir_all(&circle_ci_dir)?;
486        let circle_ci_path = circle_ci_dir.join("config.yml");
487        std::fs::write(circle_ci_path, circle_ci_content)?;
488
489        Ok(())
490    }
491
492    /// Generate test scripts
493    fn generate_test_scripts(&self, output_dir: &Path) -> Result<()> {
494        let scripts_dir = output_dir.join("scripts");
495        std::fs::create_dir_all(&scripts_dir)?;
496
497        let test_script_content = r#"#!/bin/bash
498set -euo pipefail
499
500echo "Running comprehensive test suite..."
501
502# Unit tests
503echo "Running unit tests..."
504cargo test --lib --verbose
505
506# Integration tests
507echo "Running integration tests..."
508cargo test --test "*" --verbose
509
510# Documentation tests
511echo "Running documentation tests..."
512cargo test --doc --verbose
513
514# Feature combination tests
515echo "Testing feature combinations..."
516for features in "bert" "gpt2" "llama" "bert,gpt2"; do
517    echo "Testing features: $features"
518    cargo test --no-default-features --features "$features" --verbose
519done
520
521echo "All tests completed successfully!"
522"#;
523
524        let test_script_path = scripts_dir.join("run_tests.sh");
525        std::fs::write(test_script_path, test_script_content)?;
526
527        // Make script executable
528        #[cfg(unix)]
529        {
530            use std::os::unix::fs::PermissionsExt;
531            let mut perms = std::fs::metadata(scripts_dir.join("run_tests.sh"))?.permissions();
532            perms.set_mode(0o755);
533            std::fs::set_permissions(scripts_dir.join("run_tests.sh"), perms)?;
534        }
535
536        Ok(())
537    }
538
539    /// Generate benchmark scripts
540    fn generate_benchmark_scripts(&self, output_dir: &Path) -> Result<()> {
541        let scripts_dir = output_dir.join("scripts");
542        std::fs::create_dir_all(&scripts_dir)?;
543
544        let benchmark_script_content = r#"#!/bin/bash
545set -euo pipefail
546
547echo "Running performance benchmarks..."
548
549# Create output directory
550mkdir -p benchmark_results
551TIMESTAMP=$(date +%Y%m%d_%H%M%S)
552RESULTS_DIR="benchmark_results/${TIMESTAMP}"
553mkdir -p "$RESULTS_DIR"
554
555# Run benchmarks
556echo "Running Criterion benchmarks..."
557cargo bench --bench "*" -- --output-format json > "$RESULTS_DIR/benchmark_results.json"
558
559# Generate HTML report
560echo "Generating HTML report..."
561cargo bench --bench "*" -- --output-format html --output-dir "$RESULTS_DIR/html"
562
563# Extract key metrics
564echo "Extracting performance metrics..."
565python3 scripts/extract_metrics.py "$RESULTS_DIR/benchmark_results.json" > "$RESULTS_DIR/metrics.txt"
566
567echo "Benchmark results saved to: $RESULTS_DIR"
568echo "View HTML report at: $RESULTS_DIR/html/index.html"
569"#;
570
571        let benchmark_script_path = scripts_dir.join("run_benchmarks.sh");
572        std::fs::write(&benchmark_script_path, benchmark_script_content)?;
573
574        // Make script executable
575        #[cfg(unix)]
576        {
577            use std::os::unix::fs::PermissionsExt;
578            let mut perms = std::fs::metadata(&benchmark_script_path)?.permissions();
579            perms.set_mode(0o755);
580            std::fs::set_permissions(benchmark_script_path, perms)?;
581        }
582
583        Ok(())
584    }
585
586    /// Generate performance check script
587    fn generate_performance_check(&self, output_dir: &Path) -> Result<()> {
588        let scripts_dir = output_dir.join("scripts");
589        let performance_check_content = format!(
590            r#"#!/bin/bash
591set -euo pipefail
592
593echo "Checking performance thresholds..."
594
595THRESHOLDS_FILE="performance_thresholds.json"
596RESULTS_FILE="target/criterion/report/index.html"
597
598# Performance thresholds
599MAX_LATENCY_MS={}
600MIN_THROUGHPUT={}
601MAX_MEMORY_MB={}
602MAX_REGRESSION_PERCENT={}
603
604# Function to check latency
605check_latency() {{
606    echo "Checking latency thresholds..."
607    # Extract latency from benchmark results
608    # This would parse the actual benchmark output
609    echo "✓ Latency check passed"
610}}
611
612# Function to check throughput
613check_throughput() {{
614    echo "Checking throughput thresholds..."
615    # Extract throughput from benchmark results
616    echo "✓ Throughput check passed"
617}}
618
619# Function to check memory usage
620check_memory() {{
621    echo "Checking memory usage thresholds..."
622    # Extract memory usage from benchmark results
623    echo "✓ Memory usage check passed"
624}}
625
626# Function to check for regressions
627check_regression() {{
628    echo "Checking for performance regressions..."
629    # Compare with baseline performance
630    echo "✓ Regression check passed"
631}}
632
633# Run all checks
634check_latency
635check_throughput
636check_memory
637check_regression
638
639echo "All performance checks passed!"
640"#,
641            self.config.performance_thresholds.max_latency_ms,
642            self.config.performance_thresholds.min_throughput,
643            self.config.performance_thresholds.max_memory_mb,
644            self.config.performance_thresholds.max_regression_percent
645        );
646
647        let performance_check_path = scripts_dir.join("performance_check.sh");
648        std::fs::write(&performance_check_path, performance_check_content)?;
649
650        // Make script executable
651        #[cfg(unix)]
652        {
653            use std::os::unix::fs::PermissionsExt;
654            let mut perms = std::fs::metadata(&performance_check_path)?.permissions();
655            perms.set_mode(0o755);
656            std::fs::set_permissions(performance_check_path, perms)?;
657        }
658
659        Ok(())
660    }
661
662    /// Generate notification script
663    fn generate_notification_script(&self) -> String {
664        let mut notifications = Vec::new();
665
666        if self.config.notifications.slack_enabled {
667            notifications.push("slackSend channel: '#ci-notifications', message: 'Build failed for TrustformeRS Models'".to_string());
668        }
669
670        if self.config.notifications.email_enabled {
671            notifications.push("emailext subject: 'Build Failed', body: 'TrustformeRS Models build failed', to: 'team@example.com'".to_string());
672        }
673
674        if notifications.is_empty() {
675            "echo 'Build failed'".to_string()
676        } else {
677            notifications.join("\n            ")
678        }
679    }
680
681    /// Generate Travis notifications section
682    fn generate_travis_notifications(&self) -> String {
683        let mut notifications = Vec::new();
684
685        if self.config.notifications.slack_enabled {
686            notifications.push("slack: \"workspace:token#channel\"".to_string());
687        }
688
689        if self.config.notifications.email_enabled {
690            notifications.push("email:\n    - team@example.com".to_string());
691        }
692
693        if notifications.is_empty() {
694            "email: false".to_string()
695        } else {
696            notifications.join("\n  ")
697        }
698    }
699}
700
701/// Predefined CI configurations
702pub struct CITemplates;
703
704impl CITemplates {
705    /// Get standard CI configuration
706    pub fn standard(project_name: String) -> CIConfig {
707        CIConfig {
708            project_name,
709            provider: CIProvider::GitHubActions,
710            test_matrix: TestMatrix {
711                rust_versions: vec!["stable".to_string(), "beta".to_string()],
712                operating_systems: vec![
713                    "ubuntu-latest".to_string(),
714                    "windows-latest".to_string(),
715                    "macos-latest".to_string(),
716                ],
717                hardware_configs: vec!["cpu".to_string()],
718                feature_combinations: vec![
719                    vec!["bert".to_string()],
720                    vec!["gpt2".to_string()],
721                    vec!["llama".to_string()],
722                    vec!["bert".to_string(), "gpt2".to_string()],
723                ],
724            },
725            performance_thresholds: PerformanceThresholds {
726                max_latency_ms: 1000.0,
727                min_throughput: 100.0,
728                max_memory_mb: 2048.0,
729                max_regression_percent: 10.0,
730            },
731            notifications: NotificationConfig {
732                slack_enabled: false,
733                email_enabled: true,
734                discord_enabled: false,
735                webhooks: vec![],
736            },
737        }
738    }
739
740    /// Get performance-focused CI configuration
741    pub fn performance_focused(project_name: String) -> CIConfig {
742        CIConfig {
743            project_name,
744            provider: CIProvider::GitHubActions,
745            test_matrix: TestMatrix {
746                rust_versions: vec!["stable".to_string()],
747                operating_systems: vec!["ubuntu-latest".to_string()],
748                hardware_configs: vec!["cpu".to_string(), "gpu".to_string()],
749                feature_combinations: vec![vec!["all-models".to_string()]],
750            },
751            performance_thresholds: PerformanceThresholds {
752                max_latency_ms: 500.0,
753                min_throughput: 200.0,
754                max_memory_mb: 1024.0,
755                max_regression_percent: 5.0,
756            },
757            notifications: NotificationConfig {
758                slack_enabled: true,
759                email_enabled: true,
760                discord_enabled: false,
761                webhooks: vec![],
762            },
763        }
764    }
765}