assay-cli 3.21.0

pub const CI_EVAL_YAML: &str = r#"configVersion: 1
suite: "ci_smoke"
model: "trace"
tests:
  - id: "ci_smoke_regex"
    input:
      prompt: "ci_regex"
    expected:
      type: regex_match
      pattern: "Hello\\s+CI"
      flags: ["i"]
  - id: "ci_smoke_schema"
    input:
      prompt: "ci_schema"
    expected:
      type: json_schema
      schema_file: "schemas/ci_answer.schema.json"
  - id: "ci_smoke_semantic"
    input:
      prompt: "ci_semantic"
    expected:
      type: semantic_similarity_to
      semantic_similarity_to: "Hello Semantic"
      min_score: 0.99
"#;

pub const CI_SCHEMA_JSON: &str = r#"{
  "type": "object",
  "required": ["answer"],
  "properties": {
    "answer": { "type": "string" }
  },
  "additionalProperties": false
}"#;

pub const CI_TRACES_JSONL: &str = r#"{"schema_version": 1, "type": "assay.trace", "request_id": "ci_1", "prompt": "ci_regex", "response": "hello   ci", "model": "trace", "provider": "trace"}
{"schema_version": 1, "type": "assay.trace", "request_id": "ci_2", "prompt": "ci_schema", "response": "{\"answer\":\"ok\"}", "model": "trace", "provider": "trace"}
{"schema_version": 1, "type": "assay.trace", "request_id": "ci_3", "prompt": "ci_semantic", "response": "Hello Semantic", "model": "trace", "provider": "trace", "meta": {"assay": {"embeddings": {"model":"trace-embed","response":[1.0,0.0,0.0],"reference":[1.0,0.0,0.0],"source_response":"trace","source_reference":"trace"}}}}
"#;

pub const HELLO_EVAL_YAML: &str = r#"configVersion: 1
suite: "hello_smoke"
model: "trace"
tests:
  - id: "hello_smoke_regex"
    input:
      prompt: "hello_prompt"
    expected:
      type: regex_match
      pattern: "Hello\\s+Assay"
      flags: ["i"]
"#;

pub const HELLO_TRACES_JSONL: &str = r#"{"schema_version": 1, "type": "assay.trace", "request_id": "hello_1", "prompt": "hello_prompt", "response": "Hello Assay", "model": "trace", "provider": "trace"}
"#;

/// Blessed CI workflow template (v2)
/// Generated by `assay init --ci` and `assay init-ci --provider github`
///
/// This template uses `assay-action` to install the CLI, then runs `assay ci`
/// directly for the eval-based gate, and finally uploads SARIF results to GitHub Code Scanning.
/// Canonical marketplace slug: `Rul1an/assay-action@v2`.
/// See: https://github.com/Rul1an/assay-action
pub const CI_WORKFLOW_YML: &str = r#"name: Assay Gate
on:
  push:
    branches: [main]
  pull_request:

permissions:
  contents: read
  pull-requests: write   # Required for PR comment
  security-events: write # Required for SARIF upload

jobs:
  assay:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 (Pinned for supply-chain security)

      - name: Setup Assay
        uses: Rul1an/assay-action@v2 # For strict supply-chain pinning, use a full commit SHA. Exact tags (e.g. v2.1.0) are less strict.

      - name: Run Assay Tests
        run: assay ci --config ci-eval.yaml --trace-file traces/ci.jsonl --sarif .assay/reports/sarif.json --junit .assay/reports/junit.xml --pr-comment .assay/reports/pr-comment.md

      - name: Upload SARIF
        # Skip on fork PRs to prevent permission errors, and ensure file exists
        if: always() && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) && hashFiles('.assay/reports/sarif.json') != ''
        # Pin to SHA for supply-chain security (v3.28.10)
        uses: github/codeql-action/upload-sarif@b20883b0cd1f46c72ae0ba6d1090936928f9fa30
        with:
          sarif_file: .assay/reports/sarif.json
          category: assay-gate

      - name: Find existing PR comment
        id: find-comment
        if: github.event_name == 'pull_request' && hashFiles('.assay/reports/pr-comment.md') != ''
        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
        continue-on-error: true
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '<!-- assay-governance-report -->'

      - name: Post governance summary
        if: github.event_name == 'pull_request' && hashFiles('.assay/reports/pr-comment.md') != ''
        uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
        continue-on-error: true
        with:
          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          edit-mode: replace
          body-path: .assay/reports/pr-comment.md

"#;

/// GitLab CI template for assay gate. Written to `.gitlab-ci.yml` by `assay init --ci gitlab`.
pub const GITLAB_CI_YML: &str = r#"assay:
  image: rust:1-bookworm
  script:
    - apt-get update -qq && apt-get install -y -qq curl
    - curl -sSfL https://raw.githubusercontent.com/Rul1an/assay/main/scripts/install.sh | sh
    - assay ci --config ci-eval.yaml --trace-file traces/ci.jsonl --sarif .assay/reports/sarif.json --junit .assay/reports/junit.xml
  artifacts:
    when: always
    paths:
      - .assay/reports/
"#;

pub const GITIGNORE: &str = "/.eval/\n/out/\n*.db\n*.db-shm\n*.db-wal\n/assay\n";

pub const EVAL_CONFIG_DEFAULT_YAML: &str = r#"configVersion: 1
suite: "starter"
model: "trace"
tests:
  - id: "starter_regex"
    input:
      prompt: "replace_with_real_prompt"
    expected:
      type: regex_match
      pattern: ".*"
      flags: ["s"]
"#;