token-count 0.4.0

Count tokens for LLM models using exact tokenization
Documentation
name: Release

on:
  push:
    tags:
      - 'v*'
  workflow_dispatch:
    inputs:
      tag:
        description: 'Tag to release (e.g., v0.1.0)'
        required: true
        type: string

env:
  RUST_BACKTRACE: 1
  CARGO_TERM_COLOR: always

jobs:
  build:
    name: Build - ${{ matrix.target }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # Linux x86_64
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-22.04
            archive_name: tar.gz
            binary_name: token-count
          
          # macOS Intel (cross-compiled on Apple Silicon)
          - target: x86_64-apple-darwin
            os: macos-14
            archive_name: tar.gz
            binary_name: token-count
          
          # macOS Apple Silicon
          - target: aarch64-apple-darwin
            os: macos-14
            archive_name: tar.gz
            binary_name: token-count
          
          # Windows x86_64
          - target: x86_64-pc-windows-msvc
            os: windows-2022
            archive_name: zip
            binary_name: token-count.exe

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@1.85.0
        with:
          targets: ${{ matrix.target }}

      - name: Setup cache
        uses: Swatinem/rust-cache@v2
        with:
          key: ${{ matrix.target }}

      - name: Build release binary
        run: cargo build --release --target ${{ matrix.target }} --verbose

      - name: Strip binary (Linux)
        if: matrix.os == 'ubuntu-22.04'
        run: strip target/${{ matrix.target }}/release/${{ matrix.binary_name }}

      - name: Strip binary (macOS)
        if: startsWith(matrix.os, 'macos')
        run: strip target/${{ matrix.target }}/release/${{ matrix.binary_name }}

      - name: Extract version
        id: version
        shell: bash
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            VERSION="${{ github.event.inputs.tag }}"
          else
            VERSION="${GITHUB_REF#refs/tags/}"
          fi
          # Strip 'v' prefix for archive name
          VERSION_NUM="${VERSION#v}"
          echo "version=${VERSION}" >> $GITHUB_OUTPUT
          echo "version_num=${VERSION_NUM}" >> $GITHUB_OUTPUT

      - name: Create archive directory
        shell: bash
        run: mkdir -p dist/token-count

      - name: Copy binary to archive directory
        shell: bash
        run: |
          cp target/${{ matrix.target }}/release/${{ matrix.binary_name }} dist/token-count/
          cp README.md LICENSE dist/token-count/

      - name: Create tar.gz archive (Unix)
        if: matrix.archive_name == 'tar.gz'
        working-directory: dist
        run: tar czf token-count-${{ steps.version.outputs.version_num }}-${{ matrix.target }}.tar.gz token-count/

      - name: Create zip archive (Windows)
        if: matrix.archive_name == 'zip'
        working-directory: dist
        shell: pwsh
        run: Compress-Archive -Path token-count/* -DestinationPath token-count-${{ steps.version.outputs.version_num }}-${{ matrix.target }}.zip

      - name: Generate checksum (Unix)
        if: matrix.archive_name == 'tar.gz'
        working-directory: dist
        run: shasum -a 256 token-count-${{ steps.version.outputs.version_num }}-${{ matrix.target }}.tar.gz > token-count-${{ steps.version.outputs.version_num }}-${{ matrix.target }}.tar.gz.sha256

      - name: Generate checksum (Windows)
        if: matrix.archive_name == 'zip'
        working-directory: dist
        shell: pwsh
        run: |
          $hash = (Get-FileHash -Algorithm SHA256 token-count-${{ steps.version.outputs.version_num }}-${{ matrix.target }}.zip).Hash.ToLower()
          "$hash  token-count-${{ steps.version.outputs.version_num }}-${{ matrix.target }}.zip" | Out-File -Encoding ASCII token-count-${{ steps.version.outputs.version_num }}-${{ matrix.target }}.zip.sha256

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        with:
          name: token-count-${{ matrix.target }}
          path: |
            dist/token-count-*.${{ matrix.archive_name }}
            dist/token-count-*.${{ matrix.archive_name }}.sha256
          if-no-files-found: error

  release:
    name: Create GitHub Release
    needs: build
    runs-on: ubuntu-22.04
    permissions:
      contents: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Download all artifacts
        uses: actions/download-artifact@v8
        with:
          path: artifacts

      - name: Extract version
        id: version
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            VERSION="${{ github.event.inputs.tag }}"
          else
            VERSION="${GITHUB_REF#refs/tags/}"
          fi
          VERSION_NUM="${VERSION#v}"
          echo "version=${VERSION}" >> $GITHUB_OUTPUT
          echo "version_num=${VERSION_NUM}" >> $GITHUB_OUTPUT

      - name: Consolidate artifacts
        run: |
          mkdir -p release
          find artifacts -type f \( -name "*.tar.gz" -o -name "*.zip" -o -name "*.sha256" \) -exec mv {} release/ \;

      - name: Create checksums.txt
        working-directory: release
        run: |
          echo "# SHA256 Checksums for token-count ${{ steps.version.outputs.version }}" > checksums.txt
          echo "" >> checksums.txt
          for file in *.sha256; do
            cat "$file" >> checksums.txt
          done
          rm *.sha256

      - name: Create release notes
        run: |
          cat > release/RELEASE_NOTES.md << 'EOF'
          # token-count ${{ steps.version.outputs.version }}
          
          ## Installation
          
          ### Quick Install (Linux/macOS)
          ```bash
          curl -sSfL https://raw.githubusercontent.com/shaunburdick/token-count/main/install.sh | bash
          ```
          
          ### Homebrew (macOS/Linux)
          ```bash
          brew install shaunburdick/tap/token-count
          ```
          
          ### Cargo
          ```bash
          cargo install token-count
          ```
          
          ### Manual Download
          Download the appropriate archive for your platform below, verify the checksum, and extract the binary.
          
          ## Platform Support
          - **Linux x86_64**: `token-count-${{ steps.version.outputs.version_num }}-x86_64-unknown-linux-gnu.tar.gz`
          - **macOS Intel**: `token-count-${{ steps.version.outputs.version_num }}-x86_64-apple-darwin.tar.gz`
          - **macOS Apple Silicon**: `token-count-${{ steps.version.outputs.version_num }}-aarch64-apple-darwin.tar.gz`
          - **Windows x86_64**: `token-count-${{ steps.version.outputs.version_num }}-x86_64-pc-windows-msvc.zip`
          
          ## Verification
          All binaries are provided with SHA256 checksums. See `checksums.txt` for details.
          
          ## Documentation
          - [Installation Guide](https://github.com/shaunburdick/token-count/blob/main/INSTALL.md)
          - [README](https://github.com/shaunburdick/token-count/blob/main/README.md)
          - [Changelog](https://github.com/shaunburdick/token-count/blob/main/CHANGELOG.md)
          EOF

      - name: Create GitHub Release
        uses: softprops/action-gh-release@v2
        with:
          tag_name: ${{ steps.version.outputs.version }}
          name: token-count ${{ steps.version.outputs.version }}
          body_path: release/RELEASE_NOTES.md
          draft: false
          prerelease: ${{ contains(steps.version.outputs.version, '-') }}
          files: |
            release/*.tar.gz
            release/*.zip
            release/checksums.txt
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  update-homebrew:
    name: Update Homebrew Formula
    needs: release
    runs-on: ubuntu-22.04
    # Only run on official releases (not test tags)
    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
    steps:
      - name: Checkout homebrew-tap repository
        uses: actions/checkout@v6
        with:
          repository: shaunburdick/homebrew-tap
          token: ${{ secrets.HOMEBREW_TAP_TOKEN }}
          path: homebrew-tap

      - name: Extract version
        id: version
        run: |
          VERSION="${GITHUB_REF#refs/tags/v}"
          echo "version=${VERSION}" >> $GITHUB_OUTPUT

      - name: Download release artifacts
        run: |
          VERSION="${{ steps.version.outputs.version }}"
          mkdir -p artifacts
          
          # Download checksums
          curl -sSL "https://github.com/shaunburdick/token-count/releases/download/v${VERSION}/checksums.txt" -o artifacts/checksums.txt
          
          # Parse SHA256 checksums for each platform
          echo "linux_sha256=$(grep 'x86_64-unknown-linux-gnu.tar.gz' artifacts/checksums.txt | awk '{print $1}')" >> $GITHUB_ENV
          echo "macos_intel_sha256=$(grep 'x86_64-apple-darwin.tar.gz' artifacts/checksums.txt | awk '{print $1}')" >> $GITHUB_ENV
          echo "macos_arm_sha256=$(grep 'aarch64-apple-darwin.tar.gz' artifacts/checksums.txt | awk '{print $1}')" >> $GITHUB_ENV

      - name: Generate Homebrew formula
        run: |
          VERSION="${{ steps.version.outputs.version }}"
          cat > homebrew-tap/Formula/token-count.rb << 'EOF'
          # typed: false
          # frozen_string_literal: true

          # This file is auto-generated by GitHub Actions
          # Manual edits will be overwritten on next release

          class TokenCount < Formula
            desc "Count tokens for LLM models using exact tokenization"
            homepage "https://github.com/shaunburdick/token-count"
            version "VERSION_PLACEHOLDER"
            license "MIT"

            on_macos do
              on_intel do
                url "https://github.com/shaunburdick/token-count/releases/download/vVERSION_PLACEHOLDER/token-count-VERSION_PLACEHOLDER-x86_64-apple-darwin.tar.gz"
                sha256 "MACOS_INTEL_SHA256_PLACEHOLDER"
              end
              on_arm do
                url "https://github.com/shaunburdick/token-count/releases/download/vVERSION_PLACEHOLDER/token-count-VERSION_PLACEHOLDER-aarch64-apple-darwin.tar.gz"
                sha256 "MACOS_ARM_SHA256_PLACEHOLDER"
              end
            end

            on_linux do
              on_intel do
                url "https://github.com/shaunburdick/token-count/releases/download/vVERSION_PLACEHOLDER/token-count-VERSION_PLACEHOLDER-x86_64-unknown-linux-gnu.tar.gz"
                sha256 "LINUX_SHA256_PLACEHOLDER"
              end
            end

            def install
              bin.install "token-count"
            end

            test do
              assert_match "token-count #{version}", shell_output("#{bin}/token-count --version")
              
              # Test basic functionality
              output = pipe_output("#{bin}/token-count", "Hello, world!")
              assert output.to_i > 0, "Expected token count > 0"
            end
          end
          EOF
          
          # Replace placeholders
          sed -i "s/VERSION_PLACEHOLDER/${VERSION}/g" homebrew-tap/Formula/token-count.rb
          sed -i "s/LINUX_SHA256_PLACEHOLDER/${linux_sha256}/g" homebrew-tap/Formula/token-count.rb
          sed -i "s/MACOS_INTEL_SHA256_PLACEHOLDER/${macos_intel_sha256}/g" homebrew-tap/Formula/token-count.rb
          sed -i "s/MACOS_ARM_SHA256_PLACEHOLDER/${macos_arm_sha256}/g" homebrew-tap/Formula/token-count.rb

      - name: Commit and push formula
        run: |
          cd homebrew-tap
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add Formula/token-count.rb
          git commit -m "chore: update token-count to ${{ steps.version.outputs.version }}

          Automated update via GitHub Actions
          Uses pre-built binaries from GitHub releases"
          git push

  publish-crate:
    name: Publish to crates.io
    needs: release
    runs-on: ubuntu-22.04
    # Only run on official releases (not test tags)
    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@1.85.0

      - name: Publish to crates.io
        run: cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
        continue-on-error: true  # Don't fail if version already published