web2llm 0.4.0

Fetch web pages and convert to clean Markdown for LLM pipelines
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

name: CI

on:
  push:
    branches:
      - main
      - 'v*'

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  ci:
    name: Check, Test, and Lint
    runs-on: ubuntu-latest
    outputs:
      changed: ${{ steps.version_check.outputs.changed }}
      current: ${{ steps.version_check.outputs.current }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # full history needed to resolve all tags

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy

      - name: Cache dependencies
        uses: Swatinem/rust-cache@v2

      - name: Check formatting
        run: cargo fmt --check

      - name: Run Clippy
        run: cargo clippy -- -W clippy::all

      - name: Run tests
        run: cargo test

      - name: Check if version changed
        id: version_check
        run: |
          CURR=$(grep '^version' Cargo.toml | head -1 | sed 's/version = "//;s/"//')
          echo "current=$CURR" >> $GITHUB_OUTPUT
          if git tag | grep -q "^v$CURR$"; then
            echo "changed=false" >> $GITHUB_OUTPUT
          else
            echo "changed=true" >> $GITHUB_OUTPUT
          fi

  release:
    name: Create Release
    needs: ci
    if: needs.ci.outputs.changed == 'true'
    uses: ./.github/workflows/release.yml
    with:
      version: ${{ needs.ci.outputs.current }}
    permissions:
      contents: write

  publish:
    name: Publish
    needs: ci
    if: needs.ci.outputs.changed == 'true'
    uses: ./.github/workflows/publish.yml
    secrets:
      CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}