precursor 0.2.3

Pre-protocol payload tagging, similarity clustering, and packet/firmware triage CLI.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
name: release

# Only do the release on x.y.z tags.
on:
  push:
    tags:
      - "[0-9]*.[0-9]*.[0-9]*"
  workflow_dispatch:
    inputs:
      version:
        description: "Release version (x.y.z tag name)"
        required: true
        type: string

# We need this to be able to create releases.
permissions:
  contents: write

jobs:
  # The create-release job runs purely to initialize the GitHub release itself,
  # and names the release after the `x.y.z` tag that was pushed. It's separate
  # from building the release so that we only create the release once.
  create-release:
    name: create-release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.version || github.ref_name }}
      - name: Get the release version from the tag
        id: release_version
        shell: bash
        run: |
          requested="${{ inputs.version }}"
          if [ -n "$requested" ]; then
            version="$requested"
          else
            version="${{ github.ref_name }}"
          fi
          echo "VERSION=$version" >> $GITHUB_ENV
          echo "version=$version" >> $GITHUB_OUTPUT
      - name: Show the version
        run: |
          echo "version is: $VERSION"
      - name: Check that tag version and Cargo.toml version are the same
        shell: bash
        run: |
          if ! grep -q "version = \"$VERSION\"" Cargo.toml; then
            echo "version does not match Cargo.toml" >&2
            exit 1
          fi
      - name: Create GitHub release
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh release create $VERSION --draft --verify-tag --title $VERSION
    outputs:
      version: ${{ steps.release_version.outputs.version }}

  generate-cli-assets:
    name: generate-cli-assets
    needs: ["create-release"]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.create-release.outputs.version }}

      - name: Install Rust
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable

      - name: Install help2man
        shell: bash
        run: |
          sudo apt-get update
          sudo apt-get install -y help2man

      - name: Build precursor (host)
        shell: bash
        run: cargo build --release

      - name: Generate completions and man page
        shell: bash
        run: |
          set -euo pipefail
          ci/generate_cli_assets.sh dist/cli-assets ./target/release/precursor

      - name: Upload CLI assets
        uses: actions/upload-artifact@v4
        with:
          name: precursor-cli-assets
          path: dist/cli-assets
          if-no-files-found: error

  build-release:
    name: build-release
    needs: ["create-release", "generate-cli-assets"]
    runs-on: ${{ matrix.os }}
    env:
      # For some builds, we use cross to test on 32-bit and big-endian
      # systems.
      CARGO: cargo
      # When CARGO is set to CROSS, this is set to `--target matrix.target`.
      TARGET_FLAGS:
      # When CARGO is set to CROSS, TARGET_DIR includes matrix.target.
      TARGET_DIR: ./target
      # Bump this as appropriate. We pin to a version to make sure CI
      # continues to work as cross releases in the past have broken things
      # in subtle ways.
      CROSS_VERSION: v0.2.5
      # Emit backtraces on panics.
      RUST_BACKTRACE: 1
      # Build static releases with PCRE2.
      PCRE2_SYS_STATIC: 1
    strategy:
      fail-fast: false
      matrix:
        include:
          - build: linux
            os: ubuntu-latest
            rust: nightly
            target: x86_64-unknown-linux-musl
            strip: x86_64-linux-musl-strip
          - build: stable-x86
            os: ubuntu-latest
            rust: stable
            target: i686-unknown-linux-gnu
            strip: x86_64-linux-gnu-strip
            qemu: i386
          - build: stable-aarch64
            os: ubuntu-latest
            rust: stable
            target: aarch64-unknown-linux-gnu
            strip: aarch64-linux-gnu-strip
            qemu: qemu-aarch64
          - build: stable-powerpc64
            os: ubuntu-latest
            rust: stable
            target: powerpc64-unknown-linux-gnu
            strip: powerpc64-linux-gnu-strip
            qemu: qemu-ppc64
          - build: stable-s390x
            os: ubuntu-latest
            rust: stable
            target: s390x-unknown-linux-gnu
            strip: s390x-linux-gnu-strip
            qemu: qemu-s390x
          - build: macos
            os: macos-latest
            rust: nightly
            target: x86_64-apple-darwin
          - build: macos
            os: macos-latest
            rust: nightly
            target: aarch64-apple-darwin
          - build: win-msvc
            os: windows-latest
            rust: nightly
            target: x86_64-pc-windows-msvc
          #- build: win-gnu
          #  os: windows-latest
          #  rust: nightly-x86_64-gnu
          #  target: x86_64-pc-windows-gnu
          - build: win32-msvc
            os: windows-latest
            rust: nightly
            target: i686-pc-windows-msvc

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.create-release.outputs.version }}

      - name: Install packages (Ubuntu)
        if: matrix.os == 'ubuntu-latest'
        shell: bash
        run: |
          ci/ubuntu-install-packages

      - name: Install Rust
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ matrix.rust }}
          target: ${{ matrix.target }}

      - name: Ensure target is installed
        shell: bash
        run: rustup target add ${{ matrix.target }}

      - name: Use Cross
        if: matrix.os == 'ubuntu-latest' && matrix.target != ''
        shell: bash
        run: |
          # In the past, new releases of 'cross' have broken CI. So for now, we
          # pin it. We also use their pre-compiled binary releases because cross
          # has over 100 dependencies and takes a bit to compile.
          dir="$RUNNER_TEMP/cross-download"
          mkdir "$dir"
          echo "$dir" >> $GITHUB_PATH
          cd "$dir"
          curl -LO "https://github.com/cross-rs/cross/releases/download/$CROSS_VERSION/cross-x86_64-unknown-linux-musl.tar.gz"
          tar xf cross-x86_64-unknown-linux-musl.tar.gz
          echo "CARGO=cross" >> $GITHUB_ENV

      - name: Set target variables
        shell: bash
        run: |
          echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV
          echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV

      - name: Show command used for Cargo
        shell: bash
        run: |
          echo "cargo command is: ${{ env.CARGO }}"
          echo "target flag is: ${{ env.TARGET_FLAGS }}"
          echo "target dir is: ${{ env.TARGET_DIR }}"

      - name: Build release binary
        shell: bash
        run: |
          ${{ env.CARGO }} build --verbose --release ${{ env.TARGET_FLAGS }}
          if [ "${{ matrix.os }}" = "windows-latest" ]; then
            bin="target/${{ matrix.target }}/release/precursor.exe"
          else
            bin="target/${{ matrix.target }}/release/precursor"
          fi
          echo "BIN=$bin" >> $GITHUB_ENV

      - name: Strip release binary (macos)
        if: matrix.os == 'macos-latest'
        shell: bash
        run: strip "$BIN"

      - name: Strip release binary (cross)
        if: env.CARGO == 'cross'
        shell: bash
        run: |
          docker run --rm -v \
            "$PWD/target:/target:Z" \
            "rustembedded/cross:${{ matrix.target }}" \
            "${{ matrix.strip }}" \
            "/target/${{ matrix.target }}/release/precursor"

      - name: Determine archive name
        shell: bash
        run: |
          version="${{ needs.create-release.outputs.version }}"
          echo "ARCHIVE=precursor-$version-${{ matrix.target }}" >> $GITHUB_ENV

      - name: Creating directory for archive
        shell: bash
        run: |
          mkdir -p "$ARCHIVE"/{complete,doc}
          cp "$BIN" "$ARCHIVE"/
          cp {README.md,COPYING,UNLICENSE,LICENSE-MIT} "$ARCHIVE"/
          cp CHANGELOG.md "$ARCHIVE"/doc/

      - name: Download CLI assets
        uses: actions/download-artifact@v4
        with:
          name: precursor-cli-assets
          path: ${{ env.ARCHIVE }}

      - name: Build archive (Windows)
        shell: bash
        if: matrix.os == 'windows-latest'
        run: |
          7z a "$ARCHIVE.zip" "$ARCHIVE"
          certutil -hashfile "$ARCHIVE.zip" SHA256 > "$ARCHIVE.zip.sha256"
          echo "ASSET=$ARCHIVE.zip" >> $GITHUB_ENV
          echo "ASSET_SUM=$ARCHIVE.zip.sha256" >> $GITHUB_ENV

      - name: Build archive (Unix)
        shell: bash
        if: matrix.os != 'windows-latest'
        run: |
          tar czf "$ARCHIVE.tar.gz" "$ARCHIVE"
          shasum -a 256 "$ARCHIVE.tar.gz" > "$ARCHIVE.tar.gz.sha256"
          echo "ASSET=$ARCHIVE.tar.gz" >> $GITHUB_ENV
          echo "ASSET_SUM=$ARCHIVE.tar.gz.sha256" >> $GITHUB_ENV

      - name: Upload release archive
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          version="${{ needs.create-release.outputs.version }}"
          gh release upload --clobber "$version" ${{ env.ASSET }} ${{ env.ASSET_SUM }}

  build-release-deb:
    name: build-release-deb
    needs: ["create-release"]
    runs-on: ubuntu-latest
    env:
      TARGET: x86_64-unknown-linux-gnu
      # Emit backtraces on panics.
      RUST_BACKTRACE: 1
      # Since we're distributing the dpkg, we don't know whether the user will
      # have PCRE2 installed, so just do a static build.
      PCRE2_SYS_STATIC: 1

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.create-release.outputs.version }}

      - name: Install packages (Ubuntu)
        shell: bash
        run: |
          ci/ubuntu-install-packages

      - name: Install Rust
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly
          target: ${{ env.TARGET }}

      - name: Install cargo-deb
        shell: bash
        run: |
          cargo install cargo-deb

      - name: Build release binary
        shell: bash
        run: |
          cargo deb --target ${{ env.TARGET }}
          version="${{ needs.create-release.outputs.version }}"
          echo "DEB_DIR=target/${{ env.TARGET }}/debian" >> $GITHUB_ENV
          echo "DEB_NAME=precursor_$version-1_amd64.deb" >> $GITHUB_ENV

      - name: Create sha256 sum of deb file
        shell: bash
        run: |
          cd "$DEB_DIR"
          sum="$DEB_NAME.sha256"
          shasum -a 256 "$DEB_NAME" > "$sum"
          echo "SUM=$sum" >> $GITHUB_ENV

      - name: Upload release archive
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          cd "$DEB_DIR"
          version="${{ needs.create-release.outputs.version }}"
          gh release upload --clobber "$version" "$DEB_NAME" "$SUM"

  publish-crates:
    name: publish-crates
    needs: ["create-release", "build-release", "build-release-deb"]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.create-release.outputs.version }}

      - name: Install Rust
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable

      - name: Check whether version is already on crates.io
        id: crates_check
        shell: bash
        run: |
          set -euo pipefail
          version="${{ needs.create-release.outputs.version }}"
          status="$(curl -sS -o /dev/null -w "%{http_code}" \
            -H "User-Agent: precursor-release-ci" \
            -H "Accept: application/json" \
            "https://crates.io/api/v1/crates/precursor/$version")"
          if [ "$status" = "200" ]; then
            echo "already_published=true" >> "$GITHUB_OUTPUT"
            echo "precursor $version is already published on crates.io; skipping cargo publish."
          elif [ "$status" = "404" ]; then
            echo "already_published=false" >> "$GITHUB_OUTPUT"
            echo "precursor $version is not yet published on crates.io; proceeding."
          else
            echo "Unexpected crates.io status code: $status" >&2
            exit 1
          fi

      - name: Publish to crates.io
        if: steps.crates_check.outputs.already_published != 'true'
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
        shell: bash
        run: |
          set -euo pipefail
          cargo publish --locked

      - name: Verify crates.io publication
        shell: bash
        run: |
          set -euo pipefail
          version="${{ needs.create-release.outputs.version }}"
          for attempt in $(seq 1 18); do
            status="$(curl -sS -o /dev/null -w "%{http_code}" \
              -H "User-Agent: precursor-release-ci" \
              -H "Accept: application/json" \
              "https://crates.io/api/v1/crates/precursor/$version")"
            if [ "$status" = "200" ]; then
              echo "Confirmed precursor $version on crates.io."
              exit 0
            fi
            sleep 10
          done
          echo "Timed out waiting for precursor $version to appear on crates.io." >&2
          exit 1