warcat 0.3.4

Command-line tool and library for handling Web ARChive (WARC) files
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

// Example on how to write WARC files.
using System.Diagnostics;
using System.IO.Hashing;
using System.Text;
using System.Text.Json;

namespace WarcatExample;

class Encode
{
    public static void Run()
    {
        var options = Message.Options();

        // Launch the warcat program. The options provided will tell it to read
        // JSON as a line from standard in.
        // Ensure you have warcat on the search path or adjust the path as needed.
        using (var process = new Process())
        {
            process.StartInfo.FileName = "warcat";
            process.StartInfo.ArgumentList.Add("import");
            process.StartInfo.ArgumentList.Add("--compression=none");
            process.StartInfo.ArgumentList.Add("--format=jsonl");
            process.StartInfo.RedirectStandardInput = true;
            process.Start();

            // Write a record header with the given header fields.
            // Note: this header is not valid; it is simply a concise demonstration.

            var header = new Message()
            {
                Header = new Header()
                {
                    Version = "WARC/1.1",
                    Fields = [
                        ["WARC-Record-Type", "resource"],
                            ["Content-Length", "12"],
                        ]
                }
            };
            process.StandardInput.WriteLine(JsonSerializer.Serialize(header, options));

            // Write the record block data.
            var hasher = new XxHash3();

            var data = Encoding.UTF8.GetBytes("Hello world!");
            hasher.Append(data);

            var block_chunk = new Message()
            {
                BlockChunk = new BlockChunk()
                {
                    Data = data
                }
            };
            process.StandardInput.WriteLine(JsonSerializer.Serialize(block_chunk, options));

            // Write the end of the block message.
            var block_end = new Message()
            {
                BlockEnd = new BlockEnd()
                {
                    Xxh3 = hasher.GetCurrentHashAsUInt64()
                }
            };
            process.StandardInput.WriteLine(JsonSerializer.Serialize(block_end, options));

            // Finish writing the file.
            process.StandardInput.WriteLine(JsonSerializer.Serialize(new Message() { EndOfFile = new EndOfFile() }, options));
        }
    }
}