# Notes
## What do we need to build?
- Really, all we need is a unidirecitonal iterator
- TODO: check which bits of the flume api patchql uses
## How would we iterate over a go offset?
- Open all three files.
- Check the journal
- (how do we respect locking?)
- the journal gives us the number or entries in the log
- Start iterating the offset file
- map the offset file number to size.
- map the size to a slice of bytes by reading that size from the file.
Actually, that's not even needed right?
- all we do is open the data file, read a u64 as the size, then read that many bytes. Repeat.
## Testing
cryptix tried to add tests for ssb messages but failed to understand the .map(type..) stuff on the iterator and got a panic:
```
$ cargo test
...
failures:
---- go_offset_log::test::ssb_messages stdout ----
thread 'go_offset_log::test::ssb_messages' panicked at 'called `Result::unwrap()` on an `Err` value: FromUtf8Error { bytes: [134, 166, 65, 117, 116, 104, 111, 114, 130, 164, 65, 108, 103, 111, 167, 101, 100, 50, 53, 53, 49, 57, 162, 73, 68, 218, 0, 32, 252, 86, 210, 174, 217, 193, 106, 79, 195, 74, 31, 175, 157, 175, 85, 245, 187, 143, 225, 99, 213, 34, 94, 251, 142, 150, 30, 254, 168, 193, 223, 82, 163, 75, 101, 121, 130, 164, 65, 108, 103, 111, 166, 115, 104, 97, 50, 53, 54, 164, 72, 97, 115, 104, 218, 0, 32, 187, 110, 22, 14, 1, 75, 74, 100, 92, 125, 185, 28, 161, 22, 127, 164, 235, 84, 144, 227, 238, 69, 81, 5, 59, 181, 32, 35, 20, 75, 150, 173, 168, 80, 114, 101, 118, 105, 111, 117, 115, 192, 163, 82, 97, 119, 218, 1, 156, 123, 10, 32, 32, 34, 112, 114, 101, 118, 105, 111, 117, 115, 34, 58, 32, 110, 117, 108, 108, 44, 10, 32, 32, 34, 97, 117, 116, 104, 111, 114, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 34, 115, 101, 113, 117, 101, 110, 99, 101, 34, 58, 32, 49, 44, 10, 32, 32, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 32, 49, 53, 54, 53, 55, 56, 48, 51, 56, 56, 56, 49, 50, 44, 10, 32, 32, 34, 104, 97, 115, 104, 34, 58, 32, 34, 115, 104, 97, 50, 53, 54, 34, 44, 10, 32, 32, 34, 99, 111, 110, 116, 101, 110, 116, 34, 58, 32, 123, 10, 32, 32, 32, 32, 34, 97, 98, 111, 117, 116, 34, 58
, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 116, 101, 115, 116, 32, 117, 115, 101, 114, 34, 44, 10, 32, 32, 32, 32, 34, 116, 121, 112, 101, 34, 58, 32, 34, 97, 98, 111, 117, 116, 34, 10, 32, 32, 125, 44, 10, 32, 32, 34, 115, 105, 103, 110, 97, 116, 117, 114, 101, 34, 58, 32, 34, 51, 73, 99, 99, 71, 56, 116, 83, 54, 83, 70, 106, 80, 98, 114, 76, 122, 105, 81, 53, 98, 121, 69, 47, 110, 80, 101, 43, 84, 98, 105, 52, 98, 115, 121, 99, 48, 49, 82, 84, 53, 98, 69, 114, 52, 79, 122, 76, 113, 65, 114, 55, 121, 112, 111, 48, 104, 113, 74, 114, 79, 120, 116, 79, 67, 88, 106, 80, 120, 97, 88, 104, 121, 87, 117, 85, 72, 47, 106, 116, 48, 69, 80, 77, 67, 103, 61, 61, 46, 115, 105, 103, 46, 101, 100, 50, 53, 53, 49, 57, 34, 10, 125, 168, 83, 101, 113, 117, 101, 110, 99, 101, 1, 169, 84, 105, 109, 101, 115, 116, 97, 109, 112, 168, 193, 244, 94, 88, 93, 83, 233, 164], error: Utf8Error { valid_up_to: 0, error_len: Some(1) } }', src/libcore/result.rs:999:5
note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace.
```
docoding those bytes in go works though:
```go
func TestDecode(t *testing.T) {
r := require.New(t)
var input = []byte{134, 166, 65, 117, 116, 104, 111, 114, 130, 164, 65, 108, 103, 111, 167, 101, 100, 50, 53, 53, 49, 57, 162, 73, 68, 218, 0, 32, 252, 86, 210, 174, 217, 193, 106, 79, 195, 74, 31, 175, 157, 175, 85, 245, 187, 143, 225, 99, 213, 34, 94, 251, 142, 150, 30, 254, 168, 193, 223, 82, 163, 75, 101, 121, 130, 164, 65, 108, 103, 111, 166, 115, 104, 97, 50, 53, 54, 164, 72, 97, 115, 104, 218, 0, 32, 187, 110, 22, 14, 1, 75, 74, 100, 92, 125, 185, 28, 161, 22, 127, 164, 235, 84, 144, 227, 238, 69, 81, 5, 59, 181, 32, 35, 20, 75, 150, 173, 168, 80, 114, 101, 118, 105, 111, 117, 115, 192, 163, 82, 97, 119, 218, 1, 156, 123, 10, 32, 32, 34, 112, 114, 101, 118, 105, 111, 117, 115, 34, 58, 32, 110, 117, 108, 108, 44, 10, 32, 32, 34, 97, 117, 116, 104, 111, 114, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 34, 115, 101, 113, 117, 101, 110, 99, 101, 34, 58, 32, 49, 44, 10, 32, 32, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 32, 49, 53, 54, 53, 55, 56, 48, 51, 56, 56, 56, 49, 50, 44, 10, 32, 32, 34, 104, 97, 115, 104, 34, 58, 32, 34, 115, 104, 97, 50, 53, 54, 34, 44, 10, 32, 32, 34, 99, 111, 110, 116, 101, 110, 116, 34, 58, 32, 123, 10, 32, 32, 32, 32, 34, 97, 98, 111, 117, 116, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 116, 101, 115, 116, 32, 117, 115, 101, 114, 34, 44, 10, 32, 32, 32, 32, 34, 116, 121, 112, 101, 34, 58, 32, 34, 97, 98, 111, 117, 116, 34, 10, 32, 32, 125, 44, 10, 32, 32, 34, 115, 105, 103, 110, 97, 116, 117, 114, 101, 34, 58, 32, 34, 51, 73, 99, 99, 71, 56, 116, 83, 54, 83, 70, 106, 80, 98, 114, 76, 122, 105, 81, 53, 98, 121, 69, 47, 110, 80, 101, 43, 84, 98, 105, 52, 98, 115, 121, 99, 48, 49, 82, 84, 53, 98, 69, 114, 52, 79, 122, 76, 113, 65, 114, 55, 121, 112, 111, 48, 104, 113, 74, 114, 79, 120, 116, 79, 67, 88, 106, 80, 120, 97, 88, 104, 121, 87, 117, 85, 72, 47, 106, 116, 48, 69, 80, 77, 67, 103, 61, 61, 46, 115, 105, 103, 46, 101, 100, 50, 53, 53, 49, 57, 34, 10, 125, 168, 83, 101, 113, 117, 101, 110, 99, 101, 1, 169, 84, 105, 109, 101, 115, 116, 97, 109, 112, 168, 193, 244, 94, 88, 93, 83, 233, 164}
var msgt message.StoredMessage
c := New(&msgt)
v, err := c.Unmarshal(input)
r.NoError(err)
msg, ok := v.(message.StoredMessage)
r.True(ok)
t.Log(msg)
// Author and Key are not stored a a string in msgpack but a struct with the fields Algo and ID and Algo and Hash (for key)
r.Equal("@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519", msg.Author.Ref())
r.Equal("%u24WDgFLSmRcfbkcoRZ/pOtUkOPuRVEFO7UgIxRLlq0=.sha256", msg.Key.Ref())
// Raw should be an opaque byte array
rawMessage := `{
"previous": null,
"author": "@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519",
"sequence": 1,
"timestamp": 1565780388812,
"hash": "sha256",
"content": {
"about": "@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519",
"name": "test user",
"type": "about"
},
"signature": "3IccG8tS6SFjPbrLziQ5byE/nPe+Tbi4bsyc01RT5bEr4OzLqAr7ypo0hqJrOxtOCXjPxaXhyWuUH/jt0EPMCg==.sig.ed25519"
}`
r.Equal(rawMessage, string(msg.Raw))
}
```