farmap 0.9.1

A library for working with Farcaster label datasets
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

use crate::fetch::DataReadError;
use crate::fetch::InvalidJsonlError;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use serde_jsonlines::json_lines;
use std::fs::read_dir;

#[derive(Serialize, Deserialize, Debug, PartialEq)]
pub struct UnprocessedUserLine {
    provider: usize,
    r#type: Type,
    label_type: String,
    label_value: usize,
    timestamp: usize,
}

impl UnprocessedUserLine {
    pub fn provider(&self) -> usize {
        self.provider
    }

    pub fn fid(&self) -> usize {
        self.r#type.fid as usize
    }

    pub fn label_value(&self) -> usize {
        self.label_value
    }

    pub fn timestamp(&self) -> usize {
        self.timestamp
    }

    #[deprecated(note = "use local_spam_label_importer instead")]
    #[doc(hidden)]
    pub fn import_data_from_file_with_res(
        path: &str,
    ) -> Result<Vec<UnprocessedUserLine>, DataReadError> {
        let mut result: Vec<UnprocessedUserLine> = Vec::new();
        let lines_iter = json_lines::<UnprocessedUserLine, _>(path).map_err(|_| {
            DataReadError::InvalidDataPathError {
                path: path.to_string(),
            }
        })?;

        for line in lines_iter {
            let line = if let Ok(line) = line {
                line
            } else {
                return Err(DataReadError::InvalidJsonlError(InvalidJsonlError {
                    path: path.to_string(),
                }));
            };

            result.push(line);
        }
        Ok(result)
    }

    /// collects error on a line-by-line basis and sends them with an ok. Other fatal errors invoke
    /// an error.
    #[deprecated(note = "use local_spam_label_importer instead")]
    #[doc(hidden)]
    pub fn import_data_from_file_with_collected_res(
        path: &str,
    ) -> Result<Vec<Result<UnprocessedUserLine, InvalidJsonlError>>, DataReadError> {
        Ok(json_lines::<UnprocessedUserLine, _>(path)
            .map_err(|_| DataReadError::InvalidDataPathError {
                path: path.to_owned(),
            })?
            .map(|x| {
                x.map_err(|_| InvalidJsonlError {
                    path: "test".to_string(),
                })
            })
            .collect::<Vec<_>>())
    }

    #[deprecated(note = "use local_spam_label_importer instead")]
    #[allow(deprecated)]
    #[doc(hidden)]
    pub fn import_data_from_dir_with_res(
        data_dir: &str,
    ) -> Result<Vec<UnprocessedUserLine>, DataReadError> {
        let paths = read_dir(data_dir).map_err(|_| DataReadError::InvalidDataPathError {
            path: data_dir.to_string(),
        })?;

        paths
            .flatten()
            .filter(|paths| paths.path().extension().unwrap_or_default() == "jsonl")
            .map(|path| Self::import_data_from_file_with_res(path.path().to_str().unwrap()))
            .fold_ok(Vec::<UnprocessedUserLine>::new(), |mut acc, mut x| {
                acc.append(&mut x);
                acc
            })
    }
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
struct Type {
    fid: u64,
    target: String,
}