fplus_lib/
parsers.rs

1use std::str::FromStr;
2
3use markdown::{mdast::Node, to_mdast, ParseOptions};
4use serde::{Deserialize, Serialize};
5
6use crate::core::application::file::{Client, DataType, Datacap, DatacapGroup, Project};
7
8#[derive(Serialize, Deserialize, Debug)]
9pub enum ParsedApplicationDataFields {
10    Version,
11    Address,
12    // Client Info
13    Name,
14    Region,
15    Industry,
16    Website,
17    SocialMedia,
18    SocialMediaType,
19    Role,
20    // Project Info
21    ProjectID,
22    ProjectBriefHistory,
23    AssociatedProjects,
24    DataDesc,
25    DataSrc,
26    DataPrepare,
27    DataSampleLink,
28    ConfirmPublicDataset,
29    RetrivalFreq,
30    DataLifeSpan,
31    DataGeographies,
32    DataDistribution,
33    ProviderIDs,
34    // Datacap Info
35    DatacapGroup,
36    Type,
37    TotalRequestedAmount,
38    SingleSizeDataset,
39    Replicas,
40    WeeklyAllocation,
41    InvalidField,
42}
43
44impl From<String> for ParsedApplicationDataFields {
45    fn from(s: String) -> Self {
46        match s.as_str() {
47	  "Version" => ParsedApplicationDataFields::Version,
48	  "On Chain Address" => ParsedApplicationDataFields::Address,
49	  // Client Info
50	  "Name" => ParsedApplicationDataFields::Name,
51	  "Region" => ParsedApplicationDataFields::Region,
52	  "Industry" => ParsedApplicationDataFields::Industry,
53	  "Website" => ParsedApplicationDataFields::Website,
54	  "Social Media" => ParsedApplicationDataFields::SocialMedia,
55	  "Social Media Type" => ParsedApplicationDataFields::SocialMediaType,
56	  "Role" => ParsedApplicationDataFields::Role,
57	  // Project Info
58	  "Project ID" => ParsedApplicationDataFields::ProjectID,
59	  "Brief history of your project and organization" => {
60		ParsedApplicationDataFields::ProjectBriefHistory
61	  }
62	  "Is this project associated with other projects/ecosystem stakeholders?" => {
63		ParsedApplicationDataFields::AssociatedProjects
64	  }
65	  "Describe the data being stored onto Filecoin" => {
66		ParsedApplicationDataFields::DataDesc
67	  },
68	  "Where was the data currently stored in this dataset sourced from"=> {
69		ParsedApplicationDataFields::DataSrc
70	  },
71	  "How do you plan to prepare the dataset" => {
72		ParsedApplicationDataFields::DataPrepare
73	  },
74	  "Please share a sample of the data (a link to a file, an image, a table, etc., are good ways to do this." => {
75		ParsedApplicationDataFields::DataSampleLink
76	  },
77	  "Confirm that this is a public dataset that can be retrieved by anyone on the network (i.e., no specific permissions or access rights are required to view the data)" => {
78		ParsedApplicationDataFields::ConfirmPublicDataset
79	  },
80	  "What is the expected retrieval frequency for this data" => {
81		ParsedApplicationDataFields::RetrivalFreq
82	  },
83	  "For how long do you plan to keep this dataset stored on Filecoin" => {
84		ParsedApplicationDataFields::DataLifeSpan
85	  },
86	  "In which geographies do you plan on making storage deals" => {
87		ParsedApplicationDataFields::DataGeographies
88	  },
89	  "How will you be distributing your data to storage providers" => {
90		ParsedApplicationDataFields::DataDistribution
91	  },
92	  "Please list the provider IDs and location of the storage providers you will be working with. Note that it is a requirement to list a minimum of 5 unique provider IDs, and that your client address will be verified against this list in the future" => {
93		ParsedApplicationDataFields::ProviderIDs
94	  },
95	  // Datacap info
96	  "Group" => ParsedApplicationDataFields::DatacapGroup,
97	  "Type" => ParsedApplicationDataFields::Type,
98	  "Total Requested Amount" => ParsedApplicationDataFields::TotalRequestedAmount,
99	  "Single Size Dataset" => ParsedApplicationDataFields::SingleSizeDataset,
100	  "Replicas" => ParsedApplicationDataFields::Replicas,
101	  "Weekly Allocation" => ParsedApplicationDataFields::WeeklyAllocation,
102	  // Invalid field
103	  _ => ParsedApplicationDataFields::InvalidField,
104	}
105    }
106}
107
108#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
109pub struct ParsedIssue {
110    pub version: u8,
111    pub id: String,
112    pub client: Client,
113    pub project: Project,
114    pub datacap: Datacap,
115}
116
117impl ParsedIssue {
118    pub fn from_issue_body(body: &str) -> Self {
119        let tree: Node = to_mdast(body, &ParseOptions::default()).unwrap();
120        let mut data: IssueValidData = IssueValidData::default();
121        for (index, i) in tree.children().unwrap().into_iter().enumerate().step_by(2) {
122            let prop = i.to_string();
123            let tree = tree.children().unwrap().into_iter();
124            let value = match tree.skip(index + 1).next() {
125                Some(v) => v.to_string(),
126                None => continue,
127            };
128            match prop.clone().into() {
129                ParsedApplicationDataFields::InvalidField => {
130                    continue;
131                }
132                _ => data.0.push((Prop(prop), Value(value))),
133            }
134        }
135        let client = Client::from(data.clone());
136        let project = Project::from(data.clone());
137        let datacap = Datacap::from(data.clone());
138        let version = data
139            .clone()
140            .0
141            .into_iter()
142            .find(|(prop, _)| prop.0 == "Version")
143            .unwrap()
144            .1
145             .0
146            .parse::<u8>()
147            .unwrap();
148        let id = data
149            .0
150            .into_iter()
151            .find(|(prop, _)| prop.0 == "On Chain Address")
152            .unwrap()
153            .1
154             .0;
155
156        Self {
157            id,
158            version,
159            client,
160            project,
161            datacap,
162        }
163    }
164}
165
166#[derive(Debug, Clone, Default)]
167pub struct Prop(pub String);
168#[derive(Debug, Clone, Default)]
169pub struct Value(pub String);
170
171#[derive(Debug, Clone, Default)]
172pub struct IssueValidData(pub Vec<(Prop, Value)>);
173
174impl From<IssueValidData> for Project {
175    fn from(data: IssueValidData) -> Self {
176        let mut project = Project::default();
177        for (prop, value) in data.0 {
178            match prop.0.into() {
179                ParsedApplicationDataFields::ProjectID => {
180                    project.project_id = value.0;
181                }
182                ParsedApplicationDataFields::ProjectBriefHistory => {
183                    project.history = value.0;
184                }
185                ParsedApplicationDataFields::AssociatedProjects => {
186                    project.associated_projects = value.0;
187                }
188                ParsedApplicationDataFields::DataDesc => {
189                    project.stored_data_desc = value.0;
190                }
191                ParsedApplicationDataFields::DataSrc => {
192                    project.previous_stoarge = value.0;
193                }
194                ParsedApplicationDataFields::DataPrepare => {
195                    project.dataset_prepare = value.0;
196                }
197                ParsedApplicationDataFields::DataSampleLink => {
198                    project.data_sample_link = value.0;
199                }
200                ParsedApplicationDataFields::ConfirmPublicDataset => {
201                    project.public_dataset = value.0;
202                }
203                ParsedApplicationDataFields::RetrivalFreq => {
204                    project.retrival_frequency = value.0;
205                }
206                ParsedApplicationDataFields::DataLifeSpan => {
207                    project.dataset_life_span = value.0;
208                }
209                ParsedApplicationDataFields::DataGeographies => {
210                    project.geographis = value.0;
211                }
212                ParsedApplicationDataFields::DataDistribution => {
213                    project.distribution = value.0;
214                }
215                ParsedApplicationDataFields::ProviderIDs => {
216                    project.providers = value.0;
217                }
218                _ => {}
219            }
220        }
221        project
222    }
223}
224
225impl From<IssueValidData> for Client {
226    fn from(data: IssueValidData) -> Self {
227        let mut client = Client::default();
228        for (prop, value) in data.0 {
229            match prop.0.into() {
230                ParsedApplicationDataFields::Name => {
231                    client.name = value.0;
232                }
233                ParsedApplicationDataFields::Region => {
234                    client.region = value.0;
235                }
236                ParsedApplicationDataFields::Industry => {
237                    client.industry = value.0;
238                }
239                ParsedApplicationDataFields::Website => {
240                    client.website = value.0;
241                }
242                ParsedApplicationDataFields::SocialMedia => {
243                    client.social_media = value.0;
244                }
245                ParsedApplicationDataFields::SocialMediaType => {
246                    client.social_media_type = value.0;
247                }
248                ParsedApplicationDataFields::Role => {
249                    client.role = value.0;
250                }
251                _ => {}
252            }
253        }
254        client
255    }
256}
257
258impl From<IssueValidData> for Datacap {
259    fn from(data: IssueValidData) -> Self {
260        let mut datacap = Datacap::default();
261        for (prop, value) in data.0 {
262            match prop.0.into() {
263                ParsedApplicationDataFields::DatacapGroup => {
264                    datacap._group = DatacapGroup::from_str(&value.0).unwrap();
265                }
266                ParsedApplicationDataFields::Type => {
267                    datacap.data_type = DataType::from_str(&value.0).unwrap();
268                }
269                ParsedApplicationDataFields::TotalRequestedAmount => {
270                    datacap.total_requested_amount = value.0;
271                }
272                ParsedApplicationDataFields::SingleSizeDataset => {
273                    datacap.single_size_dataset = value.0;
274                }
275                ParsedApplicationDataFields::Replicas => {
276                    datacap.replicas = value.0.parse::<u8>().unwrap();
277                }
278                ParsedApplicationDataFields::WeeklyAllocation => {
279                    datacap.weekly_allocation = value.0;
280                }
281                _ => {}
282            }
283        }
284        datacap
285    }
286}
287
288#[cfg(test)]
289mod tests {
290    use crate::external_services::github::GithubWrapper;
291
292    #[tokio::test]
293    async fn test_parser() {
294        let gh = GithubWrapper::new();
295        let issue = gh.list_issue(471).await.unwrap();
296        let parsed_ldn = super::ParsedIssue::from_issue_body(&issue.body.unwrap());
297
298        assert_eq!(parsed_ldn.version, 1);
299        assert!(!parsed_ldn.id.is_empty());
300
301        assert!(!parsed_ldn.client.name.is_empty());
302        assert!(!parsed_ldn.client.industry.is_empty());
303        assert!(!parsed_ldn.client.region.is_empty());
304        assert!(!parsed_ldn.client.website.is_empty());
305        assert!(!parsed_ldn.client.social_media.is_empty());
306        assert!(!parsed_ldn.client.social_media_type.is_empty());
307        assert!(!parsed_ldn.client.role.is_empty());
308        assert!(!parsed_ldn.project.project_id.is_empty());
309        assert!(!parsed_ldn.project.history.is_empty());
310        assert!(!parsed_ldn.project.associated_projects.is_empty());
311
312        assert!(!parsed_ldn.datacap.total_requested_amount.is_empty());
313    }
314}
fplus_lib/parsers.rs

fplus_lib/
parsers.rs
