gtars_bbcache/
client.rs

1use anyhow::{Context, Ok, Result, anyhow};
2use biocrs::biocache::BioCache;
3use biocrs::models::{NewResource, Resource};
4
5use reqwest::blocking::get;
6use std::fs::{File, create_dir_all, read_dir, remove_dir, remove_file};
7use std::io::{BufRead, BufReader, Error, ErrorKind, Write};
8use std::path::{Path, PathBuf};
9
10use super::consts::{
11    DEFAULT_BEDFILE_EXT, DEFAULT_BEDFILE_SUBFOLDER, DEFAULT_BEDSET_EXT, DEFAULT_BEDSET_SUBFOLDER,
12};
13use super::utils::{get_abs_path, get_bedbase_api};
14use gtars_core::models::bed_set::BedSet;
15use gtars_core::models::region_set::RegionSet;
16
17pub struct BBClient {
18    pub cache_folder: PathBuf,
19    pub bedbase_api: String,
20    bedfile_cache: BioCache,
21    bedset_cache: BioCache,
22}
23
24impl BBClient {
25    /// BBClient to deal with download files from bedbase and caching them.
26    /// # Arguments
27    /// - cache_folder: path to local folder as cache of files from bedbase,
28    ///   if not given it will be the environment variable `BBCLIENT_CACHE`
29    /// - bedbase_api: url to bedbase
30    pub fn new(cache_folder: Option<PathBuf>, bedbase_api: Option<String>) -> Result<Self> {
31        let cache_folder = get_abs_path(cache_folder, Some(true));
32        let bedbase_api = bedbase_api.unwrap_or_else(get_bedbase_api);
33
34        let bedfile_subfolder = &cache_folder.join(DEFAULT_BEDFILE_SUBFOLDER);
35        create_dir_all(bedfile_subfolder)?;
36        let bedfile_cache = BioCache::new(bedfile_subfolder);
37
38        let bedset_subfolder = &cache_folder.join(DEFAULT_BEDSET_SUBFOLDER);
39        create_dir_all(bedset_subfolder)?;
40        let bedset_cache = BioCache::new(bedset_subfolder);
41
42        Ok(BBClient {
43            cache_folder,
44            bedbase_api,
45            bedfile_cache,
46            bedset_cache,
47        })
48    }
49
50    /// Add id and path of cached bed file or bed set into file cacher (SQLite).
51    /// # Arguments
52    /// - cache_id: id of bed file or bed set
53    /// - cache_path: path to the cached bed file or bed set
54    /// - bedfile: if the the id and file is a bed file
55    ///
56    fn add_resource_to_cache(&mut self, cache_id: &str, cache_path: &str, bedfile: bool) {
57        let resource_to_add = NewResource::new(cache_id, cache_path).set_fpath(cache_path);
58        if bedfile {
59            self.bedfile_cache.add(&resource_to_add);
60        } else {
61            self.bedset_cache.add(&resource_to_add);
62        }
63    }
64
65    /// Loads a BED file from cache, or downloads and caches it if it doesn't exist
66    /// # Arguments
67    /// - bed_id: unique identifier of a BED file
68    ///
69    /// # Returns
70    /// - the RegionSet object of the loaded bed file
71    pub fn load_bed(&mut self, bed_id: &str) -> Result<RegionSet> {
72        let bedfile_path = self.bedfile_path(bed_id, Some(false));
73
74        if bedfile_path.exists() {
75            println!("Loading cached BED file from {:?}", bedfile_path.display());
76            return RegionSet::try_from(bedfile_path);
77        }
78
79        let region_set = RegionSet::try_from(bed_id)
80            .with_context(|| format!("Failed to create RegionSet from BEDbase id {}", bed_id))?;
81
82        self.add_resource_to_cache(
83            bed_id,
84            bedfile_path.to_str().expect("Invalid BED file path"),
85            true,
86        );
87
88        region_set.to_bed_gz(bedfile_path.clone())?;
89        println!(
90            "Downloaded BED file {} from BEDbase to path: {}",
91            bed_id,
92            bedfile_path.display()
93        );
94        Ok(region_set)
95    }
96
97    /// Load a BEDset from cache, or download and add it to the cache with its BED files
98    /// # Arguments
99    /// - bedset_id: unique identifier of a BED set
100    ///
101    /// # Returns
102    /// - the BedSet object of the loaded bed set
103    pub fn load_bedset(&mut self, bedset_id: &str) -> Result<BedSet> {
104        let bedset_path = self.bedset_path(bedset_id, Some(true));
105
106        if bedset_path.exists() {
107            println!("Loading cached BED file from {:?}", bedset_path.display());
108            return BedSet::try_from(bedset_path);
109        }
110
111        let bed_data = self.download_bedset_data(bedset_id).unwrap();
112        let mut file = File::create(bedset_path.clone())?;
113        let mut region_sets = Vec::new();
114        for bbid in bed_data {
115            writeln!(file, "{}", bbid)?;
116            let rs = self.load_bed(&bbid).unwrap();
117            region_sets.push(rs);
118        }
119        self.add_resource_to_cache(
120            bedset_id,
121            bedset_path.to_str().expect("Invalid BED set path"),
122            false,
123        );
124
125        Ok(BedSet::from(region_sets))
126    }
127
128    ///  Add a BED file to the cache
129    /// # Arguments
130    /// - bedfile: a path or url to the BED file
131    /// - force: whether to overwrite the existing file in cache
132    ///
133    /// # Returns
134    /// - the RegionSet identifier
135    pub fn add_local_bed_to_cache(
136        &mut self,
137        bedfile: PathBuf,
138        force: Option<bool>,
139    ) -> Result<String> {
140        let regionset = RegionSet::try_from(bedfile.as_path())?;
141        self.add_regionset_to_cache(regionset, force)
142    }
143
144    ///  Add a RegionSet object to the cache
145    /// # Arguments
146    /// - regionset:  a RegionSet object
147    /// - force: whether to overwrite the existing file in cache
148    ///
149    /// # Returns
150    /// - the RegionSet identifier
151    pub fn add_regionset_to_cache(
152        &mut self,
153        regionset: RegionSet,
154        force: Option<bool>,
155    ) -> Result<String> {
156        let bedfile_id = regionset.identifier();
157        let cache_path = self.bedfile_path(&bedfile_id, Some(true));
158
159        let force = force.unwrap_or(false);
160        if !force && cache_path.exists() {
161            println!("{} already exists in cache", cache_path.display());
162            return Ok(bedfile_id);
163        }
164
165        regionset.to_bed_gz(cache_path.as_path())?;
166        self.add_resource_to_cache(
167            &bedfile_id,
168            cache_path
169                .to_str()
170                .expect("cache path cannot be convert to &str"),
171            true,
172        );
173        println!("BED file cached to {}", cache_path.display());
174
175        Ok(bedfile_id)
176    }
177
178    ///  Add a BED set to the cache
179    /// # Arguments
180    /// - bedset: the BED set to be added, a BedSet class
181    ///
182    /// # Returns
183    /// - the identifier if the BedSet object
184    pub fn add_bedset_to_cache(&mut self, bedset: BedSet) -> Result<String> {
185        let bedset_id = bedset.identifier();
186        let bedset_path = self.bedset_path(&bedset_id, Some(true));
187        if bedset_path.exists() {
188            println!("{} already exists in cache", bedset_path.display());
189        } else {
190            let mut file = File::create(bedset_path.clone())?;
191            for rs in bedset.region_sets {
192                let bed_id = rs.identifier();
193                let _ = self.add_regionset_to_cache(rs, Some(false));
194                writeln!(file, "{}", bed_id)?;
195            }
196        }
197
198        self.add_resource_to_cache(
199            &bedset_id,
200            bedset_path
201                .to_str()
202                .expect("cache path cannot be convert to &str"),
203            false,
204        );
205        println!("BED set cached to {}", bedset_path.display());
206
207        Ok(bedset_id)
208    }
209
210    ///  Add a folder of bed files to the cache as a bedset
211    /// # Arguments
212    /// - folder_path: path to the folder where bed files are stored
213    ///
214    /// # Returns
215    /// - the identifier if the BedSet object
216    pub fn add_local_folder_as_bedset(&mut self, folder_path: PathBuf) -> Result<String> {
217        let mut region_sets = Vec::new();
218        for entry in read_dir(&folder_path).expect("Failed to read directory") {
219            let entry = entry.expect("Failed to read directory entry");
220            let file_path = entry.path();
221
222            if file_path.is_file() {
223                let rs = RegionSet::try_from(file_path).unwrap();
224                region_sets.push(rs);
225            }
226        }
227        let bedset = BedSet::from(region_sets);
228        Ok(self.add_bedset_to_cache(bedset).unwrap())
229    }
230
231    ///  Add a local file that contains bed file paths as a bed set
232    /// # Arguments
233    /// - file_path: path to the file of bedset info
234    ///
235    /// # Returns
236    /// - the identifier if the BedSet object
237    pub fn add_local_file_as_bedset(&mut self, file_path: PathBuf) -> Result<String> {
238        let bedset = BedSet::try_from(file_path).unwrap();
239        Ok(self.add_bedset_to_cache(bedset).unwrap())
240    }
241
242    ///  Download BED set from BEDbase API and return the list of identifiers of BED files in the set
243    /// # Arguments
244    /// - bedset_id: unique identifier of a BED set
245    ///
246    /// # Returns
247    /// - the list of identifiers of BED files in the set
248    fn download_bedset_data(&self, bedset_id: &str) -> Result<Vec<String>> {
249        let bedset_url = format!("{}/v1/bedset/{}/bedfiles", self.bedbase_api, bedset_id);
250
251        let response = get(&bedset_url)?.text()?;
252
253        let json: serde_json::Value = serde_json::from_str(&response)?;
254
255        let results = json["results"]
256            .as_array()
257            .ok_or_else(|| anyhow!("`results` is not an array"))?;
258
259        let extracted_ids: Vec<String> = results
260            .iter()
261            .filter_map(|entry| {
262                let id_val = entry.get("id");
263                id_val?.as_str().map(|s| s.to_string())
264            })
265            .collect();
266
267        Ok(extracted_ids)
268    }
269
270    ///  Get the path of a BED file's .bed.gz file with given identifier
271    /// # Arguments
272    /// - bedfile_id: the identifier of BED file
273    /// - create: whether the cache path needs creating
274    ///
275    /// # Returns
276    /// - the path to the .bed.gz file
277    fn bedfile_path(&self, bedfile_id: &str, create: Option<bool>) -> PathBuf {
278        let subfolder_name = DEFAULT_BEDFILE_SUBFOLDER;
279        let file_extension = DEFAULT_BEDFILE_EXT;
280        self.cache_path(bedfile_id, subfolder_name, file_extension, create)
281    }
282
283    ///  Get the path of a BED set's .txt file with given identifier
284    /// # Arguments
285    /// - bedset_id: the identifier of BED set
286    /// - create: whether the cache path needs creating
287    ///
288    /// # Returns
289    /// - the path to the .txt file
290    fn bedset_path(&self, bedset_id: &str, create: Option<bool>) -> PathBuf {
291        let subfolder_name = DEFAULT_BEDSET_SUBFOLDER;
292        let file_extension = DEFAULT_BEDSET_EXT;
293        self.cache_path(bedset_id, subfolder_name, file_extension, create)
294    }
295
296    ///  Get the path of a file in cache folder
297    /// # Arguments
298    /// - identifier: the identifier of BED set or BED file
299    /// - subfolder_name: "bedsets" or "bedfiles"
300    /// - file_extension: ".txt" or ".bed.gz"
301    /// - create: whether the cache path needs creating
302    ///
303    /// # Returns
304    /// - the path to the file
305    fn cache_path(
306        &self,
307        identifier: &str,
308        subfolder_name: &str,
309        file_extension: &str,
310        create: Option<bool>,
311    ) -> PathBuf {
312        let filename = format!("{}{}", identifier, file_extension);
313        let folder_path = self
314            .cache_folder
315            .join(subfolder_name)
316            .join(&identifier[0..1])
317            .join(&identifier[1..2]);
318
319        if create.unwrap_or(true) {
320            self.create_cache_folder(Some(&folder_path));
321        }
322        folder_path.join(filename)
323    }
324
325    ///  Create cache folder if it doesn't exist
326    /// # Arguments
327    /// - subfolder_path: path to the subfolder
328    fn create_cache_folder(&self, subfolder_path: Option<&Path>) {
329        let path = match subfolder_path {
330            Some(p) => p.to_path_buf(),
331            None => self.cache_folder.clone(),
332        };
333
334        if !path.exists() {
335            create_dir_all(&path).expect("Failed to create cache folder");
336        }
337    }
338
339    /// Get local path to BED file or BED set with specific identifier
340    /// # Arguments
341    /// - identifier: the unique identifier
342    ///
343    /// # Returns
344    /// - the local path of the file
345    pub fn seek(&self, identifier: &str) -> Result<PathBuf> {
346        let file_path = self.bedfile_path(identifier, Some(false));
347        if file_path.exists() {
348            Ok(file_path)
349        } else {
350            let set_path = self.bedset_path(identifier, Some(false));
351            if set_path.exists() {
352                Ok(set_path)
353            } else {
354                Err(anyhow::anyhow!("{} does not exist in cache.", identifier))
355            }
356        }
357    }
358
359    /// Remove a BED file or BED set from the cache folder as well as biocfilcache (SQLite)
360    /// # Arguments
361    /// - identifier: the identifier of BED file / BED set to be removed
362    pub fn remove(&mut self, identifier: &str) -> Result<()> {
363        let file_path = self.bedfile_path(identifier, Some(false));
364        if file_path.exists() {
365            // remove file and check if subfolders is cleaned
366            let _ = self.local_removal(file_path.clone());
367            self.bedfile_cache.remove(identifier);
368
369            println!("{} is removed.", file_path.display());
370            Ok(())
371        } else {
372            let set_path = self.bedset_path(identifier, Some(false));
373            if set_path.exists() {
374                let bedset_file = File::open(set_path.clone())?;
375                let reader = BufReader::new(bedset_file);
376
377                let bed_ids: Vec<String> = reader.lines().collect::<Result<_, _>>()?;
378
379                for bed_id in bed_ids {
380                    let _ = self.remove(&bed_id);
381                }
382
383                let _ = self.local_removal(set_path.clone());
384
385                self.bedset_cache.remove(identifier);
386
387                println!("{} is removed.", set_path.display());
388                Ok(())
389            } else {
390                Err(Error::new(
391                    ErrorKind::NotFound,
392                    format!("{} does not exist in cache.", file_path.display()),
393                )
394                .into())
395            }
396        }
397    }
398
399    /// Remove a BED file or BED set from the cache folder, and make sure the removal won't cause empty subfolders
400    /// # Arguments
401    /// - identifier: the identifier of BED file / BED set to be removed
402    fn local_removal(&self, file_path: PathBuf) -> Result<()> {
403        let sub_folder_2 = file_path.parent().map(PathBuf::from);
404        let sub_folder_1 = sub_folder_2
405            .as_ref()
406            .and_then(|p| p.parent().map(PathBuf::from));
407
408        remove_file(&file_path)?;
409
410        // Attempt to remove empty subfolders
411        if let Some(sub2) = sub_folder_2
412            && read_dir(&sub2)?.next().is_none()
413        {
414            remove_dir(&sub2)?;
415            if let Some(sub1) = sub_folder_1
416                && read_dir(&sub1)?.next().is_none()
417            {
418                remove_dir(&sub1)?;
419            }
420        }
421
422        Ok(())
423    }
424
425    /// List identifiers and paths of all BED files in cache
426    /// # Returns
427    /// - the list of resource stored in biocfilecache (identifiers & paths)
428    pub fn list_beds(&mut self) -> Result<Vec<Resource>> {
429        let bed_resources = self.bedfile_cache.list_resources(Some(20_000));
430        Ok(bed_resources)
431    }
432
433    /// List identifiers and paths of all BED sets in cache
434    /// # Returns
435    /// - the list of resource stored in biocfilecache (identifiers & paths)
436    pub fn list_bedsets(&mut self) -> Result<Vec<Resource>> {
437        let bedset_resources = self.bedset_cache.list_resources(Some(20_000));
438        Ok(bedset_resources)
439    }
440}
gtars_bbcache/client.rs

gtars_bbcache/
client.rs
