xvc_pipeline/pipeline/api/
step_dependency.rs

1use std::cell::RefCell;
2use std::path::PathBuf;
3
4use crate::deps::{sqlite_query, LinesDep, RegexDep};
5use crate::error::{Error, Result};
6use crate::pipeline::deps::file::FileDep;
7use crate::pipeline::deps::generic::GenericDep;
8use crate::pipeline::deps::glob::GlobDep;
9use crate::pipeline::deps::glob_items::GlobItemsDep;
10use crate::pipeline::deps::line_items::LineItemsDep;
11use crate::pipeline::deps::regex_items::RegexItemsDep;
12use crate::pipeline::deps::step::StepDep;
13use crate::pipeline::deps::url::UrlDigestDep;
14use crate::pipeline::deps::ParamDep;
15use crate::pipeline::step::StepSubCommand;
16
17use regex::Regex;
18use url::Url;
19use xvc_core::{XvcPath, XvcRoot};
20use xvc_core::{R1NStore, XvcEntity};
21use xvc_core::{debug, XvcOutputSender};
22use xvc_core::AbsolutePath;
23
24use crate::{pipeline::deps, XvcDependency, XvcParamFormat, XvcPipeline, XvcStep};
25
26/// Entry point for `xvc pipeline step dependency` command.
27/// Add a set of new dependencies to the given step in the pipeline.
28///
29/// Unlike other entry points, this receives the options directly to avoid a long list of
30/// parameters.
31pub fn cmd_step_dependency(
32    output_snd: &XvcOutputSender,
33    xvc_root: &XvcRoot,
34    pipeline_name: &str,
35    cmd_opts: StepSubCommand,
36) -> Result<()> {
37    if let StepSubCommand::Dependency {
38        step_name,
39        generics,
40        urls,
41        files,
42        steps,
43        glob_items,
44        globs,
45        params,
46        regex_items,
47        regexes,
48        line_items,
49        lines,
50        sqlite_query,
51    } = cmd_opts
52    {
53        XvcDependencyList::new(output_snd, xvc_root, pipeline_name, &step_name)?
54            .files(files)?
55            .glob_items(glob_items)?
56            .globs(globs)?
57            .params(params)?
58            .steps(steps)?
59            .generic_commands(generics)?
60            .regexes(regexes)?
61            .regex_items(regex_items)?
62            .lines(lines)?
63            .line_items(line_items)?
64            .urls(urls)?
65            .sqlite_query(sqlite_query)?
66            .record()
67    } else {
68        Err(anyhow::anyhow!("This method is only for StepSubCommand::Dependency").into())
69    }
70}
71///
72/// Parses dependencies using member functions, in order to avoid a single function with a lot of parameters.
73pub struct XvcDependencyList<'a> {
74    output_snd: &'a XvcOutputSender,
75    xvc_root: &'a XvcRoot,
76    current_dir: &'a AbsolutePath,
77    step_e: XvcEntity,
78    step: XvcStep,
79    deps: RefCell<Vec<XvcDependency>>,
80}
81
82impl<'a> XvcDependencyList<'a> {
83    /// Create a new dependency list.
84    ///
85    /// Finds the pipeline and the step with their names, and creates a new dependency list.
86    pub fn new(
87        output_snd: &'a XvcOutputSender,
88        xvc_root: &'a XvcRoot,
89        pipeline_name: &'a str,
90        step_name: &'a str,
91    ) -> Result<Self> {
92        let current_dir = xvc_root.config().current_dir()?;
93        let (pipeline_e, _) = XvcPipeline::from_name(xvc_root, pipeline_name)?;
94        let (step_e, step) = XvcStep::from_name(xvc_root, &pipeline_e, step_name)?;
95        Ok(Self {
96            xvc_root,
97            step_e,
98            step,
99            deps: RefCell::new(Vec::new()),
100            output_snd,
101            current_dir,
102        })
103    }
104
105    /// Add file dependencies
106    pub fn files(&mut self, files: Option<Vec<String>>) -> Result<&mut Self> {
107        let current_dir = self.current_dir;
108        if let Some(files) = files {
109            let mut deps = self.deps.borrow_mut();
110            for file in files {
111                let file_dep = FileDep::new(XvcPath::new(
112                    self.xvc_root,
113                    current_dir,
114                    &PathBuf::from(file),
115                )?);
116                deps.push(XvcDependency::File(file_dep));
117            }
118        }
119        Ok(self)
120    }
121
122    /// Add glob dependencies.
123    pub fn glob_items(&mut self, glob_items: Option<Vec<String>>) -> Result<&mut Self> {
124        if let Some(globs) = glob_items {
125            let mut deps = self.deps.borrow_mut();
126            for glob in globs {
127                let glob_dep = GlobItemsDep::new(glob);
128                deps.push(XvcDependency::GlobItems(glob_dep));
129            }
130        }
131        Ok(self)
132    }
133
134    /// Add glob digest dependencies.
135    pub fn globs(&mut self, globs: Option<Vec<String>>) -> Result<&mut Self> {
136        if let Some(globs) = globs {
137            let mut deps = self.deps.borrow_mut();
138            for glob in globs {
139                let glob_dep = GlobDep::new(glob);
140                deps.push(XvcDependency::Glob(glob_dep));
141            }
142        }
143        Ok(self)
144    }
145    /// Add param dependencies.
146    ///
147    /// Param dependencies must be in the form `param_name` or
148    /// `param_file::param_name`. In the first form, `param_file` is retrieved
149    /// from [`deps::conf_params_file`].
150    pub fn params(&mut self, params: Option<Vec<String>>) -> Result<&mut Self> {
151        let current_dir = self.current_dir;
152        if let Some(params) = params {
153            let param_splitter = Regex::new(r"((?P<param_file>.*)::)?(?P<param_name>.*)").unwrap();
154            let default_param_file_name = deps::conf_params_file(self.xvc_root.config())?;
155            let mut deps = self.deps.borrow_mut();
156            for param in params {
157                let captures = match param_splitter.captures(&param) {
158                    Some(captures) => captures,
159                    None => {
160                        return Err(Error::InvalidParameterFormat { param });
161                    }
162                };
163
164                let param_file = match captures.name("param_file") {
165                    Some(param_file) => param_file.as_str(),
166                    None => default_param_file_name.as_str(),
167                };
168                let key = match captures.name("param_name") {
169                    Some(param_name) => param_name.as_str().to_string(),
170                    None => {
171                        return Err(Error::InvalidParameterFormat { param });
172                    }
173                };
174                let pathbuf = PathBuf::from(param_file);
175                let format = XvcParamFormat::from_path(&pathbuf);
176                let path = XvcPath::new(self.xvc_root, current_dir, &pathbuf)?;
177                let param_dep = ParamDep::new(&path, Some(format), key)?;
178                deps.push(XvcDependency::Param(param_dep));
179            }
180        }
181        Ok(self)
182    }
183
184    /// Add pipeline dependencies via their names.
185    ///
186    /// Note that, these are not implemented yet in the `run` command.
187    pub fn generic_commands(&mut self, generics: Option<Vec<String>>) -> Result<&mut Self> {
188        if let Some(generics) = generics {
189            let mut deps = self.deps.borrow_mut();
190            for generic_command in generics {
191                let generic_dep = GenericDep::new(generic_command);
192                deps.push(XvcDependency::Generic(generic_dep));
193            }
194        }
195        Ok(self)
196    }
197
198    /// Add step dependencies via their names.
199    pub fn steps(&mut self, steps: Option<Vec<String>>) -> Result<&mut Self> {
200        if let Some(steps) = steps {
201            let mut deps = self.deps.borrow_mut();
202            for step_name in steps {
203                let step_dep = StepDep::new(step_name);
204                deps.push(XvcDependency::Step(step_dep));
205            }
206        }
207        Ok(self)
208    }
209
210    fn split_regex_expressions(
211        &self,
212        regexes: Option<Vec<String>>,
213    ) -> Result<Vec<(XvcPath, String)>> {
214        let mut vec = Vec::new();
215        let current_dir = self.xvc_root.config().current_dir()?;
216        if let Some(regexes) = regexes {
217            let regex_splitter = Regex::new(r"(?P<regex_file>[^:/]+):/(?P<regex>.+)").unwrap();
218            for regex in regexes {
219                let captures = match regex_splitter.captures(&regex) {
220                    Some(captures) => captures,
221                    None => {
222                        return Err(Error::InvalidRegexFormat { regex });
223                    }
224                };
225
226                let regex_file = match captures.name("regex_file") {
227                    Some(regex_file) => regex_file.as_str(),
228                    None => {
229                        return Err(Error::InvalidRegexFormat { regex });
230                    }
231                };
232
233                let regex_str = match captures.name("regex") {
234                    Some(regex_str) => regex_str.as_str().to_string(),
235                    None => {
236                        return Err(Error::InvalidRegexFormat { regex });
237                    }
238                };
239
240                // Check if the supplied regexp is well formed
241                if Regex::new(&regex_str).is_err() {
242                    return Err(Error::InvalidRegexFormat { regex: regex_str });
243                }
244
245                let pathbuf = PathBuf::from(regex_file);
246                let path = XvcPath::new(self.xvc_root, current_dir, &pathbuf)?;
247                vec.push((path, regex_str));
248            }
249        }
250
251        Ok(vec)
252    }
253
254    /// Add regex dependencies.
255    ///
256    /// Regex dependencies must be in the form `regex_file:/(?P<regex>.+)`.
257    pub fn regex_items(&mut self, regex_items: Option<Vec<String>>) -> Result<&mut Self> {
258        let regex_splits = self.split_regex_expressions(regex_items)?;
259        {
260            let mut deps = self.deps.borrow_mut();
261            regex_splits.into_iter().for_each(|(path, regex_str)| {
262                let regex_dep = RegexItemsDep::new(path, regex_str);
263                deps.push(XvcDependency::RegexItems(regex_dep));
264            });
265        }
266        Ok(self)
267    }
268
269    /// Add regex dependencies.
270    ///
271    /// Regex dependencies must be in the form `regex_file:/(?P<regex>.+)`.
272    pub fn regexes(&mut self, regexes: Option<Vec<String>>) -> Result<&mut Self> {
273        let regex_splits = self.split_regex_expressions(regexes)?;
274        {
275            let mut deps = self.deps.borrow_mut();
276            regex_splits.into_iter().for_each(|(path, regex_str)| {
277                let regex_dep = RegexDep::new(path, regex_str);
278                deps.push(XvcDependency::Regex(regex_dep));
279            });
280        }
281        Ok(self)
282    }
283
284    pub fn urls(&mut self, urls: Option<Vec<String>>) -> Result<&mut Self> {
285        if let Some(urls) = urls {
286            let mut deps = self.deps.borrow_mut();
287            for url in urls {
288                let url = Url::parse(&url)?;
289                let url_dep = UrlDigestDep::new(url);
290                deps.push(XvcDependency::UrlDigest(url_dep));
291            }
292        }
293        Ok(self)
294    }
295
296    /// Add lines dependencies.
297    /// Lines dependencies must be in the form `file::begin-end`, where begin
298    /// and end are digit strings. If begin is omitted, it defaults to 0. If end
299    /// is omitted, it defaults to [usize::MAX]
300    fn split_line_options(
301        &mut self,
302        lines: Option<Vec<String>>,
303    ) -> Result<Vec<(XvcPath, usize, usize)>> {
304        let mut vec = Vec::new();
305        let current_dir = self.current_dir;
306        if let Some(lines) = lines {
307            let lines_splitter =
308                Regex::new(r"(?P<file>[^:]+)::(?P<begin>[0-9]*)-(?P<end>[0-9]*)").unwrap();
309            for line in lines {
310                let line_c = line.clone();
311                let captures = match lines_splitter.captures(&line_c) {
312                    Some(captures) => captures,
313                    None => {
314                        return Err(Error::InvalidLinesFormat { line });
315                    }
316                };
317
318                let lines_file = match captures.name("file") {
319                    Some(lines_file) => lines_file.as_str(),
320                    None => {
321                        return Err(Error::InvalidLinesFormat { line });
322                    }
323                };
324
325                let lines_begin_str = match captures.name("begin") {
326                    Some(begin_str) => begin_str.as_str().to_string(),
327                    None => {
328                        return Err(Error::InvalidLinesFormat { line });
329                    }
330                };
331
332                let lines_end_str = match captures.name("end") {
333                    Some(end_str) => end_str.as_str().to_string(),
334                    None => {
335                        return Err(Error::InvalidLinesFormat { line });
336                    }
337                };
338
339                let begin = match lines_begin_str.len() {
340                    0 => 0usize,
341                    _ => lines_begin_str
342                        .parse::<usize>()
343                        .map_err(|_| Error::InvalidLinesFormat { line: line.clone() })?,
344                };
345
346                let end = match lines_end_str.len() {
347                    0 => usize::MAX,
348                    _ => lines_end_str
349                        .parse::<usize>()
350                        .map_err(|_| Error::InvalidLinesFormat { line: line.clone() })?,
351                };
352
353                let pathbuf = PathBuf::from(lines_file);
354                let path = XvcPath::new(self.xvc_root, current_dir, &pathbuf)?;
355                vec.push((path, begin, end));
356            }
357        }
358        Ok(vec)
359    }
360
361    pub fn lines(&mut self, lines: Option<Vec<String>>) -> Result<&mut Self> {
362        let lines_options = self.split_line_options(lines)?;
363        {
364            let mut deps = self.deps.borrow_mut();
365            lines_options.into_iter().for_each(|(path, begin, end)| {
366                let lines_dep = LinesDep::new(path, begin, end);
367                deps.push(XvcDependency::Lines(lines_dep));
368            });
369        }
370        Ok(self)
371    }
372    pub fn line_items(&mut self, line_items: Option<Vec<String>>) -> Result<&mut Self> {
373        let lines_options = self.split_line_options(line_items)?;
374        {
375            let mut deps = self.deps.borrow_mut();
376            lines_options.into_iter().for_each(|(path, begin, end)| {
377                let lines_dep = LineItemsDep::new(path, begin, end);
378                deps.push(XvcDependency::LineItems(lines_dep));
379            });
380        }
381        Ok(self)
382    }
383
384    pub fn sqlite_query(&mut self, sqlite_query: Option<Vec<String>>) -> Result<&mut Self> {
385        if let Some(sqlite_query) = sqlite_query {
386            let mut deps = self.deps.borrow_mut();
387            let path = sqlite_query[0].clone();
388            let query = sqlite_query[1].clone();
389            let pathbuf = PathBuf::from(path);
390            let xvc_path = XvcPath::new(self.xvc_root, self.current_dir, &pathbuf)?;
391            let sqlite_query = sqlite_query::SqliteQueryDep::new(xvc_path, query);
392            deps.push(XvcDependency::SqliteQueryDigest(sqlite_query));
393        }
394        Ok(self)
395    }
396    /// Records dependencies the store, as children of `self.step`.
397    pub fn record(&self) -> Result<()> {
398        self.xvc_root
399            .with_r1nstore_mut(|rs: &mut R1NStore<XvcStep, XvcDependency>| {
400                let output_snd = self.output_snd;
401                for d in self.deps.borrow().iter() {
402                    debug!(output_snd, "Adding {:?}", &d);
403                    rs.insert(
404                        self.step_e,
405                        self.step.clone(),
406                        self.xvc_root.new_entity(),
407                        d.clone(),
408                    );
409                }
410                Ok(())
411            })?;
412
413        Ok(())
414    }
415}
xvc_pipeline/pipeline/api/step_dependency.rs

xvc_pipeline/pipeline/api/
step_dependency.rs
