1use std::collections::{HashMap, HashSet};
27use std::result::Result;
28
29mod ast;
30mod errors;
31
32use ast::AstNode;
33pub use errors::{EvalError, ParseError};
34
35pub type EvalVarMap = HashMap<String, String>;
37
38#[derive(Debug, Clone, PartialEq, Eq)]
40pub enum SourceRetrievalMethod {
41 Download { url: String },
43 ExecuteCommand {
46 command: String,
48 env: HashMap<String, String>,
50 version_ctrl: Option<String>,
52 target_path: String,
54 error_persistence_version_control: Option<String>,
62 },
63 Other { raw_var_values: EvalVarMap },
65}
66
67pub struct SrcSrvStream<'a> {
69 version: u8,
71 ini_fields: HashMap<String, &'a str>,
73 var_fields: HashMap<String, (&'a str, AstNode<'a>)>,
75 source_file_entries: HashMap<String, Vec<&'a str>>,
77}
78
79impl<'a> SrcSrvStream<'a> {
80 pub fn parse(stream: &'a [u8]) -> Result<SrcSrvStream<'a>, ParseError> {
94 let stream = std::str::from_utf8(stream).map_err(|_| ParseError::InvalidUtf8)?;
95 let mut lines = stream.lines();
96
97 let first_line = lines.next().ok_or(ParseError::UnexpectedEof)?;
99 if !first_line.starts_with("SRCSRV: ini --") {
100 return Err(ParseError::MissingIniSection);
101 }
102
103 let mut ini_fields = HashMap::new();
104 let next_section_start_line = loop {
105 let line = lines.next().ok_or(ParseError::UnexpectedEof)?;
106 if line.starts_with("SRCSRV:") {
107 break line;
108 }
109
110 let (name, value) = line.split_once('=').ok_or(ParseError::MissingEquals)?;
111 ini_fields.insert(name.to_ascii_lowercase(), value);
112 };
113
114 let version = match ini_fields.get(&"VERSION".to_ascii_lowercase()) {
115 Some(&"1") => 1,
116 Some(&"2") => 2,
117 Some(&"3") => 3,
118 Some(v) => return Err(ParseError::UnrecognizedVersion(v.to_string())),
119 None => return Err(ParseError::MissingVersion),
120 };
121
122 if !next_section_start_line.starts_with("SRCSRV: variables --") {
124 return Err(ParseError::MissingVariablesSection);
125 }
126
127 let mut var_fields = HashMap::new();
128 let next_section_start_line = loop {
129 let line = lines.next().ok_or(ParseError::UnexpectedEof)?;
130 if line.starts_with("SRCSRV:") {
131 break line;
132 }
133
134 let (name, value) = line.split_once('=').ok_or(ParseError::MissingEquals)?;
135 let node = AstNode::parse(value)?;
136 var_fields.insert(name.to_ascii_lowercase(), (value, node));
137 };
138
139 if !var_fields.contains_key(&"SRCSRVTRG".to_ascii_lowercase()) {
140 return Err(ParseError::MissingSrcSrvTrgField);
141 }
142
143 if !next_section_start_line.starts_with("SRCSRV: source files --") {
145 return Err(ParseError::MissingSourceFilesSection);
146 }
147
148 let mut source_file_entries = HashMap::new();
149 let end_line = loop {
150 let line = lines.next().ok_or(ParseError::UnexpectedEof)?;
151 if line.starts_with("SRCSRV:") {
152 break line;
153 }
154
155 let vars: Vec<&str> = line.splitn(10, '*').collect();
156 source_file_entries.insert(vars[0].to_ascii_lowercase(), vars);
157 };
158
159 if !end_line.starts_with("SRCSRV: end --") {
161 return Err(ParseError::MissingTerminationLine);
162 }
163
164 Ok(SrcSrvStream {
165 version,
166 ini_fields,
167 var_fields,
168 source_file_entries,
169 })
170 }
171
172 pub fn version(&self) -> u8 {
174 self.version
175 }
176
177 pub fn index_version(&self) -> Option<&'a str> {
179 self.ini_fields.get("indexversion").cloned()
180 }
181
182 pub fn datetime(&self) -> Option<&'a str> {
184 self.ini_fields.get("datetime").cloned()
185 }
186
187 pub fn version_control_description(&self) -> Option<&'a str> {
189 self.ini_fields.get("verctrl").cloned()
190 }
191
192 pub fn source_for_path(
217 &self,
218 original_file_path: &str,
219 extraction_base_path: &str,
220 ) -> Result<Option<SourceRetrievalMethod>, EvalError> {
221 match self.source_and_raw_var_values_for_path(original_file_path, extraction_base_path)? {
222 Some((method, _)) => Ok(Some(method)),
223 None => Ok(None),
224 }
225 }
226
227 pub fn source_and_raw_var_values_for_path(
242 &self,
243 original_file_path: &str,
244 extraction_base_path: &str,
245 ) -> Result<Option<(SourceRetrievalMethod, EvalVarMap)>, EvalError> {
246 let mut map = match self.vars_for_file(original_file_path)? {
247 Some(map) => map,
248 None => return Ok(None),
249 };
250
251 let error_persistence_version_control = self
252 .get_raw_var("SRCSRVERRVAR")
253 .and_then(|var| map.get(&var.to_ascii_lowercase()).cloned());
254
255 map.insert("targ".to_string(), extraction_base_path.to_string());
256
257 let target = self.evaluate_required_field("SRCSRVTRG", &mut map)?;
258 let command = self.evaluate_optional_field("SRCSRVCMD", &mut map)?;
259 let env = self.evaluate_optional_field("SRCSRVENV", &mut map)?;
260 let version_ctrl = self.evaluate_optional_field("SRCSRVVERCTRL", &mut map)?;
261
262 if let Some(command) = command {
263 let env = match env {
264 Some(env) => env
265 .split('\x08')
266 .filter_map(|s| s.split_once('='))
267 .map(|(envname, envval)| (envname.to_owned(), envval.to_owned()))
268 .collect(),
269 None => HashMap::new(),
270 };
271 return Ok(Some((
272 SourceRetrievalMethod::ExecuteCommand {
273 command,
274 env,
275 target_path: target,
276 version_ctrl,
277 error_persistence_version_control,
278 },
279 map,
280 )));
281 }
282
283 if target.starts_with("http://") || target.starts_with("https://") {
284 return Ok(Some((SourceRetrievalMethod::Download { url: target }, map)));
285 }
286
287 Ok(Some((
288 SourceRetrievalMethod::Other {
289 raw_var_values: map.clone(),
290 },
291 map,
292 )))
293 }
294
295 pub fn error_persistence_command_output_strings(&self) -> HashSet<&'a str> {
302 self.var_fields
303 .iter()
304 .filter_map(|(var_name, (var_value, _))| {
305 if var_name.starts_with(&"SRCSRVERRDESC".to_ascii_lowercase()) {
306 Some(*var_value)
307 } else {
308 None
309 }
310 })
311 .collect()
312 }
313
314 pub fn get_ini_field(&self, field_name: &str) -> Option<&'a str> {
317 self.ini_fields
318 .get(&field_name.to_ascii_lowercase())
319 .cloned()
320 }
321
322 pub fn get_raw_var(&self, var_name: &str) -> Option<&'a str> {
326 self.var_fields
327 .get(&var_name.to_ascii_lowercase())
328 .map(|(val, _)| *val)
329 }
330
331 fn vars_for_file(&self, file_path: &str) -> Result<Option<EvalVarMap>, EvalError> {
334 let vars = match self
335 .source_file_entries
336 .get(&file_path.to_ascii_lowercase())
337 {
338 Some(vars) => vars,
339 None => return Ok(None),
340 };
341
342 Ok(Some(
343 vars.iter()
344 .enumerate()
345 .map(|(i, var)| (format!("var{}", i + 1), var.to_string()))
346 .collect(),
347 ))
348 }
349
350 fn evaluate_optional_field(
351 &self,
352 var_name: &str,
353 var_map: &mut EvalVarMap,
354 ) -> Result<Option<String>, EvalError> {
355 let var_name = var_name.to_ascii_lowercase();
356 if !self.var_fields.contains_key(&var_name) {
357 return Ok(None);
358 }
359 let val = self.eval_impl(var_name, var_map, &mut vec![])?;
360 Ok(Some(val))
361 }
362
363 fn evaluate_required_field(
364 &self,
365 var_name: &str,
366 var_map: &mut EvalVarMap,
367 ) -> Result<String, EvalError> {
368 let var_name = var_name.to_ascii_lowercase();
369 self.eval_impl(var_name, var_map, &mut vec![])
370 }
371
372 fn eval_impl(
373 &self,
374 var_name: String,
375 var_map: &mut EvalVarMap,
376 eval_stack: &mut Vec<String>,
377 ) -> Result<String, EvalError> {
378 if let Some(val) = var_map.get(&var_name) {
379 return Ok(val.clone());
380 }
381 if eval_stack.contains(&var_name) {
382 return Err(EvalError::Recursion(var_name));
383 }
384
385 eval_stack.push(var_name.clone());
386
387 let node = match self.var_fields.get(&var_name) {
388 Some((_, node)) => node,
389 None => return Err(EvalError::UnknownVariable(var_name)),
390 };
391 let mut get_var =
392 |var_name: &str| self.eval_impl(var_name.to_ascii_lowercase(), var_map, eval_stack);
393 let eval_val = node.eval(&mut get_var)?;
394 var_map.insert(var_name, eval_val.clone());
395
396 eval_stack.pop();
397
398 Ok(eval_val)
399 }
400}
401
402#[cfg(test)]
403mod tests {
404 use std::collections::HashMap;
405
406 use crate::{SourceRetrievalMethod, SrcSrvStream};
407
408 #[test]
409 fn firefox() {
410 let stream = r#"SRCSRV: ini ------------------------------------------------
411VERSION=2
412INDEXVERSION=2
413VERCTRL=http
414SRCSRV: variables ------------------------------------------
415HGSERVER=https://hg.mozilla.org/mozilla-central
416SRCSRVVERCTRL=http
417HTTP_EXTRACT_TARGET=%hgserver%/raw-file/%var3%/%var2%
418SRCSRVTRG=%http_extract_target%
419SRCSRV: source files ---------------------------------------
420/builds/worker/checkouts/gecko/mozglue/build/SSE.cpp*mozglue/build/SSE.cpp*1706d4d54ec68fae1280305b70a02cb24c16ff68
421/builds/worker/checkouts/gecko/memory/build/mozjemalloc.cpp*memory/build/mozjemalloc.cpp*1706d4d54ec68fae1280305b70a02cb24c16ff68
422/builds/worker/checkouts/gecko/vs2017_15.8.4/VC/include/algorithm*vs2017_15.8.4/VC/include/algorithm*1706d4d54ec68fae1280305b70a02cb24c16ff68
423/builds/worker/checkouts/gecko/mozglue/baseprofiler/core/ProfilerBacktrace.cpp*mozglue/baseprofiler/core/ProfilerBacktrace.cpp*1706d4d54ec68fae1280305b70a02cb24c16ff68
424/builds/worker/workspace/obj-build/dist/include/mozilla/IntegerRange.h*mfbt/IntegerRange.h*1706d4d54ec68fae1280305b70a02cb24c16ff68
425SRCSRV: end ------------------------------------------------
426
427
428"#;
429 let stream = SrcSrvStream::parse(stream.as_bytes()).unwrap();
430 assert_eq!(stream.version(), 2);
431 assert_eq!(stream.datetime(), None);
432 assert_eq!(stream.version_control_description(), Some("http"));
433 assert_eq!(
434 stream
435 .source_for_path(
436 r#"/builds/worker/checkouts/gecko/mozglue/baseprofiler/core/ProfilerBacktrace.cpp"#,
437 r#"C:\Debugger\Cached Sources"#
438 )
439 .unwrap().unwrap(),
440 SourceRetrievalMethod::Download {
441 url: "https://hg.mozilla.org/mozilla-central/raw-file/1706d4d54ec68fae1280305b70a02cb24c16ff68/mozglue/baseprofiler/core/ProfilerBacktrace.cpp".to_string()
442 }
443 );
444 }
445
446 #[test]
447 fn chrome() {
448 let stream = r#"SRCSRV: ini ------------------------------------------------
450VERSION=1
451INDEXVERSION=2
452VERCTRL=Subversion
453DATETIME=Fri Jul 30 14:11:46 2021
454SRCSRV: variables ------------------------------------------
455SRC_EXTRACT_TARGET_DIR=%targ%\%fnbksl%(%var2%)\%var3%
456SRC_EXTRACT_TARGET=%SRC_EXTRACT_TARGET_DIR%\%fnfile%(%var1%)
457SRC_EXTRACT_CMD=cmd /c "mkdir "%SRC_EXTRACT_TARGET_DIR%" & python -c "import urllib2, base64;url = \"%var4%\";u = urllib2.urlopen(url);open(r\"%SRC_EXTRACT_TARGET%\", \"wb\").write(%var5%(u.read()))"
458SRCSRVTRG=%SRC_EXTRACT_TARGET%
459SRCSRVCMD=%SRC_EXTRACT_CMD%
460SRCSRV: source files ---------------------------------------
461c:\b\s\w\ir\cache\builder\src\third_party\pdfium\core\fdrm\fx_crypt.cpp*core/fdrm/fx_crypt.cpp*dab1161c861cc239e48a17e1a5d729aa12785a53*https://pdfium.googlesource.com/pdfium.git/+/dab1161c861cc239e48a17e1a5d729aa12785a53/core/fdrm/fx_crypt.cpp?format=TEXT*base64.b64decode
462c:\b\s\w\ir\cache\builder\src\third_party\pdfium\core\fdrm\fx_crypt_aes.cpp*core/fdrm/fx_crypt_aes.cpp*dab1161c861cc239e48a17e1a5d729aa12785a53*https://pdfium.googlesource.com/pdfium.git/+/dab1161c861cc239e48a17e1a5d729aa12785a53/core/fdrm/fx_crypt_aes.cpp?format=TEXT*base64.b64decode
463SRCSRV: end ------------------------------------------------"#;
464 let stream = SrcSrvStream::parse(stream.as_bytes()).unwrap();
465 assert_eq!(stream.version(), 1);
466 assert_eq!(stream.datetime(), Some("Fri Jul 30 14:11:46 2021"));
467 assert_eq!(stream.version_control_description(), Some("Subversion"));
468 assert_eq!(
469 stream
470 .source_for_path(
471 r#"c:\b\s\w\ir\cache\builder\src\third_party\pdfium\core\fdrm\fx_crypt.cpp"#,
472 r#"C:\Debugger\Cached Sources"#,
473 )
474 .unwrap().unwrap(),
475 SourceRetrievalMethod::ExecuteCommand {
476 command: r#"cmd /c "mkdir "C:\Debugger\Cached Sources\core\fdrm\fx_crypt.cpp\dab1161c861cc239e48a17e1a5d729aa12785a53" & python -c "import urllib2, base64;url = \"https://pdfium.googlesource.com/pdfium.git/+/dab1161c861cc239e48a17e1a5d729aa12785a53/core/fdrm/fx_crypt.cpp?format=TEXT\";u = urllib2.urlopen(url);open(r\"C:\Debugger\Cached Sources\core\fdrm\fx_crypt.cpp\dab1161c861cc239e48a17e1a5d729aa12785a53\fx_crypt.cpp\", \"wb\").write(base64.b64decode(u.read()))""#.to_string(),
477 env: HashMap::new(),
478 target_path: r#"C:\Debugger\Cached Sources\core\fdrm\fx_crypt.cpp\dab1161c861cc239e48a17e1a5d729aa12785a53\fx_crypt.cpp"#.to_string(),
479 version_ctrl: None,
480 error_persistence_version_control: None,
481 }
482 );
483 }
484
485 #[test]
486 fn team_foundation() {
487 let stream = r#"SRCSRV: ini ------------------------------------------------
489VERSION=3
490INDEXVERSION=2
491VERCTRL=Team Foundation Server
492DATETIME=Thu Mar 10 16:15:55 2016
493SRCSRV: variables ------------------------------------------
494TFS_EXTRACT_CMD=tf.exe view /version:%var4% /noprompt "$%var3%" /server:%fnvar%(%var2%) /output:%srcsrvtrg%
495TFS_EXTRACT_TARGET=%targ%\%var2%%fnbksl%(%var3%)\%var4%\%fnfile%(%var1%)
496VSTFDEVDIV_DEVDIV2=http://vstfdevdiv.redmond.corp.microsoft.com:8080/DevDiv2
497SRCSRVVERCTRL=tfs
498SRCSRVERRDESC=access
499SRCSRVERRVAR=var2
500SRCSRVTRG=%TFS_extract_target%
501SRCSRVCMD=%TFS_extract_cmd%
502SRCSRV: source files ---------------------------------------
503f:\dd\externalapis\legacy\vctools\vc12\inc\cvconst.h*VSTFDEVDIV_DEVDIV2*/DevDiv/Fx/Rel/NetFxRel3Stage/externalapis/legacy/vctools/vc12/inc/cvconst.h*1363200
504f:\dd\externalapis\legacy\vctools\vc12\inc\cvinfo.h*VSTFDEVDIV_DEVDIV2*/DevDiv/Fx/Rel/NetFxRel3Stage/externalapis/legacy/vctools/vc12/inc/cvinfo.h*1363200
505f:\dd\externalapis\legacy\vctools\vc12\inc\vc\ammintrin.h*VSTFDEVDIV_DEVDIV2*/DevDiv/Fx/Rel/NetFxRel3Stage/externalapis/legacy/vctools/vc12/inc/vc/ammintrin.h*1363200
506SRCSRV: end ------------------------------------------------"#;
507 let stream = SrcSrvStream::parse(stream.as_bytes()).unwrap();
508 assert_eq!(stream.version(), 3);
509 assert_eq!(stream.datetime(), Some("Thu Mar 10 16:15:55 2016"));
510 assert_eq!(
511 stream.version_control_description(),
512 Some("Team Foundation Server")
513 );
514 assert_eq!(
515 stream
516 .source_for_path(
517 r#"F:\dd\externalapis\legacy\vctools\vc12\inc\cvinfo.h"#,
518 r#"C:\Debugger\Cached Sources"#,
519 )
520 .unwrap().unwrap(),
521 SourceRetrievalMethod::ExecuteCommand {
522 command: r#"tf.exe view /version:1363200 /noprompt "$/DevDiv/Fx/Rel/NetFxRel3Stage/externalapis/legacy/vctools/vc12/inc/cvinfo.h" /server:http://vstfdevdiv.redmond.corp.microsoft.com:8080/DevDiv2 /output:C:\Debugger\Cached Sources\VSTFDEVDIV_DEVDIV2\DevDiv\Fx\Rel\NetFxRel3Stage\externalapis\legacy\vctools\vc12\inc\cvinfo.h\1363200\cvinfo.h"#.to_string(),
523 env: HashMap::new(),
524 version_ctrl: Some("tfs".to_string()),
525 target_path: r#"C:\Debugger\Cached Sources\VSTFDEVDIV_DEVDIV2\DevDiv\Fx\Rel\NetFxRel3Stage\externalapis\legacy\vctools\vc12\inc\cvinfo.h\1363200\cvinfo.h"#.to_string(),
526 error_persistence_version_control: Some("VSTFDEVDIV_DEVDIV2".to_string()),
527 }
528 );
529 }
530
531 #[test]
532 fn renderdoc() {
533 let stream = r#"SRCSRV: ini ------------------------------------------------
535VERSION=2
536VERCTRL=http
537SRCSRV: variables ------------------------------------------
538HTTP_ALIAS=https://raw.githubusercontent.com/baldurk/renderdoc/v1.15/
539HTTP_EXTRACT_TARGET=%HTTP_ALIAS%%var2%
540SRCSRVTRG=%HTTP_EXTRACT_TARGET%
541SRCSRV: source files ---------------------------------------
542C:\build\renderdoc\qrenderdoc\Code\BufferFormatter.cpp*qrenderdoc/Code/BufferFormatter.cpp
543C:\build\renderdoc\qrenderdoc\Windows\Dialogs\AnalyticsConfirmDialog.cpp*qrenderdoc/Windows/Dialogs/AnalyticsConfirmDialog.cpp
544C:\build\renderdoc\renderdoc\data\glsl\gl_texsample.h*renderdoc/data/glsl/gl_texsample.h
545C:\build\renderdoc\renderdoc\driver\d3d12\d3d12_device.cpp*renderdoc/driver/d3d12/d3d12_device.cpp
546C:\build\renderdoc\renderdoc\maths\matrix.cpp*renderdoc/maths/matrix.cpp
547C:\build\renderdoc\util\test\demos\texture_zoo.cpp*util/test/demos/texture_zoo.cpp
548C:\build\renderdoc\Win32\Release\renderdoc_app.h*Win32/Release/renderdoc_app.h
549C:\build\renderdoc\x64\Release\renderdoc_app.h*x64/Release/renderdoc_app.h
550SRCSRV: end ------------------------------------------------"#;
551 let stream = SrcSrvStream::parse(stream.as_bytes()).unwrap();
552 assert_eq!(stream.version(), 2);
553 assert_eq!(stream.datetime(), None);
554 assert_eq!(stream.version_control_description(), Some("http"));
555 assert_eq!(
556 stream
557 .source_for_path(
558 r#"C:\build\renderdoc\renderdoc\data\glsl\gl_texsample.h"#,
559 r#"C:\Debugger\Cached Sources"#,
560 )
561 .unwrap().unwrap(),
562 SourceRetrievalMethod::Download {
563 url: "https://raw.githubusercontent.com/baldurk/renderdoc/v1.15/renderdoc/data/glsl/gl_texsample.h".to_string(),
564 }
565 );
566 }
567}