1use std::{
8 collections::BTreeMap,
9 error::Error,
10 fmt,
11 time::{Duration, Instant},
12};
13
14use rustferry_remote::CancellationToken;
15use serde::Deserialize;
16use sha2::{Digest, Sha256};
17
18use crate::{
19 strict_json,
20 transport::{CommitSha, Repository, RunId, RunSnapshot, WorkflowId},
21};
22
23pub const MAX_JOB_LOG_BYTES: u64 = 16 * 1024 * 1024;
25pub const MAX_JOB_LOG_POLL_BYTES: u64 = 64 * 1024 * 1024;
27pub const MAX_JOB_LOG_LINE_BYTES: usize = 4_096;
29pub const MAX_WORKER_LOG_RECORDS: usize = 60_000;
31pub const MAX_WORKER_LOG_EVENTS: usize = MAX_WORKER_LOG_RECORDS - 1;
33pub const MAX_WORKER_LOG_APPEND_BATCH: usize = 256;
35pub const MAX_WORKER_LOG_PROJECTED_BYTES: u64 = 48 * 1024 * 1024;
37pub const WORKER_LOG_LINE_CODE: &str = "worker.log_line";
39pub const WORKER_LOG_PHASE: &str = "github_actions";
41pub const WORKER_LOGS_COMPLETE_CODE: &str = "worker.logs_complete";
43pub const EMPTY_JOB_LOG_LINE_MARKER: &str = "[empty GitHub log line]";
45pub const UNSAFE_JOB_LOG_LINE_MARKER: &str = "[unsafe GitHub log line redacted]";
47
48const API_HOST: &str = "api.github.com";
49const MAX_ATTEMPT: u64 = u16::MAX as u64;
50const MAX_PAGES: u16 = 100;
51const MAX_JOBS: usize = u16::MAX as usize;
52const MAX_METADATA_BYTES: usize = 16 * 1024 * 1024;
53const MAX_REDIRECTS: u8 = 5;
54const MAX_HEADER_BYTES: usize = 8 * 1024;
55const BODY_CHUNK_BYTES: usize = 64 * 1024;
56const MAX_TIMEOUT: Duration = Duration::from_hours(1);
57const MAX_POLL_TIMEOUT: Duration = Duration::from_hours(1);
58const CANONICAL_EVENT_DOMAIN: &[u8] = b"rustferry.github.worker-log.v1\0";
59const REDACTION_MARKER: &str = "<redacted>";
60const CANONICAL_JOB_SET_DOMAIN: &[u8] = b"rustferry.github.worker-log-job-set.v1\0";
61const CANONICAL_EVENT_SET_DOMAIN: &[u8] = b"rustferry.github.worker-log-event-set.v1\0";
62const CANONICAL_COMPLETION_DOMAIN: &[u8] = b"rustferry.github.worker-logs-complete.v1\0";
63const MAX_PROJECTED_STORE_EVENT_BYTES: u64 = 8 * 1024;
66const PROJECTED_STORE_EVENT_ENVELOPE_BYTES: u64 = 1024;
67
68#[derive(Clone, Copy, Debug, Eq, PartialEq)]
70pub enum GithubJobLogConfigError {
71 OutOfRange {
73 field: &'static str,
75 minimum: u64,
77 maximum: u64,
79 },
80 RunNotTerminal,
82}
83
84impl fmt::Display for GithubJobLogConfigError {
85 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
86 match self {
87 Self::OutOfRange {
88 field,
89 minimum,
90 maximum,
91 } => write!(formatter, "{field} must be between {minimum} and {maximum}"),
92 Self::RunNotTerminal => {
93 formatter.write_str("GitHub job logs require a terminal run snapshot")
94 }
95 }
96 }
97}
98
99impl Error for GithubJobLogConfigError {}
100
101#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
103pub struct GithubRunAttempt(u16);
104
105impl GithubRunAttempt {
106 pub fn new(value: u64) -> Result<Self, GithubJobLogConfigError> {
112 let value = u16::try_from(value).map_err(|_| GithubJobLogConfigError::OutOfRange {
113 field: "run attempt",
114 minimum: 1,
115 maximum: MAX_ATTEMPT,
116 })?;
117 if value == 0 {
118 return Err(GithubJobLogConfigError::OutOfRange {
119 field: "run attempt",
120 minimum: 1,
121 maximum: MAX_ATTEMPT,
122 });
123 }
124 Ok(Self(value))
125 }
126
127 pub const fn get(self) -> u16 {
129 self.0
130 }
131}
132
133#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
135pub struct GithubActionsJobId(u64);
136
137impl GithubActionsJobId {
138 pub fn new(value: u64) -> Result<Self, GithubJobLogConfigError> {
144 if value == 0 {
145 return Err(GithubJobLogConfigError::OutOfRange {
146 field: "Actions job id",
147 minimum: 1,
148 maximum: u64::MAX,
149 });
150 }
151 Ok(Self(value))
152 }
153
154 pub const fn get(self) -> u64 {
156 self.0
157 }
158}
159
160#[derive(Clone, Debug, Eq, PartialEq)]
162pub struct GithubRunAttemptLogIdentity {
163 repository: Repository,
164 run_id: RunId,
165 attempt: GithubRunAttempt,
166 head_sha: CommitSha,
167 workflow_id: WorkflowId,
168}
169
170impl GithubRunAttemptLogIdentity {
171 pub fn from_terminal_run(
177 repository: Repository,
178 run: &RunSnapshot,
179 ) -> Result<Self, GithubJobLogConfigError> {
180 if !run.status().is_terminal() || run.conclusion().is_none() {
181 return Err(GithubJobLogConfigError::RunNotTerminal);
182 }
183 Ok(Self {
184 repository,
185 run_id: run.handle().id(),
186 attempt: GithubRunAttempt::new(run.run_attempt())?,
187 head_sha: run.handle().head_sha().clone(),
188 workflow_id: run.handle().workflow_id(),
189 })
190 }
191
192 pub const fn repository(&self) -> &Repository {
194 &self.repository
195 }
196
197 pub const fn run_id(&self) -> RunId {
199 self.run_id
200 }
201
202 pub const fn attempt(&self) -> GithubRunAttempt {
204 self.attempt
205 }
206
207 pub const fn head_sha(&self) -> &CommitSha {
209 &self.head_sha
210 }
211
212 pub const fn workflow_id(&self) -> WorkflowId {
214 self.workflow_id
215 }
216
217 pub fn expected_completion_source_sequence(&self) -> u64 {
219 completion_source_sequence(self.attempt)
220 }
221
222 pub fn validate_durable_completion_marker<'a>(
228 &self,
229 worker_events: impl IntoIterator<Item = (u64, &'a str)>,
230 occurred_at_ms: u64,
231 source_sequence: u64,
232 source_event_sha256: &str,
233 message: &str,
234 ) -> Result<(), GithubJobLogError> {
235 let marker = self.validate_completion_marker_fields(
236 occurred_at_ms,
237 source_sequence,
238 source_event_sha256,
239 message,
240 )?;
241 validate_durable_event_set(
242 self.attempt,
243 marker.event_count,
244 marker.event_set_sha256,
245 worker_events,
246 )
247 }
248
249 fn validate_completion_marker_fields<'a>(
250 &self,
251 occurred_at_ms: u64,
252 source_sequence: u64,
253 source_event_sha256: &str,
254 message: &'a str,
255 ) -> Result<CompletionMarkerMessage<'a>, GithubJobLogError> {
256 let marker = parse_completion_marker_message(message)
257 .ok_or(GithubJobLogError::InvalidCompletionMarker)?;
258 if marker.run_id != self.run_id.get()
259 || marker.attempt != u64::from(self.attempt.get())
260 || marker.head_sha != self.head_sha.as_str()
261 || marker.workflow_id != self.workflow_id.get()
262 || source_sequence != self.expected_completion_source_sequence()
263 {
264 return Err(GithubJobLogError::InvalidCompletionMarker);
265 }
266 let canonical_message = completion_message(
267 self,
268 marker.job_count,
269 marker.job_set_sha256,
270 marker.event_count,
271 marker.event_set_sha256,
272 );
273 if message != canonical_message
274 || !is_lowercase_sha256(source_event_sha256)
275 || canonical_completion_sha256(
276 self,
277 occurred_at_ms,
278 marker.job_count,
279 marker.job_set_sha256,
280 marker.event_count,
281 marker.event_set_sha256,
282 source_sequence,
283 message,
284 ) != source_event_sha256
285 {
286 return Err(GithubJobLogError::InvalidCompletionMarker);
287 }
288 Ok(marker)
289 }
290}
291
292#[derive(Clone, Debug, Eq, PartialEq)]
297pub struct GithubDurableRunAttemptLogIdentity {
298 identity: GithubRunAttemptLogIdentity,
299}
300
301impl GithubDurableRunAttemptLogIdentity {
302 pub fn new(
308 repository: Repository,
309 run_id: RunId,
310 attempt: u64,
311 head_sha: CommitSha,
312 workflow_id: WorkflowId,
313 ) -> Result<Self, GithubJobLogConfigError> {
314 Ok(Self {
315 identity: GithubRunAttemptLogIdentity {
316 repository,
317 run_id,
318 attempt: GithubRunAttempt::new(attempt)?,
319 head_sha,
320 workflow_id,
321 },
322 })
323 }
324
325 pub fn expected_completion_source_sequence(&self) -> u64 {
327 self.identity.expected_completion_source_sequence()
328 }
329
330 pub fn validate_durable_completion_marker<'a>(
336 &self,
337 worker_events: impl IntoIterator<Item = (u64, &'a str)>,
338 occurred_at_ms: u64,
339 source_sequence: u64,
340 source_event_sha256: &str,
341 message: &str,
342 ) -> Result<(), GithubJobLogError> {
343 self.identity.validate_durable_completion_marker(
344 worker_events,
345 occurred_at_ms,
346 source_sequence,
347 source_event_sha256,
348 message,
349 )
350 }
351}
352
353#[derive(Clone, Copy, Debug, Eq, PartialEq)]
355pub struct GithubJobLogLimits {
356 pages: u16,
357 per_page: u8,
358 jobs: usize,
359 events: usize,
360 projected_bytes: u64,
361 metadata_bytes: usize,
362 job_bytes: u64,
363 poll_bytes: u64,
364 request_timeout: Duration,
365 poll_timeout: Duration,
366 redirects: u8,
367}
368
369impl GithubJobLogLimits {
370 #[allow(clippy::too_many_arguments)]
376 pub fn new(
377 pages: u16,
378 per_page: u8,
379 jobs: usize,
380 events: usize,
381 projected_bytes: u64,
382 metadata_bytes: usize,
383 job_bytes: u64,
384 poll_bytes: u64,
385 request_timeout: Duration,
386 poll_timeout: Duration,
387 redirects: u8,
388 ) -> Result<Self, GithubJobLogConfigError> {
389 validate_bound("log pages", u64::from(pages), 1, u64::from(MAX_PAGES))?;
390 validate_bound("log page size", u64::from(per_page), 1, 100)?;
391 validate_bound("log jobs", jobs as u64, 1, MAX_JOBS as u64)?;
392 validate_bound("log events", events as u64, 1, MAX_WORKER_LOG_EVENTS as u64)?;
393 validate_bound(
394 "projected log bytes",
395 projected_bytes,
396 1,
397 MAX_WORKER_LOG_PROJECTED_BYTES,
398 )?;
399 validate_bound(
400 "log metadata bytes",
401 metadata_bytes as u64,
402 1,
403 MAX_METADATA_BYTES as u64,
404 )?;
405 validate_bound("per-job log bytes", job_bytes, 1, MAX_JOB_LOG_BYTES)?;
406 validate_bound("per-poll log bytes", poll_bytes, 1, MAX_JOB_LOG_POLL_BYTES)?;
407 validate_duration("log request timeout", request_timeout, MAX_TIMEOUT)?;
408 validate_duration("log poll timeout", poll_timeout, MAX_POLL_TIMEOUT)?;
409 validate_bound(
410 "log redirects",
411 u64::from(redirects),
412 1,
413 u64::from(MAX_REDIRECTS),
414 )?;
415 Ok(Self {
416 pages,
417 per_page,
418 jobs,
419 events,
420 projected_bytes,
421 metadata_bytes,
422 job_bytes,
423 poll_bytes,
424 request_timeout,
425 poll_timeout,
426 redirects,
427 })
428 }
429
430 pub fn secure_defaults() -> Self {
436 Self::new(
437 10,
438 100,
439 1_000,
440 MAX_WORKER_LOG_EVENTS,
441 MAX_WORKER_LOG_PROJECTED_BYTES,
442 2 * 1024 * 1024,
443 MAX_JOB_LOG_BYTES,
444 MAX_JOB_LOG_POLL_BYTES,
445 Duration::from_mins(1),
446 Duration::from_mins(10),
447 3,
448 )
449 .expect("constant GitHub job-log limits are valid")
450 }
451}
452
453fn validate_bound(
454 field: &'static str,
455 value: u64,
456 minimum: u64,
457 maximum: u64,
458) -> Result<(), GithubJobLogConfigError> {
459 if !(minimum..=maximum).contains(&value) {
460 return Err(GithubJobLogConfigError::OutOfRange {
461 field,
462 minimum,
463 maximum,
464 });
465 }
466 Ok(())
467}
468
469fn validate_duration(
470 field: &'static str,
471 value: Duration,
472 maximum: Duration,
473) -> Result<(), GithubJobLogConfigError> {
474 if value.is_zero() || value > maximum {
475 return Err(GithubJobLogConfigError::OutOfRange {
476 field,
477 minimum: 1,
478 maximum: maximum.as_secs(),
479 });
480 }
481 Ok(())
482}
483
484#[derive(Clone, Copy, Debug, Eq, PartialEq)]
486pub enum GithubJobLogAuthorization {
487 GithubApi,
489 Omit,
491}
492
493#[derive(Clone, Copy, Debug, Eq, PartialEq)]
495pub enum GithubJobLogAccept {
496 Json,
498 PlainText,
500}
501
502pub struct GithubJobLogHttpRequest {
504 target: String,
505 api_path: bool,
506 authorization: GithubJobLogAuthorization,
507 accept: GithubJobLogAccept,
508 timeout: Duration,
509}
510
511impl fmt::Debug for GithubJobLogHttpRequest {
512 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
513 formatter
514 .debug_struct("GithubJobLogHttpRequest")
515 .field(
516 "target",
517 &if self.api_path {
518 self.target.as_str()
519 } else {
520 "<validated signed redirect URL>"
521 },
522 )
523 .field("api_path", &self.api_path)
524 .field("authorization", &self.authorization)
525 .field("accept", &self.accept)
526 .field("timeout", &self.timeout)
527 .finish()
528 }
529}
530
531impl GithubJobLogHttpRequest {
532 fn api(target: String, accept: GithubJobLogAccept, timeout: Duration) -> Self {
533 Self {
534 target,
535 api_path: true,
536 authorization: GithubJobLogAuthorization::GithubApi,
537 accept,
538 timeout,
539 }
540 }
541
542 fn redirect(target: String, timeout: Duration) -> Self {
543 Self {
544 target,
545 api_path: false,
546 authorization: GithubJobLogAuthorization::Omit,
547 accept: GithubJobLogAccept::PlainText,
548 timeout,
549 }
550 }
551
552 pub fn target(&self) -> &str {
557 &self.target
558 }
559
560 pub const fn is_api_path(&self) -> bool {
562 self.api_path
563 }
564
565 pub const fn authorization(&self) -> GithubJobLogAuthorization {
567 self.authorization
568 }
569
570 pub const fn accept(&self) -> GithubJobLogAccept {
572 self.accept
573 }
574
575 pub const fn timeout(&self) -> Duration {
577 self.timeout
578 }
579}
580
581#[derive(Clone, Copy, Debug, Eq, PartialEq)]
583pub enum GithubJobLogHttpError {
584 Cancelled,
586 TimedOut,
588 Unavailable,
590 Protocol,
592 BodyRead,
594}
595
596impl fmt::Display for GithubJobLogHttpError {
597 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
598 formatter.write_str(match self {
599 Self::Cancelled => "GitHub job-log request was cancelled",
600 Self::TimedOut => "GitHub job-log request timed out",
601 Self::Unavailable => "GitHub job-log endpoint is unavailable",
602 Self::Protocol => "GitHub job-log HTTP response was malformed",
603 Self::BodyRead => "GitHub job-log response body could not be read",
604 })
605 }
606}
607
608impl Error for GithubJobLogHttpError {}
609
610pub trait GithubJobLogHttpBody {
612 fn next_chunk(
619 &mut self,
620 maximum_bytes: usize,
621 cancellation: &CancellationToken,
622 ) -> Result<Option<Vec<u8>>, GithubJobLogHttpError>;
623}
624
625pub struct GithubJobLogHttpResponse<B> {
627 status: u16,
628 content_type: Option<String>,
629 location: Option<String>,
630 body: B,
631}
632
633impl<B> fmt::Debug for GithubJobLogHttpResponse<B> {
634 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
635 formatter
636 .debug_struct("GithubJobLogHttpResponse")
637 .field("status", &self.status)
638 .field("content_type_present", &self.content_type.is_some())
639 .field(
640 "location",
641 &self.location.as_ref().map(|_| "<redacted redirect URL>"),
642 )
643 .finish_non_exhaustive()
644 }
645}
646
647impl<B> GithubJobLogHttpResponse<B> {
648 pub fn new(
652 status: u16,
653 content_type: Option<String>,
654 location: Option<String>,
655 body: B,
656 ) -> Self {
657 Self {
658 status,
659 content_type,
660 location,
661 body,
662 }
663 }
664
665 pub const fn status(&self) -> u16 {
667 self.status
668 }
669
670 pub fn content_type(&self) -> Option<&str> {
672 self.content_type.as_deref()
673 }
674
675 pub fn location(&self) -> Option<&str> {
677 self.location.as_deref()
678 }
679
680 fn into_parts(self) -> (u16, Option<String>, Option<String>, B) {
681 (self.status, self.content_type, self.location, self.body)
682 }
683}
684
685pub trait GithubJobLogHttpClient {
687 type Body: GithubJobLogHttpBody;
689
690 fn get(
697 &mut self,
698 request: &GithubJobLogHttpRequest,
699 cancellation: &CancellationToken,
700 ) -> Result<GithubJobLogHttpResponse<Self::Body>, GithubJobLogHttpError>;
701}
702
703#[derive(Clone, Copy, Debug, Eq, PartialEq)]
705pub enum GithubJobLogError {
706 Cancelled,
708 TimedOut,
710 Transport(GithubJobLogHttpError),
712 TransportContractViolation,
714 UnexpectedStatus,
716 MalformedJobMetadata,
718 JobIdentityMismatch,
720 ConflictingJobIdentity,
722 IncompleteJobPagination,
724 JobSetNotTerminal,
726 JobPaginationLimitReached,
728 JobLimitReached,
730 InvalidRedirect,
732 RedirectHostNotAllowed,
734 RedirectLimitReached,
736 UnexpectedContentType,
738 JobLogTooLarge,
740 PollTooLarge,
742 EventLimitReached,
744 ProjectedByteLimitReached,
746 SourceSequenceOverflow,
748 InvalidCompletionMarker,
750}
751
752impl fmt::Display for GithubJobLogError {
753 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
754 match self {
755 Self::Cancelled => formatter.write_str("GitHub job-log fetch was cancelled"),
756 Self::TimedOut => formatter.write_str("GitHub job-log fetch timed out"),
757 Self::Transport(error) => error.fmt(formatter),
758 Self::TransportContractViolation => {
759 formatter.write_str("GitHub job-log transport violated its byte bound")
760 }
761 Self::UnexpectedStatus => {
762 formatter.write_str("GitHub job-log endpoint returned an unexpected status")
763 }
764 Self::MalformedJobMetadata => {
765 formatter.write_str("GitHub returned malformed job metadata")
766 }
767 Self::JobIdentityMismatch => {
768 formatter.write_str("GitHub job metadata did not match the exact run attempt")
769 }
770 Self::ConflictingJobIdentity => {
771 formatter.write_str("GitHub returned conflicting metadata for one job ID")
772 }
773 Self::IncompleteJobPagination => {
774 formatter.write_str("GitHub job pagination was incomplete")
775 }
776 Self::JobSetNotTerminal => {
777 formatter.write_str("GitHub run-attempt jobs are not all terminal")
778 }
779 Self::JobPaginationLimitReached => {
780 formatter.write_str("GitHub job pagination limit was reached")
781 }
782 Self::JobLimitReached => formatter.write_str("GitHub job count exceeds the limit"),
783 Self::InvalidRedirect => formatter.write_str("GitHub job-log redirect was invalid"),
784 Self::RedirectHostNotAllowed => {
785 formatter.write_str("GitHub job-log redirect host was not allowed")
786 }
787 Self::RedirectLimitReached => {
788 formatter.write_str("GitHub job-log redirect limit was reached")
789 }
790 Self::UnexpectedContentType => {
791 formatter.write_str("GitHub job-log response had an unexpected content type")
792 }
793 Self::JobLogTooLarge => formatter.write_str("GitHub job log exceeds the byte limit"),
794 Self::PollTooLarge => {
795 formatter.write_str("GitHub job-log poll exceeds the aggregate byte limit")
796 }
797 Self::EventLimitReached => {
798 formatter.write_str("GitHub job-log poll exceeds the event limit")
799 }
800 Self::ProjectedByteLimitReached => {
801 formatter.write_str("GitHub job-log projection exceeds the store byte reserve")
802 }
803 Self::SourceSequenceOverflow => {
804 formatter.write_str("GitHub worker log source sequence exceeds its bound")
805 }
806 Self::InvalidCompletionMarker => {
807 formatter.write_str("GitHub worker log completion marker was invalid")
808 }
809 }
810 }
811}
812
813impl Error for GithubJobLogError {}
814
815impl From<GithubJobLogHttpError> for GithubJobLogError {
816 fn from(value: GithubJobLogHttpError) -> Self {
817 match value {
818 GithubJobLogHttpError::Cancelled => Self::Cancelled,
819 GithubJobLogHttpError::TimedOut => Self::TimedOut,
820 other => Self::Transport(other),
821 }
822 }
823}
824
825#[derive(Clone, Debug, Eq, PartialEq)]
827pub struct GithubWorkerLogEvent {
828 identity: GithubRunAttemptLogIdentity,
829 job_id: GithubActionsJobId,
830 sorted_job_index: u16,
831 line_ordinal: u32,
832 occurred_at_ms: u64,
833 source_sequence: u64,
834 source_event_sha256: String,
835 message: String,
836 unsafe_line: bool,
837 projected_store_bytes: u64,
838}
839
840impl GithubWorkerLogEvent {
841 pub const fn identity(&self) -> &GithubRunAttemptLogIdentity {
843 &self.identity
844 }
845
846 pub const fn job_id(&self) -> GithubActionsJobId {
848 self.job_id
849 }
850
851 pub const fn sorted_job_index(&self) -> u16 {
853 self.sorted_job_index
854 }
855
856 pub const fn line_ordinal(&self) -> u32 {
858 self.line_ordinal
859 }
860
861 pub const fn occurred_at_ms(&self) -> u64 {
863 self.occurred_at_ms
864 }
865
866 pub const fn source_sequence(&self) -> u64 {
868 self.source_sequence
869 }
870
871 pub fn source_event_sha256(&self) -> &str {
873 &self.source_event_sha256
874 }
875
876 pub fn message(&self) -> &str {
878 &self.message
879 }
880
881 pub const fn unsafe_line(&self) -> bool {
883 self.unsafe_line
884 }
885
886 pub const fn projected_store_bytes(&self) -> u64 {
888 self.projected_store_bytes
889 }
890}
891
892#[derive(Clone, Debug, Eq, PartialEq)]
894pub struct GithubWorkerLogsComplete {
895 identity: GithubRunAttemptLogIdentity,
896 occurred_at_ms: u64,
897 job_count: u32,
898 job_set_sha256: String,
899 event_count: u64,
900 event_set_sha256: String,
901 source_sequence: u64,
902 source_event_sha256: String,
903 message: String,
904 projected_store_bytes: u64,
905}
906
907impl GithubWorkerLogsComplete {
908 pub const fn identity(&self) -> &GithubRunAttemptLogIdentity {
910 &self.identity
911 }
912
913 pub const fn code(&self) -> &'static str {
915 WORKER_LOGS_COMPLETE_CODE
916 }
917
918 pub const fn occurred_at_ms(&self) -> u64 {
920 self.occurred_at_ms
921 }
922
923 pub const fn job_count(&self) -> u32 {
925 self.job_count
926 }
927
928 pub fn job_set_sha256(&self) -> &str {
930 &self.job_set_sha256
931 }
932
933 pub const fn event_count(&self) -> u64 {
935 self.event_count
936 }
937
938 pub fn event_set_sha256(&self) -> &str {
940 &self.event_set_sha256
941 }
942
943 pub const fn source_sequence(&self) -> u64 {
945 self.source_sequence
946 }
947
948 pub fn source_event_sha256(&self) -> &str {
950 &self.source_event_sha256
951 }
952
953 pub fn message(&self) -> &str {
955 &self.message
956 }
957
958 pub const fn projected_store_bytes(&self) -> u64 {
960 self.projected_store_bytes
961 }
962}
963
964#[derive(Clone, Debug, Eq, PartialEq)]
966pub struct GithubJobLogPoll {
967 identity: GithubRunAttemptLogIdentity,
968 job_ids: Vec<GithubActionsJobId>,
969 events: Vec<GithubWorkerLogEvent>,
970 fetched_log_bytes: u64,
971 projected_store_bytes: u64,
972 completion: GithubWorkerLogsComplete,
973}
974
975impl GithubJobLogPoll {
976 pub const fn identity(&self) -> &GithubRunAttemptLogIdentity {
978 &self.identity
979 }
980
981 pub fn job_ids(&self) -> &[GithubActionsJobId] {
983 &self.job_ids
984 }
985
986 pub fn events(&self) -> &[GithubWorkerLogEvent] {
988 &self.events
989 }
990
991 pub const fn fetched_log_bytes(&self) -> u64 {
993 self.fetched_log_bytes
994 }
995
996 pub const fn projected_store_bytes(&self) -> u64 {
998 self.projected_store_bytes
999 }
1000
1001 pub fn event_batches(&self) -> impl ExactSizeIterator<Item = &[GithubWorkerLogEvent]> {
1003 self.events.chunks(MAX_WORKER_LOG_APPEND_BATCH)
1004 }
1005
1006 pub const fn completion_marker(&self) -> &GithubWorkerLogsComplete {
1008 &self.completion
1009 }
1010}
1011
1012pub struct GithubJobLogFetcher<C> {
1014 client: C,
1015 limits: GithubJobLogLimits,
1016}
1017
1018impl<C> GithubJobLogFetcher<C> {
1019 pub const fn new(client: C, limits: GithubJobLogLimits) -> Self {
1021 Self { client, limits }
1022 }
1023
1024 pub fn into_client(self) -> C {
1026 self.client
1027 }
1028
1029 pub const fn client(&self) -> &C {
1031 &self.client
1032 }
1033}
1034
1035impl<C: GithubJobLogHttpClient> GithubJobLogFetcher<C> {
1036 pub fn fetch_attempt(
1049 &mut self,
1050 identity: &GithubRunAttemptLogIdentity,
1051 cancellation: &CancellationToken,
1052 ) -> Result<GithubJobLogPoll, GithubJobLogError> {
1053 let deadline = Deadline::new(self.limits.poll_timeout);
1054 let jobs = self.list_attempt_jobs(identity, cancellation, deadline)?;
1055 let bound_jobs = jobs.values().copied().collect::<Vec<_>>();
1056 let job_ids = bound_jobs.iter().map(|job| job.id).collect::<Vec<_>>();
1057 let mut events = Vec::new();
1058 let mut poll_bytes = 0_u64;
1059 let mut projected_store_bytes = 0_u64;
1060 for (index, job) in bound_jobs.iter().copied().enumerate() {
1061 let sorted_job_index =
1062 u16::try_from(index).map_err(|_| GithubJobLogError::SourceSequenceOverflow)?;
1063 self.fetch_one_job(
1064 identity,
1065 job,
1066 sorted_job_index,
1067 cancellation,
1068 deadline,
1069 &mut poll_bytes,
1070 &mut projected_store_bytes,
1071 &mut events,
1072 )?;
1073 }
1074 check_active(cancellation, deadline)?;
1075 let completion = project_completion(identity, &bound_jobs, &events)?;
1076 check_active(cancellation, deadline)?;
1077 reserve_projected_bytes(
1078 &mut projected_store_bytes,
1079 completion.projected_store_bytes,
1080 self.limits.projected_bytes,
1081 )?;
1082 check_active(cancellation, deadline)?;
1083 Ok(GithubJobLogPoll {
1084 identity: identity.clone(),
1085 job_ids,
1086 events,
1087 fetched_log_bytes: poll_bytes,
1088 projected_store_bytes,
1089 completion,
1090 })
1091 }
1092
1093 fn list_attempt_jobs(
1094 &mut self,
1095 identity: &GithubRunAttemptLogIdentity,
1096 cancellation: &CancellationToken,
1097 deadline: Deadline,
1098 ) -> Result<BTreeMap<u64, BoundJob>, GithubJobLogError> {
1099 let mut jobs = BTreeMap::<u64, BoundJob>::new();
1100 let mut expected_total = None;
1101 for page in 1..=self.limits.pages {
1102 let endpoint = format!(
1103 "/repos/{}/{}/actions/runs/{}/attempts/{}/jobs?per_page={}&page={page}",
1104 identity.repository.owner(),
1105 identity.repository.name(),
1106 identity.run_id.get(),
1107 identity.attempt.get(),
1108 self.limits.per_page,
1109 );
1110 let request = GithubJobLogHttpRequest::api(
1111 endpoint,
1112 GithubJobLogAccept::Json,
1113 self.request_timeout(cancellation, deadline)?,
1114 );
1115 let response = self.client.get(&request, cancellation);
1116 check_active(cancellation, deadline)?;
1117 let response = response?;
1118 let (status, content_type, location, mut body) = response.into_parts();
1119 if status != 200 || location.is_some() {
1120 return Err(GithubJobLogError::UnexpectedStatus);
1121 }
1122 require_content_type(content_type.as_deref(), "application/json")?;
1123 let bytes = read_bounded_body(
1124 &mut body,
1125 self.limits.metadata_bytes as u64,
1126 cancellation,
1127 deadline,
1128 )?;
1129 let page_wire = strict_json::decode::<JobsPageWire>(&bytes, self.limits.metadata_bytes)
1130 .map_err(|_| GithubJobLogError::MalformedJobMetadata)?;
1131 check_active(cancellation, deadline)?;
1132 let total = usize::try_from(page_wire.total_count)
1133 .map_err(|_| GithubJobLogError::JobLimitReached)?;
1134 if total > self.limits.jobs {
1135 return Err(GithubJobLogError::JobLimitReached);
1136 }
1137 if expected_total
1138 .replace(total)
1139 .is_some_and(|known| known != total)
1140 || page_wire.jobs.len() > usize::from(self.limits.per_page)
1141 {
1142 return Err(GithubJobLogError::MalformedJobMetadata);
1143 }
1144 let page_len = page_wire.jobs.len();
1145 for job in page_wire.jobs {
1146 let bound = validate_job_identity(&job, identity)?;
1147 if let Some(previous) = jobs.insert(bound.id.get(), bound)
1148 && previous != bound
1149 {
1150 return Err(GithubJobLogError::ConflictingJobIdentity);
1151 }
1152 }
1153 if jobs.len() > self.limits.jobs {
1154 return Err(GithubJobLogError::JobLimitReached);
1155 }
1156 if jobs.len() == total {
1157 check_active(cancellation, deadline)?;
1158 return Ok(jobs);
1159 }
1160 if page_len < usize::from(self.limits.per_page) {
1161 return Err(GithubJobLogError::IncompleteJobPagination);
1162 }
1163 }
1164 Err(GithubJobLogError::JobPaginationLimitReached)
1165 }
1166
1167 #[allow(clippy::too_many_arguments)]
1168 fn fetch_one_job(
1169 &mut self,
1170 identity: &GithubRunAttemptLogIdentity,
1171 job: BoundJob,
1172 sorted_job_index: u16,
1173 cancellation: &CancellationToken,
1174 deadline: Deadline,
1175 poll_bytes: &mut u64,
1176 projected_store_bytes: &mut u64,
1177 events: &mut Vec<GithubWorkerLogEvent>,
1178 ) -> Result<(), GithubJobLogError> {
1179 let endpoint = format!(
1180 "/repos/{}/{}/actions/jobs/{}/logs",
1181 identity.repository.owner(),
1182 identity.repository.name(),
1183 job.id.get(),
1184 );
1185 let request = GithubJobLogHttpRequest::api(
1186 endpoint,
1187 GithubJobLogAccept::Json,
1188 self.request_timeout(cancellation, deadline)?,
1189 );
1190 let response = self.client.get(&request, cancellation);
1191 check_active(cancellation, deadline)?;
1192 let response = response?;
1193 let (status, _, location, _) = response.into_parts();
1194 if status != 302 {
1195 return Err(GithubJobLogError::UnexpectedStatus);
1196 }
1197 let mut next = location.ok_or(GithubJobLogError::InvalidRedirect)?;
1198 for hop in 1..=self.limits.redirects {
1199 validate_redirect_url(&next)?;
1200 let request = GithubJobLogHttpRequest::redirect(
1201 next,
1202 self.request_timeout(cancellation, deadline)?,
1203 );
1204 let response = self.client.get(&request, cancellation);
1205 check_active(cancellation, deadline)?;
1206 let response = response?;
1207 let (status, content_type, location, body) = response.into_parts();
1208 if status == 302 {
1209 if hop == self.limits.redirects {
1210 return Err(GithubJobLogError::RedirectLimitReached);
1211 }
1212 next = location.ok_or(GithubJobLogError::InvalidRedirect)?;
1213 continue;
1214 }
1215 if status != 200 || location.is_some() {
1216 return Err(GithubJobLogError::UnexpectedStatus);
1217 }
1218 require_content_type(content_type.as_deref(), "text/plain")?;
1219 return self.project_job_body(
1220 body,
1221 identity,
1222 job,
1223 sorted_job_index,
1224 cancellation,
1225 deadline,
1226 poll_bytes,
1227 projected_store_bytes,
1228 events,
1229 );
1230 }
1231 Err(GithubJobLogError::RedirectLimitReached)
1232 }
1233
1234 #[allow(clippy::too_many_arguments)]
1235 fn project_job_body(
1236 &self,
1237 mut body: C::Body,
1238 identity: &GithubRunAttemptLogIdentity,
1239 job: BoundJob,
1240 sorted_job_index: u16,
1241 cancellation: &CancellationToken,
1242 deadline: Deadline,
1243 poll_bytes: &mut u64,
1244 projected_store_bytes: &mut u64,
1245 events: &mut Vec<GithubWorkerLogEvent>,
1246 ) -> Result<(), GithubJobLogError> {
1247 let mut job_bytes = 0_u64;
1248 let mut decoder = LogLineDecoder::default();
1249 loop {
1250 check_active(cancellation, deadline)?;
1251 let remaining_job = self.limits.job_bytes.saturating_sub(job_bytes);
1252 let remaining_poll = self.limits.poll_bytes.saturating_sub(*poll_bytes);
1253 let requested = usize::try_from(
1254 remaining_job
1255 .min(remaining_poll)
1256 .min(BODY_CHUNK_BYTES as u64)
1257 .saturating_add(1),
1258 )
1259 .unwrap_or(BODY_CHUNK_BYTES + 1)
1260 .min(BODY_CHUNK_BYTES + 1);
1261 let chunk = body.next_chunk(requested, cancellation);
1262 check_active(cancellation, deadline)?;
1263 let chunk = chunk?;
1264 let Some(chunk) = chunk else {
1265 break;
1266 };
1267 if chunk.is_empty() || chunk.len() > requested {
1268 return Err(GithubJobLogError::TransportContractViolation);
1269 }
1270 let chunk_bytes = chunk.len() as u64;
1271 job_bytes = job_bytes
1272 .checked_add(chunk_bytes)
1273 .ok_or(GithubJobLogError::JobLogTooLarge)?;
1274 *poll_bytes = poll_bytes
1275 .checked_add(chunk_bytes)
1276 .ok_or(GithubJobLogError::PollTooLarge)?;
1277 if job_bytes > self.limits.job_bytes {
1278 return Err(GithubJobLogError::JobLogTooLarge);
1279 }
1280 if *poll_bytes > self.limits.poll_bytes {
1281 return Err(GithubJobLogError::PollTooLarge);
1282 }
1283 decoder.push(&chunk, |line_ordinal, line| {
1284 if events.len() >= self.limits.events {
1285 return Err(GithubJobLogError::EventLimitReached);
1286 }
1287 let event = project_event(identity, job, sorted_job_index, line_ordinal, line)?;
1288 reserve_projected_bytes(
1289 projected_store_bytes,
1290 event.projected_store_bytes,
1291 self.limits.projected_bytes,
1292 )?;
1293 events.push(event);
1294 Ok(())
1295 })?;
1296 }
1297 decoder.finish(|line_ordinal, line| {
1298 if events.len() >= self.limits.events {
1299 return Err(GithubJobLogError::EventLimitReached);
1300 }
1301 let event = project_event(identity, job, sorted_job_index, line_ordinal, line)?;
1302 reserve_projected_bytes(
1303 projected_store_bytes,
1304 event.projected_store_bytes,
1305 self.limits.projected_bytes,
1306 )?;
1307 events.push(event);
1308 Ok(())
1309 })?;
1310 check_active(cancellation, deadline)
1311 }
1312
1313 fn request_timeout(
1314 &self,
1315 cancellation: &CancellationToken,
1316 deadline: Deadline,
1317 ) -> Result<Duration, GithubJobLogError> {
1318 check_active(cancellation, deadline)?;
1319 request_timeout(deadline, self.limits.request_timeout)
1320 }
1321}
1322
1323#[derive(Debug, Deserialize)]
1324struct JobWire {
1325 id: u64,
1326 run_id: u64,
1327 run_attempt: Option<u64>,
1328 head_sha: String,
1329 status: String,
1330 completed_at: Option<String>,
1331}
1332
1333#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1334struct BoundJob {
1335 id: GithubActionsJobId,
1336 completed_at_ms: u64,
1337}
1338
1339#[derive(Debug, Deserialize)]
1340struct JobsPageWire {
1341 total_count: u64,
1342 jobs: Vec<JobWire>,
1343}
1344
1345fn validate_job_identity(
1346 job: &JobWire,
1347 expected: &GithubRunAttemptLogIdentity,
1348) -> Result<BoundJob, GithubJobLogError> {
1349 let id =
1350 GithubActionsJobId::new(job.id).map_err(|_| GithubJobLogError::MalformedJobMetadata)?;
1351 if job.run_id != expected.run_id.get()
1352 || job
1353 .run_attempt
1354 .is_some_and(|attempt| attempt != u64::from(expected.attempt.get()))
1355 || job.head_sha != expected.head_sha.as_str()
1356 {
1357 return Err(GithubJobLogError::JobIdentityMismatch);
1358 }
1359 if job.status != "completed" {
1360 return Err(GithubJobLogError::JobSetNotTerminal);
1361 }
1362 let completed_at_ms = job
1363 .completed_at
1364 .as_deref()
1365 .and_then(parse_rfc3339_millis)
1366 .ok_or(GithubJobLogError::MalformedJobMetadata)?;
1367 Ok(BoundJob {
1368 id,
1369 completed_at_ms,
1370 })
1371}
1372
1373fn parse_rfc3339_millis(value: &str) -> Option<u64> {
1374 let bytes = value.as_bytes();
1375 if bytes.len() < 20
1376 || bytes.get(4) != Some(&b'-')
1377 || bytes.get(7) != Some(&b'-')
1378 || bytes.get(10) != Some(&b'T')
1379 || bytes.get(13) != Some(&b':')
1380 || bytes.get(16) != Some(&b':')
1381 {
1382 return None;
1383 }
1384 let year = i64::from(parse_decimal(bytes.get(0..4)?)?);
1385 let month = parse_decimal(bytes.get(5..7)?)?;
1386 let day = parse_decimal(bytes.get(8..10)?)?;
1387 let hour = parse_decimal(bytes.get(11..13)?)?;
1388 let minute = parse_decimal(bytes.get(14..16)?)?;
1389 let second = parse_decimal(bytes.get(17..19)?)?;
1390 if year < 1970
1391 || !(1..=12).contains(&month)
1392 || day == 0
1393 || day > days_in_month(year, month)
1394 || hour > 23
1395 || minute > 59
1396 || second > 59
1397 {
1398 return None;
1399 }
1400 let mut cursor = 19;
1401 let mut milliseconds = 0_u64;
1402 if bytes.get(cursor) == Some(&b'.') {
1403 cursor += 1;
1404 let fraction_start = cursor;
1405 while bytes.get(cursor).is_some_and(u8::is_ascii_digit) {
1406 if cursor - fraction_start < 3 {
1407 milliseconds = milliseconds
1408 .checked_mul(10)?
1409 .checked_add(u64::from(bytes[cursor] - b'0'))?;
1410 }
1411 cursor += 1;
1412 }
1413 let digits = cursor - fraction_start;
1414 if digits == 0 || digits > 9 {
1415 return None;
1416 }
1417 for _ in digits..3 {
1418 milliseconds = milliseconds.checked_mul(10)?;
1419 }
1420 }
1421 let offset_seconds = match bytes.get(cursor) {
1422 Some(b'Z') if cursor + 1 == bytes.len() => 0_i64,
1423 Some(sign @ (b'+' | b'-'))
1424 if cursor + 6 == bytes.len() && bytes.get(cursor + 3) == Some(&b':') =>
1425 {
1426 let offset_hour = parse_decimal(bytes.get(cursor + 1..cursor + 3)?)?;
1427 let offset_minute = parse_decimal(bytes.get(cursor + 4..cursor + 6)?)?;
1428 if offset_hour > 23 || offset_minute > 59 {
1429 return None;
1430 }
1431 let magnitude = i64::from(offset_hour) * 3_600 + i64::from(offset_minute) * 60;
1432 if *sign == b'+' { magnitude } else { -magnitude }
1433 }
1434 _ => return None,
1435 };
1436 let days = days_from_unix_epoch(year, month, day)?;
1437 let seconds = days
1438 .checked_mul(86_400)?
1439 .checked_add(i64::from(hour * 3_600 + minute * 60 + second))?
1440 .checked_sub(offset_seconds)?;
1441 u64::try_from(seconds)
1442 .ok()?
1443 .checked_mul(1_000)?
1444 .checked_add(milliseconds)
1445}
1446
1447fn parse_decimal(bytes: &[u8]) -> Option<u32> {
1448 (!bytes.is_empty() && bytes.iter().all(u8::is_ascii_digit)).then(|| {
1449 bytes
1450 .iter()
1451 .fold(0_u32, |value, byte| value * 10 + u32::from(*byte - b'0'))
1452 })
1453}
1454
1455fn days_in_month(year: i64, month: u32) -> u32 {
1456 match month {
1457 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
1458 4 | 6 | 9 | 11 => 30,
1459 2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29,
1460 2 => 28,
1461 _ => 0,
1462 }
1463}
1464
1465fn days_from_unix_epoch(year: i64, month: u32, day: u32) -> Option<i64> {
1466 let adjusted_year = year - i64::from(month <= 2);
1467 let era = adjusted_year.div_euclid(400);
1468 let year_of_era = adjusted_year - era * 400;
1469 let adjusted_month = i64::from(month) + if month > 2 { -3 } else { 9 };
1470 let day_of_year = (153 * adjusted_month + 2) / 5 + i64::from(day) - 1;
1471 let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
1472 (era * 146_097 + day_of_era).checked_sub(719_468)
1473}
1474
1475#[derive(Clone, Copy)]
1476struct Deadline {
1477 started: Instant,
1478 timeout: Duration,
1479}
1480
1481impl Deadline {
1482 fn new(timeout: Duration) -> Self {
1483 Self {
1484 started: Instant::now(),
1485 timeout,
1486 }
1487 }
1488
1489 fn remaining(self) -> Option<Duration> {
1490 self.timeout.checked_sub(self.started.elapsed())
1491 }
1492}
1493
1494fn check_active(
1495 cancellation: &CancellationToken,
1496 deadline: Deadline,
1497) -> Result<(), GithubJobLogError> {
1498 if cancellation.is_cancelled() {
1499 return Err(GithubJobLogError::Cancelled);
1500 }
1501 if deadline
1502 .remaining()
1503 .is_none_or(|remaining| remaining.is_zero())
1504 {
1505 return Err(GithubJobLogError::TimedOut);
1506 }
1507 Ok(())
1508}
1509
1510fn request_timeout(deadline: Deadline, maximum: Duration) -> Result<Duration, GithubJobLogError> {
1511 let remaining = deadline
1512 .remaining()
1513 .filter(|remaining| !remaining.is_zero())
1514 .ok_or(GithubJobLogError::TimedOut)?;
1515 Ok(remaining.min(maximum))
1516}
1517
1518fn read_bounded_body<B: GithubJobLogHttpBody>(
1519 body: &mut B,
1520 maximum: u64,
1521 cancellation: &CancellationToken,
1522 deadline: Deadline,
1523) -> Result<Vec<u8>, GithubJobLogError> {
1524 let capacity = usize::try_from(maximum.min(BODY_CHUNK_BYTES as u64)).unwrap_or_default();
1525 let mut output = Vec::with_capacity(capacity);
1526 loop {
1527 check_active(cancellation, deadline)?;
1528 let remaining = maximum.saturating_sub(output.len() as u64);
1529 let requested = usize::try_from(remaining.min(BODY_CHUNK_BYTES as u64).saturating_add(1))
1530 .unwrap_or(BODY_CHUNK_BYTES + 1)
1531 .min(BODY_CHUNK_BYTES + 1);
1532 let chunk = body.next_chunk(requested, cancellation);
1533 check_active(cancellation, deadline)?;
1534 let chunk = chunk?;
1535 let Some(chunk) = chunk else {
1536 return Ok(output);
1537 };
1538 if chunk.is_empty() || chunk.len() > requested {
1539 return Err(GithubJobLogError::TransportContractViolation);
1540 }
1541 if output.len() as u64 + chunk.len() as u64 > maximum {
1542 return Err(GithubJobLogError::MalformedJobMetadata);
1543 }
1544 output.extend_from_slice(&chunk);
1545 }
1546}
1547
1548fn require_content_type(
1549 content_type: Option<&str>,
1550 expected: &str,
1551) -> Result<(), GithubJobLogError> {
1552 let value = content_type.ok_or(GithubJobLogError::UnexpectedContentType)?;
1553 if value.is_empty()
1554 || value.len() > MAX_HEADER_BYTES
1555 || value.bytes().any(|byte| byte.is_ascii_control())
1556 {
1557 return Err(GithubJobLogError::UnexpectedContentType);
1558 }
1559 let media_type = value
1560 .split_once(';')
1561 .map_or(value, |(media_type, _)| media_type)
1562 .trim();
1563 if !media_type.eq_ignore_ascii_case(expected) {
1564 return Err(GithubJobLogError::UnexpectedContentType);
1565 }
1566 Ok(())
1567}
1568
1569fn validate_redirect_url(value: &str) -> Result<(), GithubJobLogError> {
1570 if value.is_empty()
1571 || value.len() > MAX_HEADER_BYTES
1572 || !value.is_ascii()
1573 || value
1574 .bytes()
1575 .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace() || byte == b'\\')
1576 || value.contains('#')
1577 {
1578 return Err(GithubJobLogError::InvalidRedirect);
1579 }
1580 let remainder = value
1581 .strip_prefix("https://")
1582 .ok_or(GithubJobLogError::InvalidRedirect)?;
1583 let authority_end = remainder.find(['/', '?']).unwrap_or(remainder.len());
1584 let authority = &remainder[..authority_end];
1585 if authority.is_empty() || authority.contains(['@', ':', '%']) {
1586 return Err(GithubJobLogError::InvalidRedirect);
1587 }
1588 let host = authority.to_ascii_lowercase();
1589 if !valid_dns_host(&host) {
1590 return Err(GithubJobLogError::InvalidRedirect);
1591 }
1592 if !allowed_redirect_host(&host) {
1593 return Err(GithubJobLogError::RedirectHostNotAllowed);
1594 }
1595 Ok(())
1596}
1597
1598fn valid_dns_host(host: &str) -> bool {
1599 host.len() <= 253
1600 && !host.starts_with('.')
1601 && !host.ends_with('.')
1602 && host.split('.').all(|label| {
1603 !label.is_empty()
1604 && label.len() <= 63
1605 && !label.starts_with('-')
1606 && !label.ends_with('-')
1607 && label
1608 .bytes()
1609 .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
1610 })
1611}
1612
1613fn allowed_redirect_host(host: &str) -> bool {
1614 host == "pipelines.actions.githubusercontent.com"
1615 || host == "results-receiver.actions.githubusercontent.com"
1616 || host == "actions.githubusercontent.com"
1617 || host.ends_with(".actions.githubusercontent.com")
1618 || host == "blob.core.windows.net"
1619 || host.ends_with(".blob.core.windows.net")
1620 || host == API_HOST
1622}
1623
1624#[derive(Default)]
1625struct LogLineDecoder {
1626 pending: Vec<u8>,
1627 discarding_unsafe: bool,
1628 unsafe_block: bool,
1629 ordinal: u32,
1630}
1631
1632impl LogLineDecoder {
1633 fn push(
1634 &mut self,
1635 chunk: &[u8],
1636 mut emit: impl FnMut(u32, SanitizedLine) -> Result<(), GithubJobLogError>,
1637 ) -> Result<(), GithubJobLogError> {
1638 for byte in chunk {
1639 if *byte == b'\n' {
1640 self.emit_pending(&mut emit)?;
1641 continue;
1642 }
1643 if self.discarding_unsafe {
1644 continue;
1645 }
1646 self.pending.push(*byte);
1647 let permitted_crlf = self.pending.len() == MAX_JOB_LOG_LINE_BYTES + 1
1648 && self.pending.last() == Some(&b'\r');
1649 if self.pending.len() > MAX_JOB_LOG_LINE_BYTES && !permitted_crlf {
1650 self.pending.clear();
1651 self.discarding_unsafe = true;
1652 self.unsafe_block = true;
1653 }
1654 }
1655 Ok(())
1656 }
1657
1658 fn finish(
1659 mut self,
1660 mut emit: impl FnMut(u32, SanitizedLine) -> Result<(), GithubJobLogError>,
1661 ) -> Result<(), GithubJobLogError> {
1662 if self.discarding_unsafe || !self.pending.is_empty() {
1663 self.emit_pending(&mut emit)?;
1664 }
1665 Ok(())
1666 }
1667
1668 fn emit_pending(
1669 &mut self,
1670 emit: &mut impl FnMut(u32, SanitizedLine) -> Result<(), GithubJobLogError>,
1671 ) -> Result<(), GithubJobLogError> {
1672 self.ordinal = self
1673 .ordinal
1674 .checked_add(1)
1675 .ok_or(GithubJobLogError::SourceSequenceOverflow)?;
1676 let (opens_unsafe_block, closes_unsafe_block, unsafe_single_line) =
1677 std::str::from_utf8(&self.pending).map_or_else(
1678 |_| {
1679 let lossy = String::from_utf8_lossy(&self.pending);
1680 let (_, closes, _) = unsafe_block_transition(&lossy);
1681 (true, closes, true)
1682 },
1683 unsafe_block_transition,
1684 );
1685 let line = if self.discarding_unsafe
1686 || self.unsafe_block
1687 || opens_unsafe_block
1688 || closes_unsafe_block
1689 || unsafe_single_line
1690 {
1691 SanitizedLine::unsafe_marker()
1692 } else {
1693 if self.pending.last() == Some(&b'\r') {
1694 self.pending.pop();
1695 }
1696 sanitize_line(&self.pending)
1697 };
1698 self.unsafe_block |= opens_unsafe_block;
1699 if closes_unsafe_block && !opens_unsafe_block {
1700 self.unsafe_block = false;
1701 }
1702 self.pending.clear();
1703 self.discarding_unsafe = false;
1704 emit(self.ordinal, line)
1705 }
1706}
1707
1708struct SanitizedLine {
1709 message: String,
1710 unsafe_line: bool,
1711}
1712
1713impl SanitizedLine {
1714 fn unsafe_marker() -> Self {
1715 Self {
1716 message: UNSAFE_JOB_LOG_LINE_MARKER.to_owned(),
1717 unsafe_line: true,
1718 }
1719 }
1720}
1721
1722fn sanitize_line(bytes: &[u8]) -> SanitizedLine {
1723 if bytes.is_empty() {
1724 return SanitizedLine {
1725 message: EMPTY_JOB_LOG_LINE_MARKER.to_owned(),
1726 unsafe_line: false,
1727 };
1728 }
1729 let Ok(text) = std::str::from_utf8(bytes) else {
1730 return SanitizedLine::unsafe_marker();
1731 };
1732 let (opens_unsafe_block, closes_unsafe_block, unsafe_single_line) =
1733 unsafe_block_transition(text);
1734 if text.chars().any(is_unsafe_log_character)
1735 || opens_unsafe_block
1736 || closes_unsafe_block
1737 || unsafe_single_line
1738 || contains_sensitive_assignment_or_authorization(text)
1739 || contains_opaque_credential_shaped_token(text)
1740 {
1741 return SanitizedLine::unsafe_marker();
1742 }
1743 let message = redact_line(text);
1744 if message.is_empty()
1745 || message.len() > MAX_JOB_LOG_LINE_BYTES
1746 || message.chars().any(char::is_control)
1747 || output_still_unsafe(&message)
1748 || projected_store_record_bytes(&message) > MAX_PROJECTED_STORE_EVENT_BYTES
1749 {
1750 return SanitizedLine::unsafe_marker();
1751 }
1752 SanitizedLine {
1753 message,
1754 unsafe_line: false,
1755 }
1756}
1757
1758fn unsafe_block_transition(text: &str) -> (bool, bool, bool) {
1759 let lower = text.trim().to_ascii_lowercase();
1760 let opens = lower.contains("-----begin ")
1761 || lower.contains("---- begin ")
1762 || lower.contains("begin ssh2 encrypted private key")
1763 || lower.contains("putty-user-key-file-")
1764 || lower.contains("private-lines:");
1765 let closes = lower.contains("-----end ")
1766 || lower.contains("---- end ")
1767 || lower.contains("end ssh2 encrypted private key")
1768 || lower.contains("private-mac:");
1769 let workflow_command = [
1770 "::add-mask::",
1771 "::set-output ",
1772 "::save-state ",
1773 "::set-secret::",
1774 "::stop-commands::",
1775 ]
1776 .iter()
1777 .any(|marker| lower.contains(marker));
1778 (opens, closes, workflow_command)
1779}
1780
1781fn contains_opaque_credential_shaped_token(text: &str) -> bool {
1782 text.split(|character: char| {
1783 !character.is_ascii_alphanumeric() && !matches!(character, '+' | '/' | '_' | '-' | '=')
1784 })
1785 .any(|value| value.len() >= 20)
1786}
1787
1788fn contains_sensitive_assignment_or_authorization(text: &str) -> bool {
1789 let bytes = text.as_bytes();
1790 let lower = text.to_ascii_lowercase();
1791 for (separator, byte) in bytes.iter().copied().enumerate() {
1792 if !matches!(byte, b'=' | b':') {
1793 continue;
1794 }
1795 let mut key_end = separator;
1796 while key_end > 0 && bytes[key_end - 1].is_ascii_whitespace() {
1797 key_end -= 1;
1798 }
1799 let quote = key_end
1800 .checked_sub(1)
1801 .and_then(|index| bytes.get(index))
1802 .copied()
1803 .filter(|byte| matches!(byte, b'\'' | b'"'));
1804 if quote.is_some() {
1805 key_end -= 1;
1806 }
1807 let mut key_start = key_end;
1808 while key_start > 0
1809 && (bytes[key_start - 1].is_ascii_alphanumeric()
1810 || matches!(bytes[key_start - 1], b'_' | b'-' | b'.'))
1811 {
1812 key_start -= 1;
1813 }
1814 if key_start == key_end || !sensitive_assignment_name(&lower[key_start..key_end]) {
1815 continue;
1816 }
1817 return true;
1818 }
1819 has_space_delimited_sensitive_value(bytes, &lower)
1820}
1821
1822fn has_space_delimited_sensitive_value(bytes: &[u8], lower: &str) -> bool {
1823 let mut cursor = 0;
1824 while cursor < bytes.len() {
1825 if !bytes[cursor].is_ascii_alphanumeric() {
1826 cursor += 1;
1827 continue;
1828 }
1829 let name_start = cursor;
1830 while cursor < bytes.len()
1831 && (bytes[cursor].is_ascii_alphanumeric()
1832 || matches!(bytes[cursor], b'_' | b'-' | b'.'))
1833 {
1834 cursor += 1;
1835 }
1836 let name = &lower[name_start..cursor];
1837 let sensitive_name = sensitive_assignment_name(name) || matches!(name, "bearer" | "basic");
1838 let mut value_start = cursor;
1839 while value_start < bytes.len() && bytes[value_start].is_ascii_whitespace() {
1840 value_start += 1;
1841 }
1842 if sensitive_name && value_start > cursor && value_start < bytes.len() {
1843 return true;
1844 }
1845 }
1846 false
1847}
1848
1849fn is_unsafe_log_character(character: char) -> bool {
1850 character.is_control()
1851 || matches!(
1852 character,
1853 '\u{200b}'..='\u{200f}'
1854 | '\u{202a}'..='\u{202e}'
1855 | '\u{2060}'
1856 | '\u{2066}'..='\u{2069}'
1857 | '\u{feff}'
1858 )
1859}
1860
1861fn redact_line(text: &str) -> String {
1862 let bytes = text.as_bytes();
1863 let lower = text.to_ascii_lowercase();
1864 let mut mask = vec![false; bytes.len()];
1865 redact_urls(bytes, &mut mask);
1866 redact_github_tokens(bytes, &lower, &mut mask);
1867 redact_jwts(bytes, &mut mask);
1868 redact_sensitive_named_assignments(bytes, &lower, &mut mask);
1869 redact_sensitive_assignments(bytes, &lower, &mut mask);
1870 redact_authorization_schemes(bytes, &lower, &mut mask);
1871 render_masked(text, &mask)
1872}
1873
1874fn redact_urls(bytes: &[u8], mask: &mut [bool]) {
1875 let mut cursor = 0;
1876 while let Some(relative) = find_bytes(&bytes[cursor..], b"://") {
1877 let scheme_end = cursor + relative;
1878 let mut scheme_start = scheme_end;
1879 while scheme_start > 0
1880 && (bytes[scheme_start - 1].is_ascii_alphanumeric()
1881 || matches!(bytes[scheme_start - 1], b'+' | b'-' | b'.'))
1882 {
1883 scheme_start -= 1;
1884 }
1885 if scheme_start == scheme_end || !bytes[scheme_start].is_ascii_alphabetic() {
1886 cursor = scheme_end + 3;
1887 continue;
1888 }
1889 let url_start = scheme_start;
1890 let url_body_start = scheme_end + 3;
1891 let end = bytes[url_body_start..]
1892 .iter()
1893 .position(u8::is_ascii_whitespace)
1894 .map_or(bytes.len(), |offset| url_body_start + offset);
1895 mark(mask, url_start, end);
1898 cursor = end;
1899 }
1900
1901 cursor = 0;
1902 while let Some(relative) = find_bytes(&bytes[cursor..], b"//") {
1903 let start = cursor + relative;
1904 if start > 0 && bytes[start - 1] == b':' {
1905 cursor = start + 2;
1906 continue;
1907 }
1908 let end = bytes[start + 2..]
1909 .iter()
1910 .position(u8::is_ascii_whitespace)
1911 .map_or(bytes.len(), |offset| start + 2 + offset);
1912 let authority_end = bytes[start + 2..end]
1913 .iter()
1914 .position(|byte| matches!(byte, b'/' | b'?' | b'#'))
1915 .map_or(end, |offset| start + 2 + offset);
1916 if bytes[start + 2..authority_end].contains(&b'@') {
1917 mark(mask, start, end);
1918 }
1919 cursor = end.max(start + 2);
1920 }
1921}
1922
1923fn redact_github_tokens(bytes: &[u8], lower: &str, mask: &mut [bool]) {
1924 for prefix in ["github_pat_", "ghp_", "gho_", "ghu_", "ghs_", "ghr_"] {
1925 let mut cursor = 0;
1926 while let Some(relative) = lower[cursor..].find(prefix) {
1927 let start = cursor + relative;
1928 let mut end = start + prefix.len();
1929 while end < bytes.len() && (bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_') {
1930 end += 1;
1931 }
1932 mark(mask, start, end);
1933 cursor = end;
1934 }
1935 }
1936}
1937
1938fn redact_jwts(bytes: &[u8], mask: &mut [bool]) {
1939 let mut cursor = 0;
1940 while cursor + 4 < bytes.len() {
1941 if bytes[cursor..].starts_with(b"eyJ") && is_token_boundary(bytes, cursor) {
1942 let mut end = cursor;
1943 let mut dots = 0;
1944 while end < bytes.len()
1945 && (bytes[end].is_ascii_alphanumeric()
1946 || matches!(bytes[end], b'_' | b'-' | b'.' | b'='))
1947 {
1948 dots += usize::from(bytes[end] == b'.');
1949 end += 1;
1950 }
1951 if dots == 2 && end.saturating_sub(cursor) >= 20 {
1952 mark(mask, cursor, end);
1953 }
1954 cursor = end.max(cursor + 1);
1955 } else {
1956 cursor += 1;
1957 }
1958 }
1959}
1960
1961fn redact_sensitive_named_assignments(bytes: &[u8], lower: &str, mask: &mut [bool]) {
1962 for (separator, byte) in bytes.iter().copied().enumerate() {
1963 if !matches!(byte, b'=' | b':') {
1964 continue;
1965 }
1966 let mut key_end = separator;
1967 while key_end > 0 && bytes[key_end - 1].is_ascii_whitespace() {
1968 key_end -= 1;
1969 }
1970 let mut key_start = key_end;
1971 while key_start > 0
1972 && (bytes[key_start - 1].is_ascii_alphanumeric()
1973 || matches!(bytes[key_start - 1], b'_' | b'-' | b'.'))
1974 {
1975 key_start -= 1;
1976 }
1977 if key_start == key_end || !sensitive_assignment_name(&lower[key_start..key_end]) {
1978 continue;
1979 }
1980 let mut value_start = separator + 1;
1981 while value_start < bytes.len() && bytes[value_start].is_ascii_whitespace() {
1982 value_start += 1;
1983 }
1984 let value_end = sensitive_value_end(bytes, value_start, false);
1985 mark(mask, key_start, value_end.max(value_start));
1986 }
1987
1988 let mut cursor = 0;
1989 while let Some(relative) = find_bytes(&bytes[cursor..], b"--") {
1990 let flag_start = cursor + relative;
1991 let name_start = flag_start + 2;
1992 let mut name_end = name_start;
1993 while name_end < bytes.len()
1994 && (bytes[name_end].is_ascii_alphanumeric()
1995 || matches!(bytes[name_end], b'_' | b'-' | b'.'))
1996 {
1997 name_end += 1;
1998 }
1999 if name_end == name_start || !sensitive_assignment_name(&lower[name_start..name_end]) {
2000 cursor = name_end.max(name_start + 1);
2001 continue;
2002 }
2003 let mut value_start = name_end;
2004 if bytes.get(value_start) == Some(&b'=') {
2005 value_start += 1;
2006 } else {
2007 while value_start < bytes.len() && bytes[value_start].is_ascii_whitespace() {
2008 value_start += 1;
2009 }
2010 }
2011 let value_end = sensitive_value_end(bytes, value_start, false);
2012 mark(mask, flag_start, value_end.max(value_start));
2013 cursor = value_end.max(name_end);
2014 }
2015}
2016
2017fn sensitive_assignment_name(name: &str) -> bool {
2018 let components = name
2019 .split(|character: char| !character.is_ascii_alphanumeric())
2020 .filter(|component| !component.is_empty())
2021 .collect::<Vec<_>>();
2022 components.iter().any(|component| {
2023 matches!(
2024 *component,
2025 "token"
2026 | "secret"
2027 | "password"
2028 | "passphrase"
2029 | "authorization"
2030 | "credential"
2031 | "credentials"
2032 )
2033 }) || (components.contains(&"private") && components.contains(&"key"))
2034 || (components.contains(&"api") && components.contains(&"key"))
2035 || (components.contains(&"access")
2036 && (components.contains(&"key") || components.contains(&"token")))
2037 || [
2038 "token",
2039 "secret",
2040 "password",
2041 "passphrase",
2042 "credential",
2043 "authorization",
2044 "privatekey",
2045 "apikey",
2046 "accesskey",
2047 ]
2048 .iter()
2049 .any(|marker| name.contains(marker))
2050}
2051
2052fn redact_sensitive_assignments(bytes: &[u8], lower: &str, mask: &mut [bool]) {
2053 for key in [
2054 "proxy-authorization",
2055 "authorization",
2056 "aws_secret_access_key",
2057 "secret_access_key",
2058 "signing_password",
2059 "p12_password",
2060 "github_token",
2061 "gh_token",
2062 "refresh_token",
2063 "refresh-token",
2064 "refresh token",
2065 "access_token",
2066 "access-token",
2067 "access token",
2068 "client_secret",
2069 "client-secret",
2070 "client secret",
2071 "private_key",
2072 "private-key",
2073 "private key",
2074 "api_key",
2075 "api-key",
2076 "api key",
2077 "password",
2078 "passphrase",
2079 "token",
2080 "secret",
2081 ] {
2082 let mut cursor = 0;
2083 while let Some(relative) = lower[cursor..].find(key) {
2084 let start = cursor + relative;
2085 let key_end = start + key.len();
2086 if !sensitive_key_boundary(bytes, start, key_end) {
2087 cursor = key_end;
2088 continue;
2089 }
2090 let mut separator = key_end;
2091 while separator < bytes.len() && bytes[separator].is_ascii_whitespace() {
2092 separator += 1;
2093 }
2094 let has_marker = separator < bytes.len() && matches!(bytes[separator], b'=' | b':');
2095 let command_flag =
2096 start >= 2 && &bytes[start - 2..start] == b"--" && separator > key_end;
2097 if !has_marker && !command_flag {
2098 cursor = key_end;
2099 continue;
2100 }
2101 if has_marker {
2102 separator += 1;
2103 while separator < bytes.len() && bytes[separator].is_ascii_whitespace() {
2104 separator += 1;
2105 }
2106 }
2107 let end = sensitive_value_end(bytes, separator, key.contains("authorization"));
2108 let marker_start = if start >= 2 && &bytes[start - 2..start] == b"--" {
2109 start - 2
2110 } else {
2111 start
2112 };
2113 mark(mask, marker_start, end.max(separator));
2114 cursor = end.max(key_end);
2115 }
2116 }
2117}
2118
2119fn redact_authorization_schemes(bytes: &[u8], lower: &str, mask: &mut [bool]) {
2120 for scheme in ["bearer", "basic"] {
2121 let mut cursor = 0;
2122 while let Some(relative) = lower[cursor..].find(scheme) {
2123 let start = cursor + relative;
2124 let end = start + scheme.len();
2125 if is_word_boundary(bytes, start, end)
2126 && end < bytes.len()
2127 && bytes[end].is_ascii_whitespace()
2128 {
2129 let mut value = end;
2130 while value < bytes.len() && bytes[value].is_ascii_whitespace() {
2131 value += 1;
2132 }
2133 let value_end = sensitive_value_end(bytes, value, false);
2134 mark(mask, start, value_end.max(value));
2135 cursor = value_end.max(end);
2136 } else {
2137 cursor = end;
2138 }
2139 }
2140 }
2141}
2142
2143fn sensitive_key_boundary(bytes: &[u8], start: usize, end: usize) -> bool {
2144 let before = start.checked_sub(1).and_then(|index| bytes.get(index));
2145 let after = bytes.get(end);
2146 before.is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_')
2147 && after.is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_')
2148}
2149
2150fn is_word_boundary(bytes: &[u8], start: usize, end: usize) -> bool {
2151 bytes
2152 .get(start.wrapping_sub(1))
2153 .is_none_or(|byte| !byte.is_ascii_alphanumeric())
2154 && bytes
2155 .get(end)
2156 .is_none_or(|byte| !byte.is_ascii_alphanumeric())
2157}
2158
2159fn is_token_boundary(bytes: &[u8], start: usize) -> bool {
2160 start == 0
2161 || bytes
2162 .get(start - 1)
2163 .is_none_or(|byte| !byte.is_ascii_alphanumeric() && !matches!(byte, b'_' | b'-'))
2164}
2165
2166fn sensitive_value_end(bytes: &[u8], start: usize, rest_of_line: bool) -> usize {
2167 if rest_of_line {
2168 return bytes.len();
2169 }
2170 if start >= bytes.len() {
2171 return start;
2172 }
2173 if matches!(bytes[start], b'\'' | b'"') {
2174 let quote = bytes[start];
2175 let mut escaped = false;
2176 for (offset, byte) in bytes[start + 1..].iter().copied().enumerate() {
2177 if escaped {
2178 escaped = false;
2179 } else if byte == b'\\' {
2180 escaped = true;
2181 } else if byte == quote {
2182 return start + offset + 2;
2183 }
2184 }
2185 return bytes.len();
2186 }
2187 bytes[start..]
2188 .iter()
2189 .position(|byte| byte.is_ascii_whitespace() || matches!(byte, b'&' | b',' | b';'))
2190 .map_or(bytes.len(), |offset| start + offset)
2191}
2192
2193fn mark(mask: &mut [bool], start: usize, end: usize) {
2194 let bounded_end = end.min(mask.len());
2195 for selected in mask.iter_mut().take(bounded_end).skip(start) {
2196 *selected = true;
2197 }
2198}
2199
2200fn render_masked(text: &str, mask: &[bool]) -> String {
2201 let mut output = String::with_capacity(text.len());
2202 let mut redacting = false;
2203 for (index, character) in text.char_indices() {
2204 let end = index + character.len_utf8();
2205 let selected = mask[index..end].iter().any(|selected| *selected);
2206 if selected {
2207 if !redacting {
2208 output.push_str(REDACTION_MARKER);
2209 redacting = true;
2210 }
2211 } else {
2212 output.push(character);
2213 redacting = false;
2214 }
2215 }
2216 output
2217}
2218
2219fn output_still_unsafe(value: &str) -> bool {
2220 let lower = value.to_ascii_lowercase();
2221 let unsafe_marker = [
2222 "authorization:",
2223 "authorization =",
2224 "bearer ",
2225 "basic ",
2226 "github_pat_",
2227 "ghp_",
2228 "gho_",
2229 "ghu_",
2230 "ghs_",
2231 "ghr_",
2232 "password=",
2233 "password:",
2234 "client_secret",
2235 "access_token",
2236 "refresh_token",
2237 "private_key",
2238 " token ",
2239 " secret ",
2240 " password ",
2241 "-----begin ",
2242 ]
2243 .iter()
2244 .any(|marker| lower.contains(marker));
2245 let unsafe_prefix = ["token ", "secret ", "password "]
2246 .iter()
2247 .any(|marker| lower.starts_with(marker));
2248 unsafe_marker || unsafe_prefix || value.contains('?') || credential_url(value)
2249}
2250
2251fn credential_url(value: &str) -> bool {
2252 value.split_whitespace().any(|word| {
2253 word.split_once("://").is_some_and(|(_, remainder)| {
2254 remainder
2255 .split(['/', '?', '#'])
2256 .next()
2257 .is_some_and(|authority| authority.contains('@'))
2258 })
2259 })
2260}
2261
2262fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
2263 haystack
2264 .windows(needle.len())
2265 .position(|candidate| candidate == needle)
2266}
2267
2268fn project_event(
2269 identity: &GithubRunAttemptLogIdentity,
2270 job: BoundJob,
2271 sorted_job_index: u16,
2272 line_ordinal: u32,
2273 line: SanitizedLine,
2274) -> Result<GithubWorkerLogEvent, GithubJobLogError> {
2275 let source_sequence = encode_source_sequence(identity.attempt, sorted_job_index, line_ordinal)?;
2276 let digest = canonical_event_sha256(
2277 identity,
2278 job.id,
2279 sorted_job_index,
2280 line_ordinal,
2281 job.completed_at_ms,
2282 source_sequence,
2283 &line.message,
2284 line.unsafe_line,
2285 );
2286 let projected_store_bytes = projected_store_record_bytes(&line.message);
2287 Ok(GithubWorkerLogEvent {
2288 identity: identity.clone(),
2289 job_id: job.id,
2290 sorted_job_index,
2291 line_ordinal,
2292 occurred_at_ms: job.completed_at_ms,
2293 source_sequence,
2294 source_event_sha256: digest,
2295 message: line.message,
2296 unsafe_line: line.unsafe_line,
2297 projected_store_bytes,
2298 })
2299}
2300
2301struct CompletionMarkerMessage<'a> {
2302 run_id: u64,
2303 attempt: u64,
2304 head_sha: &'a str,
2305 workflow_id: u64,
2306 job_count: u32,
2307 job_set_sha256: &'a str,
2308 event_count: u64,
2309 event_set_sha256: &'a str,
2310}
2311
2312fn parse_completion_marker_message(message: &str) -> Option<CompletionMarkerMessage<'_>> {
2313 if message.is_empty()
2314 || message.len() > MAX_JOB_LOG_LINE_BYTES
2315 || message.chars().any(char::is_control)
2316 {
2317 return None;
2318 }
2319 let mut fields = message.split(' ');
2320 let run_id = parse_canonical_u64(fields.next()?.strip_prefix("run_id=")?)?;
2321 let attempt = parse_canonical_u64(fields.next()?.strip_prefix("run_attempt=")?)?;
2322 let head_sha = fields.next()?.strip_prefix("head_sha=")?;
2323 let workflow_id = parse_canonical_u64(fields.next()?.strip_prefix("workflow_id=")?)?;
2324 let job_count = parse_canonical_u64(fields.next()?.strip_prefix("job_count=")?)?;
2325 let job_set_sha256 = fields.next()?.strip_prefix("job_set_sha256=")?;
2326 let event_count = parse_canonical_u64(fields.next()?.strip_prefix("event_count=")?)?;
2327 let event_set_sha256 = fields.next()?.strip_prefix("event_set_sha256=")?;
2328 if fields.next().is_some()
2329 || !(1..=MAX_JOBS as u64).contains(&job_count)
2330 || event_count > MAX_WORKER_LOG_EVENTS as u64
2331 || !is_lowercase_sha256(job_set_sha256)
2332 || !is_lowercase_sha256(event_set_sha256)
2333 {
2334 return None;
2335 }
2336 Some(CompletionMarkerMessage {
2337 run_id,
2338 attempt,
2339 head_sha,
2340 workflow_id,
2341 job_count: u32::try_from(job_count).ok()?,
2342 job_set_sha256,
2343 event_count,
2344 event_set_sha256,
2345 })
2346}
2347
2348fn parse_canonical_u64(value: &str) -> Option<u64> {
2349 if value.is_empty()
2350 || (value.len() > 1 && value.starts_with('0'))
2351 || !value.bytes().all(|byte| byte.is_ascii_digit())
2352 {
2353 return None;
2354 }
2355 value.parse().ok()
2356}
2357
2358fn is_lowercase_sha256(value: &str) -> bool {
2359 value.len() == 64
2360 && value
2361 .bytes()
2362 .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f'))
2363}
2364
2365fn completion_message(
2366 identity: &GithubRunAttemptLogIdentity,
2367 job_count: u32,
2368 job_set_sha256: &str,
2369 event_count: u64,
2370 event_set_sha256: &str,
2371) -> String {
2372 format!(
2373 "run_id={} run_attempt={} head_sha={} workflow_id={} job_count={job_count} job_set_sha256={job_set_sha256} event_count={event_count} event_set_sha256={event_set_sha256}",
2374 identity.run_id.get(),
2375 identity.attempt.get(),
2376 identity.head_sha.as_str(),
2377 identity.workflow_id.get(),
2378 )
2379}
2380
2381fn project_completion(
2382 identity: &GithubRunAttemptLogIdentity,
2383 jobs: &[BoundJob],
2384 events: &[GithubWorkerLogEvent],
2385) -> Result<GithubWorkerLogsComplete, GithubJobLogError> {
2386 let occurred_at_ms = jobs
2387 .iter()
2388 .map(|job| job.completed_at_ms)
2389 .max()
2390 .ok_or(GithubJobLogError::IncompleteJobPagination)?;
2391 let job_count =
2392 u32::try_from(jobs.len()).map_err(|_| GithubJobLogError::SourceSequenceOverflow)?;
2393 let event_count = events.len() as u64;
2394 let job_set_sha256 = canonical_job_set_sha256(jobs);
2395 let event_set_sha256 = canonical_event_set_sha256(events);
2396 let source_sequence = completion_source_sequence(identity.attempt);
2397 let message = completion_message(
2398 identity,
2399 job_count,
2400 &job_set_sha256,
2401 event_count,
2402 &event_set_sha256,
2403 );
2404 let source_event_sha256 = canonical_completion_sha256(
2405 identity,
2406 occurred_at_ms,
2407 job_count,
2408 &job_set_sha256,
2409 event_count,
2410 &event_set_sha256,
2411 source_sequence,
2412 &message,
2413 );
2414 let projected_store_bytes = projected_store_record_bytes(&message);
2415 if projected_store_bytes > MAX_PROJECTED_STORE_EVENT_BYTES {
2416 return Err(GithubJobLogError::ProjectedByteLimitReached);
2417 }
2418 Ok(GithubWorkerLogsComplete {
2419 identity: identity.clone(),
2420 occurred_at_ms,
2421 job_count,
2422 job_set_sha256,
2423 event_count,
2424 event_set_sha256,
2425 source_sequence,
2426 source_event_sha256,
2427 message,
2428 projected_store_bytes,
2429 })
2430}
2431
2432fn canonical_job_set_sha256(jobs: &[BoundJob]) -> String {
2433 let mut digest = Sha256::new();
2434 digest.update(CANONICAL_JOB_SET_DOMAIN);
2435 digest.update((jobs.len() as u64).to_be_bytes());
2436 for job in jobs {
2437 digest.update(job.id.get().to_be_bytes());
2438 digest.update(job.completed_at_ms.to_be_bytes());
2439 }
2440 hex::encode(digest.finalize())
2441}
2442
2443fn canonical_event_set_sha256(events: &[GithubWorkerLogEvent]) -> String {
2444 let mut digest = Sha256::new();
2445 digest.update(CANONICAL_EVENT_SET_DOMAIN);
2446 digest.update((events.len() as u64).to_be_bytes());
2447 for event in events {
2448 digest.update(event.source_sequence.to_be_bytes());
2449 digest_field(&mut digest, event.source_event_sha256.as_bytes());
2450 }
2451 hex::encode(digest.finalize())
2452}
2453
2454fn validate_durable_event_set<'a>(
2455 attempt: GithubRunAttempt,
2456 expected_count: u64,
2457 expected_sha256: &str,
2458 worker_events: impl IntoIterator<Item = (u64, &'a str)>,
2459) -> Result<(), GithubJobLogError> {
2460 let mut digest = Sha256::new();
2461 digest.update(CANONICAL_EVENT_SET_DOMAIN);
2462 digest.update(expected_count.to_be_bytes());
2463 let mut count = 0_u64;
2464 let mut previous_sequence = None;
2465 for (source_sequence, source_event_sha256) in worker_events {
2466 count = count
2467 .checked_add(1)
2468 .ok_or(GithubJobLogError::InvalidCompletionMarker)?;
2469 if count > expected_count
2470 || count > MAX_WORKER_LOG_EVENTS as u64
2471 || previous_sequence.is_some_and(|previous| source_sequence <= previous)
2472 || !valid_worker_event_sequence(attempt, source_sequence)
2473 || !is_lowercase_sha256(source_event_sha256)
2474 {
2475 return Err(GithubJobLogError::InvalidCompletionMarker);
2476 }
2477 digest.update(source_sequence.to_be_bytes());
2478 digest_field(&mut digest, source_event_sha256.as_bytes());
2479 previous_sequence = Some(source_sequence);
2480 }
2481 if count != expected_count || hex::encode(digest.finalize()) != expected_sha256 {
2482 return Err(GithubJobLogError::InvalidCompletionMarker);
2483 }
2484 Ok(())
2485}
2486
2487fn valid_worker_event_sequence(attempt: GithubRunAttempt, source_sequence: u64) -> bool {
2488 let job_index = (source_sequence >> 16) & u64::from(u16::MAX);
2489 let line_ordinal = source_sequence & u64::from(u16::MAX);
2490 source_sequence >> 32 == u64::from(attempt.get())
2491 && job_index < u64::from(u16::MAX)
2492 && (1..=MAX_WORKER_LOG_EVENTS as u64).contains(&line_ordinal)
2493}
2494
2495#[allow(clippy::too_many_arguments)]
2496fn canonical_completion_sha256(
2497 identity: &GithubRunAttemptLogIdentity,
2498 occurred_at_ms: u64,
2499 job_count: u32,
2500 job_set_sha256: &str,
2501 event_count: u64,
2502 event_set_sha256: &str,
2503 source_sequence: u64,
2504 message: &str,
2505) -> String {
2506 let mut digest = Sha256::new();
2507 digest.update(CANONICAL_COMPLETION_DOMAIN);
2508 digest_identity(&mut digest, identity);
2509 digest.update(occurred_at_ms.to_be_bytes());
2510 digest.update(job_count.to_be_bytes());
2511 digest_field(&mut digest, job_set_sha256.as_bytes());
2512 digest.update(event_count.to_be_bytes());
2513 digest_field(&mut digest, event_set_sha256.as_bytes());
2514 digest.update(source_sequence.to_be_bytes());
2515 digest_field(&mut digest, WORKER_LOGS_COMPLETE_CODE.as_bytes());
2516 digest_field(&mut digest, message.as_bytes());
2517 hex::encode(digest.finalize())
2518}
2519
2520fn reserve_projected_bytes(
2521 total: &mut u64,
2522 event_bytes: u64,
2523 maximum: u64,
2524) -> Result<(), GithubJobLogError> {
2525 let updated = total
2526 .checked_add(event_bytes)
2527 .ok_or(GithubJobLogError::ProjectedByteLimitReached)?;
2528 if updated > maximum {
2529 return Err(GithubJobLogError::ProjectedByteLimitReached);
2530 }
2531 *total = updated;
2532 Ok(())
2533}
2534
2535fn projected_store_record_bytes(message: &str) -> u64 {
2536 let encoded_message_bytes = serde_json::to_vec(message)
2537 .map_or(MAX_PROJECTED_STORE_EVENT_BYTES, |bytes| bytes.len() as u64);
2538 PROJECTED_STORE_EVENT_ENVELOPE_BYTES.saturating_add(encoded_message_bytes)
2539}
2540
2541fn encode_source_sequence(
2542 attempt: GithubRunAttempt,
2543 sorted_job_index: u16,
2544 line_ordinal: u32,
2545) -> Result<u64, GithubJobLogError> {
2546 if sorted_job_index == u16::MAX || line_ordinal as usize > MAX_WORKER_LOG_EVENTS {
2547 return Err(GithubJobLogError::SourceSequenceOverflow);
2548 }
2549 let line_ordinal = u16::try_from(line_ordinal)
2550 .ok()
2551 .filter(|ordinal| *ordinal != 0 && *ordinal != u16::MAX)
2552 .ok_or(GithubJobLogError::SourceSequenceOverflow)?;
2553 Ok((u64::from(attempt.get()) << 32)
2554 | (u64::from(sorted_job_index) << 16)
2555 | u64::from(line_ordinal))
2556}
2557
2558fn completion_source_sequence(attempt: GithubRunAttempt) -> u64 {
2559 (u64::from(attempt.get()) << 32) | (u64::from(u16::MAX) << 16) | u64::from(u16::MAX)
2560}
2561
2562#[allow(clippy::too_many_arguments)]
2563fn canonical_event_sha256(
2564 identity: &GithubRunAttemptLogIdentity,
2565 job_id: GithubActionsJobId,
2566 sorted_job_index: u16,
2567 line_ordinal: u32,
2568 occurred_at_ms: u64,
2569 source_sequence: u64,
2570 message: &str,
2571 unsafe_line: bool,
2572) -> String {
2573 let mut digest = Sha256::new();
2574 digest.update(CANONICAL_EVENT_DOMAIN);
2575 digest_identity(&mut digest, identity);
2576 digest.update(job_id.get().to_be_bytes());
2577 digest.update(sorted_job_index.to_be_bytes());
2578 digest.update(line_ordinal.to_be_bytes());
2579 digest.update(occurred_at_ms.to_be_bytes());
2580 digest.update(source_sequence.to_be_bytes());
2581 digest.update([u8::from(unsafe_line)]);
2582 digest_field(&mut digest, message.as_bytes());
2583 hex::encode(digest.finalize())
2584}
2585
2586fn digest_identity(digest: &mut Sha256, identity: &GithubRunAttemptLogIdentity) {
2587 digest_field(digest, identity.repository.owner().as_bytes());
2588 digest_field(digest, identity.repository.name().as_bytes());
2589 digest.update(identity.run_id.get().to_be_bytes());
2590 digest.update(identity.attempt.get().to_be_bytes());
2591 digest_field(digest, identity.head_sha.as_str().as_bytes());
2592 digest.update(identity.workflow_id.get().to_be_bytes());
2593}
2594
2595fn digest_field(digest: &mut Sha256, value: &[u8]) {
2596 digest.update((value.len() as u64).to_be_bytes());
2597 digest.update(value);
2598}
2599
2600#[cfg(test)]
2601mod tests {
2602 use std::collections::VecDeque;
2603
2604 use super::*;
2605 use crate::transport::{RunConclusion, RunEvent, RunHandle, RunStatus};
2606
2607 #[derive(Default)]
2608 struct TestClient {
2609 responses: VecDeque<GithubJobLogHttpResponse<TestBody>>,
2610 requests: Vec<CapturedRequest>,
2611 cancel_during_get: bool,
2612 fail_during_get: bool,
2613 }
2614
2615 #[derive(Clone, Debug, Eq, PartialEq)]
2616 struct CapturedRequest {
2617 target: String,
2618 api_path: bool,
2619 authorization: GithubJobLogAuthorization,
2620 accept: GithubJobLogAccept,
2621 }
2622
2623 #[derive(Default)]
2624 struct TestBody {
2625 chunks: VecDeque<Vec<u8>>,
2626 cancel_during_read: bool,
2627 fail_during_read: bool,
2628 }
2629
2630 impl TestBody {
2631 fn chunks(chunks: impl IntoIterator<Item = impl Into<Vec<u8>>>) -> Self {
2632 Self {
2633 chunks: chunks.into_iter().map(Into::into).collect(),
2634 cancel_during_read: false,
2635 fail_during_read: false,
2636 }
2637 }
2638
2639 fn cancelling(chunks: impl IntoIterator<Item = impl Into<Vec<u8>>>) -> Self {
2640 Self {
2641 chunks: chunks.into_iter().map(Into::into).collect(),
2642 cancel_during_read: true,
2643 fail_during_read: false,
2644 }
2645 }
2646
2647 fn cancelling_with_error() -> Self {
2648 Self {
2649 chunks: VecDeque::new(),
2650 cancel_during_read: true,
2651 fail_during_read: true,
2652 }
2653 }
2654 }
2655
2656 impl GithubJobLogHttpBody for TestBody {
2657 fn next_chunk(
2658 &mut self,
2659 maximum_bytes: usize,
2660 cancellation: &CancellationToken,
2661 ) -> Result<Option<Vec<u8>>, GithubJobLogHttpError> {
2662 if cancellation.is_cancelled() {
2663 return Err(GithubJobLogHttpError::Cancelled);
2664 }
2665 if self.cancel_during_read {
2666 cancellation.cancel();
2667 }
2668 if self.fail_during_read {
2669 return Err(GithubJobLogHttpError::BodyRead);
2670 }
2671 let Some(mut chunk) = self.chunks.pop_front() else {
2672 return Ok(None);
2673 };
2674 if chunk.len() > maximum_bytes {
2675 let remainder = chunk.split_off(maximum_bytes);
2676 self.chunks.push_front(remainder);
2677 }
2678 Ok(Some(chunk))
2679 }
2680 }
2681
2682 impl GithubJobLogHttpClient for TestClient {
2683 type Body = TestBody;
2684
2685 fn get(
2686 &mut self,
2687 request: &GithubJobLogHttpRequest,
2688 cancellation: &CancellationToken,
2689 ) -> Result<GithubJobLogHttpResponse<Self::Body>, GithubJobLogHttpError> {
2690 if cancellation.is_cancelled() {
2691 return Err(GithubJobLogHttpError::Cancelled);
2692 }
2693 self.requests.push(CapturedRequest {
2694 target: request.target().to_owned(),
2695 api_path: request.is_api_path(),
2696 authorization: request.authorization(),
2697 accept: request.accept(),
2698 });
2699 if self.cancel_during_get {
2700 cancellation.cancel();
2701 }
2702 if self.fail_during_get {
2703 return Err(GithubJobLogHttpError::Protocol);
2704 }
2705 Ok(self
2706 .responses
2707 .pop_front()
2708 .expect("test queued one response per request"))
2709 }
2710 }
2711
2712 fn response(
2713 status: u16,
2714 content_type: Option<&str>,
2715 location: Option<&str>,
2716 chunks: impl IntoIterator<Item = impl Into<Vec<u8>>>,
2717 ) -> GithubJobLogHttpResponse<TestBody> {
2718 GithubJobLogHttpResponse::new(
2719 status,
2720 content_type.map(str::to_owned),
2721 location.map(str::to_owned),
2722 TestBody::chunks(chunks),
2723 )
2724 }
2725
2726 fn identity() -> GithubRunAttemptLogIdentity {
2727 let handle = RunHandle::restore(
2728 42,
2729 7,
2730 ".github/workflows/ferry.yml".to_owned(),
2731 "0123456789abcdef0123456789abcdef01234567".to_owned(),
2732 "ferry/jobs/test".to_owned(),
2733 RunEvent::Push,
2734 )
2735 .expect("run handle");
2736 let run = RunSnapshot::restore(
2737 handle,
2738 1,
2739 2,
2740 RunStatus::Completed,
2741 Some(RunConclusion::Success),
2742 )
2743 .expect("terminal run");
2744 GithubRunAttemptLogIdentity::from_terminal_run(
2745 Repository::new("owner", "repo").expect("repository"),
2746 &run,
2747 )
2748 .expect("log identity")
2749 }
2750
2751 fn limits(job_bytes: u64, poll_bytes: u64) -> GithubJobLogLimits {
2752 GithubJobLogLimits::new(
2753 3,
2754 100,
2755 100,
2756 10_000,
2757 MAX_WORKER_LOG_PROJECTED_BYTES,
2758 64 * 1024,
2759 job_bytes,
2760 poll_bytes,
2761 Duration::from_secs(1),
2762 Duration::from_secs(10),
2763 3,
2764 )
2765 .expect("limits")
2766 }
2767
2768 fn private_key_boundary(kind: &str) -> String {
2769 format!("-----{kind} PRIVATE KEY-----")
2770 }
2771
2772 fn metadata(jobs: &str, total: usize) -> Vec<u8> {
2773 format!(r#"{{"total_count":{total},"jobs":[{jobs}]}}"#).into_bytes()
2774 }
2775
2776 fn job(id: u64) -> String {
2777 format!(
2778 r#"{{"id":{id},"run_id":42,"head_sha":"0123456789abcdef0123456789abcdef01234567","status":"completed","completed_at":"2020-01-20T17:44:39.123Z"}}"#
2779 )
2780 }
2781
2782 fn queue_job_log(client: &mut TestClient, job_id: u64, chunks: Vec<Vec<u8>>) {
2783 client.responses.push_back(response(
2784 302,
2785 None,
2786 Some(&format!(
2787 "https://pipelines.actions.githubusercontent.com/{job_id}?sig=secret"
2788 )),
2789 Vec::<Vec<u8>>::new(),
2790 ));
2791 client.responses.push_back(response(
2792 200,
2793 Some("text/plain; charset=utf-8"),
2794 None,
2795 chunks,
2796 ));
2797 }
2798
2799 fn log_poll(log: Vec<u8>) -> GithubJobLogPoll {
2800 let mut client = TestClient::default();
2801 client.responses.push_back(response(
2802 200,
2803 Some("application/json"),
2804 None,
2805 [metadata(&job(3), 1)],
2806 ));
2807 let chunks = if log.is_empty() {
2808 Vec::new()
2809 } else {
2810 vec![log]
2811 };
2812 queue_job_log(&mut client, 3, chunks);
2813 GithubJobLogFetcher::new(client, limits(1024, 1024))
2814 .fetch_attempt(&identity(), &CancellationToken::new())
2815 .expect("bounded poll")
2816 }
2817
2818 fn one_line_poll() -> GithubJobLogPoll {
2819 log_poll(b"safe\n".to_vec())
2820 }
2821
2822 fn event_pairs(poll: &GithubJobLogPoll) -> impl Iterator<Item = (u64, &str)> {
2823 poll.events()
2824 .iter()
2825 .map(|event| (event.source_sequence(), event.source_event_sha256()))
2826 }
2827
2828 fn durable_identity(
2829 identity: &GithubRunAttemptLogIdentity,
2830 ) -> GithubDurableRunAttemptLogIdentity {
2831 GithubDurableRunAttemptLogIdentity::new(
2832 identity.repository().clone(),
2833 identity.run_id(),
2834 u64::from(identity.attempt().get()),
2835 identity.head_sha().clone(),
2836 identity.workflow_id(),
2837 )
2838 .expect("durable identity")
2839 }
2840
2841 #[test]
2842 fn fetches_exact_attempt_sorts_numeric_jobs_and_deduplicates_rows() {
2843 let mut client = TestClient::default();
2844 let jobs = format!("{},{},{}", job(20), job(3), job(3));
2845 client.responses.push_back(response(
2846 200,
2847 Some("application/json"),
2848 None,
2849 [metadata(&jobs, 2)],
2850 ));
2851 queue_job_log(&mut client, 3, vec![b"three\n".to_vec()]);
2852 queue_job_log(&mut client, 20, vec![b"twenty\n".to_vec()]);
2853
2854 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 2048));
2855 let poll = fetcher
2856 .fetch_attempt(&identity(), &CancellationToken::new())
2857 .expect("fetch");
2858 assert_eq!(
2859 poll.job_ids().iter().map(|id| id.get()).collect::<Vec<_>>(),
2860 [3, 20]
2861 );
2862 assert_eq!(
2863 poll.events()
2864 .iter()
2865 .map(GithubWorkerLogEvent::message)
2866 .collect::<Vec<_>>(),
2867 ["three", "twenty"]
2868 );
2869 assert_eq!(poll.events()[0].source_sequence(), (2_u64 << 32) | 1);
2870 assert_eq!(
2871 poll.events()[1].source_sequence(),
2872 (2_u64 << 32) | (1_u64 << 16) | 1
2873 );
2874 assert_ne!(
2875 poll.events()[0].source_event_sha256(),
2876 poll.events()[1].source_event_sha256()
2877 );
2878 assert!(poll.events().iter().all(|event| {
2879 event.source_event_sha256().len() == 64
2880 && event
2881 .source_event_sha256()
2882 .bytes()
2883 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
2884 }));
2885
2886 let client = fetcher.into_client();
2887 assert_eq!(
2888 client.requests[0].target,
2889 "/repos/owner/repo/actions/runs/42/attempts/2/jobs?per_page=100&page=1"
2890 );
2891 assert_eq!(
2892 client.requests[1].target,
2893 "/repos/owner/repo/actions/jobs/3/logs"
2894 );
2895 assert_eq!(client.requests[1].accept, GithubJobLogAccept::Json);
2896 assert_eq!(
2897 client.requests[2].authorization,
2898 GithubJobLogAuthorization::Omit
2899 );
2900 assert_eq!(client.requests[2].accept, GithubJobLogAccept::PlainText);
2901 assert!(!client.requests[2].api_path);
2902 }
2903
2904 #[test]
2905 fn redirect_allowlist_rejects_userinfo_and_foreign_hosts_without_leaking_url() {
2906 for url in [
2907 "https://evil.example/log?token=super-secret",
2908 "https://alice:password@pipelines.actions.githubusercontent.com/log",
2909 "http://pipelines.actions.githubusercontent.com/log",
2910 ] {
2911 let mut client = TestClient::default();
2912 client.responses.push_back(response(
2913 200,
2914 Some("application/json"),
2915 None,
2916 [metadata(&job(3), 1)],
2917 ));
2918 client
2919 .responses
2920 .push_back(response(302, None, Some(url), Vec::<Vec<u8>>::new()));
2921 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
2922 let error = fetcher
2923 .fetch_attempt(&identity(), &CancellationToken::new())
2924 .expect_err("redirect rejected");
2925 assert!(matches!(
2926 error,
2927 GithubJobLogError::RedirectHostNotAllowed | GithubJobLogError::InvalidRedirect
2928 ));
2929 let rendered = format!("{error:?} {error}");
2930 assert!(!rendered.contains("super-secret"));
2931 assert!(!rendered.contains("password"));
2932 assert_eq!(fetcher.client().requests.len(), 2);
2933 }
2934 }
2935
2936 #[test]
2937 fn redirect_request_debug_redacts_signed_target_and_omits_authorization() {
2938 let request = GithubJobLogHttpRequest::redirect(
2939 "https://pipelines.actions.githubusercontent.com/log?sig=top-secret".to_owned(),
2940 Duration::from_secs(1),
2941 );
2942 let rendered = format!("{request:?}");
2943 assert!(!rendered.contains("top-secret"));
2944 assert_eq!(request.authorization(), GithubJobLogAuthorization::Omit);
2945 }
2946
2947 #[test]
2948 fn enforces_per_job_and_aggregate_poll_byte_bounds() {
2949 let mut client = TestClient::default();
2950 client.responses.push_back(response(
2951 200,
2952 Some("application/json"),
2953 None,
2954 [metadata(&job(3), 1)],
2955 ));
2956 queue_job_log(&mut client, 3, vec![b"12345".to_vec(), b"6".to_vec()]);
2957 let mut fetcher = GithubJobLogFetcher::new(client, limits(5, 5));
2958 assert_eq!(
2959 fetcher.fetch_attempt(&identity(), &CancellationToken::new()),
2960 Err(GithubJobLogError::JobLogTooLarge)
2961 );
2962
2963 let mut client = TestClient::default();
2964 client.responses.push_back(response(
2965 200,
2966 Some("application/json"),
2967 None,
2968 [metadata(&format!("{},{}", job(3), job(4)), 2)],
2969 ));
2970 queue_job_log(&mut client, 3, vec![b"1234".to_vec()]);
2971 queue_job_log(&mut client, 4, vec![b"5678".to_vec()]);
2972 let mut fetcher = GithubJobLogFetcher::new(client, limits(8, 7));
2973 assert_eq!(
2974 fetcher.fetch_attempt(&identity(), &CancellationToken::new()),
2975 Err(GithubJobLogError::PollTooLarge)
2976 );
2977 }
2978
2979 #[test]
2980 fn streaming_parser_handles_utf8_crlf_and_redaction_across_chunks() {
2981 let mut client = TestClient::default();
2982 client.responses.push_back(response(
2983 200,
2984 Some("application/json"),
2985 None,
2986 [metadata(&job(3), 1)],
2987 ));
2988 queue_job_log(
2989 &mut client,
2990 3,
2991 vec![
2992 b"snowman \xe2".to_vec(),
2993 b"\x98\x83\r".to_vec(),
2994 b"\nAuthorization: Bea".to_vec(),
2995 b"rer top-secret\r\nhttps://alice:pw@example.test/path?token=query-secret\n"
2996 .to_vec(),
2997 b"github_pat_abc123".to_vec(),
2998 ],
2999 );
3000 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3001 let poll = fetcher
3002 .fetch_attempt(&identity(), &CancellationToken::new())
3003 .expect("fetch");
3004 let messages = poll
3005 .events()
3006 .iter()
3007 .map(GithubWorkerLogEvent::message)
3008 .collect::<Vec<_>>();
3009 assert_eq!(messages[0], "snowman ☃");
3010 assert_eq!(messages[1], UNSAFE_JOB_LOG_LINE_MARKER);
3011 assert!(!messages[2].contains("alice"));
3012 assert!(!messages[2].contains("query-secret"));
3013 assert!(!messages[2].contains(['?', '@']));
3014 assert_eq!(messages[3], REDACTION_MARKER);
3015 assert!(messages.iter().all(|message| {
3016 !message.contains("top-secret")
3017 && !message.contains("abc123")
3018 && !output_still_unsafe(message)
3019 }));
3020 }
3021
3022 #[test]
3023 fn malformed_utf8_controls_and_long_lines_use_whole_line_marker() {
3024 let mut client = TestClient::default();
3025 client.responses.push_back(response(
3026 200,
3027 Some("application/json"),
3028 None,
3029 [metadata(&job(3), 1)],
3030 ));
3031 let mut chunks = vec![vec![0xff, b'\n'], b"ansi \x1b[31m\n".to_vec()];
3032 let mut long = vec![b'x'; MAX_JOB_LOG_LINE_BYTES + 10];
3033 long.push(b'\n');
3034 chunks.push(long);
3035 queue_job_log(&mut client, 3, chunks);
3036 let mut fetcher = GithubJobLogFetcher::new(
3037 client,
3038 limits((MAX_JOB_LOG_LINE_BYTES + 32) as u64, 8 * 1024),
3039 );
3040 let poll = fetcher
3041 .fetch_attempt(&identity(), &CancellationToken::new())
3042 .expect("unsafe lines are projected");
3043 assert_eq!(poll.events().len(), 3);
3044 assert!(
3045 poll.events().iter().all(|event| {
3046 event.unsafe_line() && event.message() == UNSAFE_JOB_LOG_LINE_MARKER
3047 })
3048 );
3049 }
3050
3051 #[test]
3052 fn pem_blocks_and_named_secret_environment_values_never_reach_projection() {
3053 let mut client = TestClient::default();
3054 client.responses.push_back(response(
3055 200,
3056 Some("application/json"),
3057 None,
3058 [metadata(&job(3), 1)],
3059 ));
3060 queue_job_log(
3061 &mut client,
3062 3,
3063 vec![
3064 format!("{}\n", private_key_boundary("BEGIN")).into_bytes(),
3065 b"c3VwZXItc2VjcmV0LWtleS1ieXRlcw==\n-----END PRIVATE KEY-----\n".to_vec(),
3066 b"AWS_SECRET_ACCESS_KEY=aws-secret-value\n".to_vec(),
3067 ],
3068 );
3069 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3070 let poll = fetcher
3071 .fetch_attempt(&identity(), &CancellationToken::new())
3072 .expect("fetch");
3073 assert_eq!(poll.events().len(), 4);
3074 assert!(
3075 poll.events()[..3].iter().all(|event| {
3076 event.unsafe_line() && event.message() == UNSAFE_JOB_LOG_LINE_MARKER
3077 })
3078 );
3079 assert!(poll.events()[3].unsafe_line());
3080 assert_eq!(poll.events()[3].message(), UNSAFE_JOB_LOG_LINE_MARKER);
3081 let rendered = format!("{poll:?}");
3082 assert!(!rendered.contains("c3VwZX"));
3083 assert!(!rendered.contains("aws-secret-value"));
3084 }
3085
3086 #[test]
3087 fn labeled_opaque_material_uses_whole_line_marker() {
3088 let encoded = format!(
3089 "2026-08-10T00:00:00Z private material: {}",
3090 "Ab0/".repeat(16)
3091 );
3092 let sanitized = sanitize_line(encoded.as_bytes());
3093 assert!(sanitized.unsafe_line);
3094 assert_eq!(sanitized.message, UNSAFE_JOB_LOG_LINE_MARKER);
3095 }
3096
3097 #[test]
3098 fn opaque_credentials_quoted_secrets_and_uri_variants_never_leak() {
3099 for value in [
3100 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
3101 concat!("AKIA", "1234567890ABCDEF"),
3102 "2026-08-10T00:00:00Z value=0123456789abcdef0123",
3103 r#"{"token":"short-secret"}"#,
3104 r#"TOKEN="abc\"def""#,
3105 "myToken=short-secret",
3106 r#"--token "x\"short-secret""#,
3107 r#"Bearer "x\"short-secret""#,
3108 "PASSWORD=,hunter2",
3109 "PASSWORD=;hunter2",
3110 "PASSWORD=&hunter2",
3111 ] {
3112 let sanitized = sanitize_line(value.as_bytes());
3113 assert!(sanitized.unsafe_line, "{value}");
3114 assert_eq!(sanitized.message, UNSAFE_JOB_LOG_LINE_MARKER);
3115 }
3116
3117 for (value, secret, expected) in [
3118 (
3119 "s3://bucket/private-path-secret",
3120 "private-path-secret",
3121 UNSAFE_JOB_LOG_LINE_MARKER,
3122 ),
3123 ("git+ssh://h/pw", "h/pw", REDACTION_MARKER),
3124 ("//alice:pw@host/path", "alice:pw", REDACTION_MARKER),
3125 ] {
3126 let sanitized = sanitize_line(value.as_bytes());
3127 assert!(!sanitized.message.contains(secret));
3128 assert_eq!(sanitized.message, expected);
3129 }
3130
3131 let concatenated_name = sanitize_line(b"myToken=x");
3132 assert!(concatenated_name.unsafe_line);
3133 assert_eq!(concatenated_name.message, UNSAFE_JOB_LOG_LINE_MARKER);
3134 }
3135
3136 #[test]
3137 fn prefixed_and_oversized_private_blocks_remain_armed_until_a_close_marker() {
3138 let mut client = TestClient::default();
3139 client.responses.push_back(response(
3140 200,
3141 Some("application/json"),
3142 None,
3143 [metadata(&job(3), 1)],
3144 ));
3145 let mut oversized = vec![b'x'; MAX_JOB_LOG_LINE_BYTES + 32];
3146 oversized.extend_from_slice(format!(" {}\n", private_key_boundary("BEGIN")).as_bytes());
3147 let mut log = format!(
3148 "2026-08-10T00:00:00Z {}\n2026-08-10T00:00:01Z c2hvcnQtc2VjcmV0\n2026-08-10T00:00:02Z {}\npem-safe\n2026-08-10T00:00:03Z PuTTY-User-Key-File-3: ssh-rsa\n2026-08-10T00:00:04Z Private-Lines: 1\n2026-08-10T00:00:05Z cHBrc2VjcmV0\n2026-08-10T00:00:06Z Private-MAC: deadbeef\nppk-safe\n",
3149 private_key_boundary("BEGIN"),
3150 private_key_boundary("END")
3151 )
3152 .into_bytes();
3153 log.extend_from_slice(&oversized);
3154 log.extend_from_slice(
3155 b"short-secret-after-oversize\n2026-08-10T00:00:07Z -----END PRIVATE KEY-----\nafter-oversize-safe\n",
3156 );
3157 queue_job_log(&mut client, 3, vec![log]);
3158 let mut fetcher = GithubJobLogFetcher::new(client, limits(16 * 1024, 16 * 1024));
3159 let poll = fetcher
3160 .fetch_attempt(&identity(), &CancellationToken::new())
3161 .expect("fetch");
3162 for index in [0, 1, 2, 4, 5, 6, 7, 9, 10, 11] {
3163 assert!(poll.events()[index].unsafe_line(), "event {index}");
3164 assert_eq!(poll.events()[index].message(), UNSAFE_JOB_LOG_LINE_MARKER);
3165 }
3166 assert_eq!(poll.events()[3].message(), "pem-safe");
3167 assert_eq!(poll.events()[8].message(), "ppk-safe");
3168 assert_eq!(poll.events()[12].message(), "after-oversize-safe");
3169 let rendered = format!("{poll:?}");
3170 assert!(!rendered.contains("c2hvcnQ"));
3171 assert!(!rendered.contains("cHBrc2"));
3172 assert!(!rendered.contains("short-secret-after-oversize"));
3173 }
3174
3175 #[test]
3176 fn malformed_utf8_private_block_opener_remains_armed_until_close() {
3177 let mut client = TestClient::default();
3178 client.responses.push_back(response(
3179 200,
3180 Some("application/json"),
3181 None,
3182 [metadata(&job(3), 1)],
3183 ));
3184 let mut opener = b"2026-08-10T00:00:00Z -----BE".to_vec();
3185 opener.push(0xff);
3186 opener.extend_from_slice(b"GIN PRIVATE KEY-----\n");
3187 queue_job_log(
3188 &mut client,
3189 3,
3190 vec![
3191 opener,
3192 b"short-fragment\n-----END PRIVATE KEY-----\nafter-close\n".to_vec(),
3193 ],
3194 );
3195 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3196 let poll = fetcher
3197 .fetch_attempt(&identity(), &CancellationToken::new())
3198 .expect("malformed private block is projected safely");
3199 assert!(
3200 poll.events()[..3].iter().all(|event| {
3201 event.unsafe_line() && event.message() == UNSAFE_JOB_LOG_LINE_MARKER
3202 })
3203 );
3204 assert_eq!(poll.events()[3].message(), "after-close");
3205 assert!(!format!("{poll:?}").contains("short-fragment"));
3206 }
3207
3208 #[test]
3209 fn ambiguous_close_and_open_line_remains_armed_until_a_later_close() {
3210 let mut client = TestClient::default();
3211 client.responses.push_back(response(
3212 200,
3213 Some("application/json"),
3214 None,
3215 [metadata(&job(3), 1)],
3216 ));
3217 queue_job_log(
3218 &mut client,
3219 3,
3220 vec![
3221 format!(
3222 "{} {}\nshort-fragment\n{}\nafter-close\n",
3223 private_key_boundary("END"),
3224 private_key_boundary("BEGIN"),
3225 private_key_boundary("END")
3226 )
3227 .into_bytes(),
3228 ],
3229 );
3230 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3231 let poll = fetcher
3232 .fetch_attempt(&identity(), &CancellationToken::new())
3233 .expect("ambiguous private block is projected safely");
3234 assert!(
3235 poll.events()[..3].iter().all(|event| {
3236 event.unsafe_line() && event.message() == UNSAFE_JOB_LOG_LINE_MARKER
3237 })
3238 );
3239 assert_eq!(poll.events()[3].message(), "after-close");
3240 assert!(!format!("{poll:?}").contains("short-fragment"));
3241 }
3242
3243 #[test]
3244 fn conflicting_duplicate_job_identity_fails_closed() {
3245 let conflicting = r#"{"id":3,"run_id":42,"head_sha":"ffffffffffffffffffffffffffffffffffffffff","status":"completed","completed_at":"2020-01-20T17:44:39Z"}"#;
3246 let mut client = TestClient::default();
3247 client.responses.push_back(response(
3248 200,
3249 Some("application/json"),
3250 None,
3251 [metadata(&format!("{},{}", job(3), conflicting), 1)],
3252 ));
3253 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3254 assert_eq!(
3255 fetcher.fetch_attempt(&identity(), &CancellationToken::new()),
3256 Err(GithubJobLogError::JobIdentityMismatch)
3257 );
3258 }
3259
3260 #[test]
3261 fn replay_produces_identical_sequences_and_sanitized_digests() {
3262 fn fetch() -> GithubJobLogPoll {
3263 let mut client = TestClient::default();
3264 client.responses.push_back(response(
3265 200,
3266 Some("application/json"),
3267 None,
3268 [metadata(&job(3), 1)],
3269 ));
3270 queue_job_log(
3271 &mut client,
3272 3,
3273 vec![b"token=never-in-digest\nsafe\n".to_vec()],
3274 );
3275 GithubJobLogFetcher::new(client, limits(1024, 1024))
3276 .fetch_attempt(&identity(), &CancellationToken::new())
3277 .expect("fetch")
3278 }
3279 let first = fetch();
3280 let second = fetch();
3281 assert_eq!(first, second);
3282 assert!(first.events()[0].unsafe_line());
3283 assert_eq!(first.events()[0].message(), UNSAFE_JOB_LOG_LINE_MARKER);
3284 assert!(!format!("{first:?}").contains("never-in-digest"));
3285 }
3286
3287 #[test]
3288 fn store_contract_reserves_completion_and_exposes_256_record_batches() {
3289 let defaults = GithubJobLogLimits::secure_defaults();
3290 assert_eq!(defaults.events, MAX_WORKER_LOG_EVENTS);
3291 assert_eq!(defaults.projected_bytes, MAX_WORKER_LOG_PROJECTED_BYTES);
3292 assert_eq!(MAX_WORKER_LOG_EVENTS + 1, MAX_WORKER_LOG_RECORDS);
3293 assert_eq!(MAX_WORKER_LOG_APPEND_BATCH, 256);
3294 assert_eq!(
3295 GithubJobLogLimits::new(
3296 1,
3297 1,
3298 1,
3299 MAX_WORKER_LOG_EVENTS + 1,
3300 MAX_WORKER_LOG_PROJECTED_BYTES,
3301 1024,
3302 1024,
3303 1024,
3304 Duration::from_secs(1),
3305 Duration::from_secs(1),
3306 1,
3307 ),
3308 Err(GithubJobLogConfigError::OutOfRange {
3309 field: "log events",
3310 minimum: 1,
3311 maximum: MAX_WORKER_LOG_EVENTS as u64,
3312 })
3313 );
3314
3315 let mut client = TestClient::default();
3316 client.responses.push_back(response(
3317 200,
3318 Some("application/json"),
3319 None,
3320 [metadata(&job(3), 1)],
3321 ));
3322 queue_job_log(&mut client, 3, vec!["line\n".repeat(257).into_bytes()]);
3323 let mut fetcher = GithubJobLogFetcher::new(client, limits(4096, 4096));
3324 let poll = fetcher
3325 .fetch_attempt(&identity(), &CancellationToken::new())
3326 .expect("bounded poll");
3327 assert_eq!(
3328 poll.event_batches().map(<[_]>::len).collect::<Vec<_>>(),
3329 [256, 1]
3330 );
3331 let completion = poll.completion_marker();
3332 assert_eq!(completion.code(), WORKER_LOGS_COMPLETE_CODE);
3333 assert_eq!(completion.job_count(), 1);
3334 assert_eq!(completion.event_count(), 257);
3335 assert_eq!(completion.occurred_at_ms(), 1_579_542_279_123);
3336 assert!(
3337 !poll
3338 .events()
3339 .iter()
3340 .any(|event| event.source_sequence() == completion.source_sequence())
3341 );
3342 assert_eq!(
3343 poll.projected_store_bytes(),
3344 poll.events()
3345 .iter()
3346 .map(GithubWorkerLogEvent::projected_store_bytes)
3347 .sum::<u64>()
3348 + completion.projected_store_bytes()
3349 );
3350 assert!(completion.message().contains("event_count=257"));
3351 assert_eq!(completion.source_event_sha256().len(), 64);
3352 }
3353
3354 #[test]
3355 fn durable_completion_verifier_binds_every_non_message_component() {
3356 let poll = one_line_poll();
3357 let identity = poll.identity();
3358 let marker = poll.completion_marker();
3359 assert_eq!(
3360 identity.validate_durable_completion_marker(
3361 event_pairs(&poll),
3362 marker.occurred_at_ms(),
3363 marker.source_sequence(),
3364 marker.source_event_sha256(),
3365 marker.message(),
3366 ),
3367 Ok(())
3368 );
3369 assert_eq!(
3370 identity.expected_completion_source_sequence(),
3371 marker.source_sequence()
3372 );
3373
3374 let mut other_identity = identity.clone();
3375 other_identity.repository = Repository::new("other", "repo").expect("repository");
3376 assert_eq!(
3377 other_identity.validate_durable_completion_marker(
3378 event_pairs(&poll),
3379 marker.occurred_at_ms(),
3380 marker.source_sequence(),
3381 marker.source_event_sha256(),
3382 marker.message(),
3383 ),
3384 Err(GithubJobLogError::InvalidCompletionMarker)
3385 );
3386 for (occurred_at_ms, source_sequence, digest) in [
3387 (
3388 marker.occurred_at_ms() + 1,
3389 marker.source_sequence(),
3390 marker.source_event_sha256(),
3391 ),
3392 (
3393 marker.occurred_at_ms(),
3394 marker.source_sequence() - 1,
3395 marker.source_event_sha256(),
3396 ),
3397 (
3398 marker.occurred_at_ms(),
3399 marker.source_sequence(),
3400 "0000000000000000000000000000000000000000000000000000000000000000",
3401 ),
3402 ] {
3403 assert_eq!(
3404 identity.validate_durable_completion_marker(
3405 event_pairs(&poll),
3406 occurred_at_ms,
3407 source_sequence,
3408 digest,
3409 marker.message(),
3410 ),
3411 Err(GithubJobLogError::InvalidCompletionMarker)
3412 );
3413 }
3414 }
3415
3416 #[test]
3417 fn offline_durable_verifier_binds_repository_run_and_digest() {
3418 let poll = one_line_poll();
3419 let identity = poll.identity();
3420 let marker = poll.completion_marker();
3421 let durable = durable_identity(identity);
3422 assert_eq!(
3423 durable.validate_durable_completion_marker(
3424 event_pairs(&poll),
3425 marker.occurred_at_ms(),
3426 marker.source_sequence(),
3427 marker.source_event_sha256(),
3428 marker.message(),
3429 ),
3430 Ok(())
3431 );
3432 assert_eq!(
3433 durable.expected_completion_source_sequence(),
3434 marker.source_sequence()
3435 );
3436
3437 for (repository, run_id, digest) in [
3438 (
3439 Repository::new("other", "repo").expect("repository"),
3440 identity.run_id(),
3441 marker.source_event_sha256(),
3442 ),
3443 (
3444 identity.repository().clone(),
3445 RunId::new(identity.run_id().get() + 1).expect("run id"),
3446 marker.source_event_sha256(),
3447 ),
3448 (
3449 identity.repository().clone(),
3450 identity.run_id(),
3451 "0000000000000000000000000000000000000000000000000000000000000000",
3452 ),
3453 ] {
3454 let forged = GithubDurableRunAttemptLogIdentity::new(
3455 repository,
3456 run_id,
3457 u64::from(identity.attempt().get()),
3458 identity.head_sha().clone(),
3459 identity.workflow_id(),
3460 )
3461 .expect("durable identity shape");
3462 assert_eq!(
3463 forged.validate_durable_completion_marker(
3464 event_pairs(&poll),
3465 marker.occurred_at_ms(),
3466 marker.source_sequence(),
3467 digest,
3468 marker.message(),
3469 ),
3470 Err(GithubJobLogError::InvalidCompletionMarker)
3471 );
3472 }
3473 }
3474
3475 #[test]
3476 fn durable_verifier_requires_the_exact_ordered_worker_event_set() {
3477 let poll = log_poll(b"first\nsecond\n".to_vec());
3478 let marker = poll.completion_marker();
3479 let durable = durable_identity(poll.identity());
3480 let pairs = event_pairs(&poll).collect::<Vec<_>>();
3481 let validate = |worker_events: Vec<(u64, &str)>| {
3482 durable.validate_durable_completion_marker(
3483 worker_events,
3484 marker.occurred_at_ms(),
3485 marker.source_sequence(),
3486 marker.source_event_sha256(),
3487 marker.message(),
3488 )
3489 };
3490 assert_eq!(validate(pairs.clone()), Ok(()));
3491
3492 let extra_sequence = (pairs[1].0 & !u64::from(u16::MAX)) | 3;
3493 let cross_attempt = pairs
3494 .iter()
3495 .map(|(sequence, digest)| (sequence + (1_u64 << 32), *digest))
3496 .collect::<Vec<_>>();
3497 for forged in [
3498 Vec::new(),
3499 vec![pairs[0]],
3500 vec![pairs[0], pairs[1], (extra_sequence, pairs[1].1)],
3501 vec![pairs[0], pairs[0]],
3502 vec![pairs[1], pairs[0]],
3503 cross_attempt,
3504 vec![pairs[0], (marker.source_sequence(), pairs[1].1)],
3505 vec![
3506 (
3507 pairs[0].0,
3508 "0000000000000000000000000000000000000000000000000000000000000000",
3509 ),
3510 pairs[1],
3511 ],
3512 ] {
3513 assert_eq!(
3514 validate(forged),
3515 Err(GithubJobLogError::InvalidCompletionMarker)
3516 );
3517 }
3518
3519 let empty_poll = log_poll(Vec::new());
3520 let empty_marker = empty_poll.completion_marker();
3521 assert_eq!(empty_marker.event_count(), 0);
3522 assert_eq!(
3523 durable_identity(empty_poll.identity()).validate_durable_completion_marker(
3524 event_pairs(&empty_poll),
3525 empty_marker.occurred_at_ms(),
3526 empty_marker.source_sequence(),
3527 empty_marker.source_event_sha256(),
3528 empty_marker.message(),
3529 ),
3530 Ok(())
3531 );
3532 }
3533
3534 #[test]
3535 fn durable_completion_verifier_rejects_noncanonical_message_fields() {
3536 let poll = one_line_poll();
3537 let identity = poll.identity();
3538 let marker = poll.completion_marker();
3539 let mut reordered = marker.message().split(' ').collect::<Vec<_>>();
3540 reordered.swap(0, 1);
3541 let uppercase_hash = marker.job_set_sha256().to_ascii_uppercase();
3542 assert_ne!(uppercase_hash, marker.job_set_sha256());
3543 let missing_field = marker
3544 .message()
3545 .rsplit_once(' ')
3546 .expect("marker has fields")
3547 .0;
3548 for forged_message in [
3549 marker.message().replace("run_id=42", "run_id=43"),
3550 marker.message().replace("job_count=1", "job_count=2"),
3551 marker.message().replace("event_count=1", "event_count=2"),
3552 marker
3553 .message()
3554 .replace(marker.job_set_sha256(), &uppercase_hash),
3555 reordered.join(" "),
3556 format!("{} extra=value", marker.message()),
3557 missing_field.to_owned(),
3558 ] {
3559 let error = identity
3560 .validate_durable_completion_marker(
3561 event_pairs(&poll),
3562 marker.occurred_at_ms(),
3563 marker.source_sequence(),
3564 marker.source_event_sha256(),
3565 &forged_message,
3566 )
3567 .expect_err("forged marker rejected");
3568 assert_eq!(error, GithubJobLogError::InvalidCompletionMarker);
3569 assert!(!format!("{error:?} {error}").contains(&forged_message));
3570 }
3571 }
3572
3573 #[test]
3574 fn source_sequence_boundaries_are_injective_reserved_and_js_safe() {
3575 const JS_MAX_SAFE_INTEGER: u64 = (1_u64 << 53) - 1;
3576 let attempts = [
3577 GithubRunAttempt::new(1).expect("minimum attempt"),
3578 GithubRunAttempt::new(u64::from(u16::MAX)).expect("maximum attempt"),
3579 ];
3580 let jobs = [0, u16::MAX - 1];
3581 let maximum_line = u32::try_from(MAX_WORKER_LOG_EVENTS).expect("event cap fits u32");
3582 let lines = [1, maximum_line];
3583 let mut sequences = std::collections::BTreeSet::new();
3584 for attempt in attempts {
3585 let marker = completion_source_sequence(attempt);
3586 assert!(marker <= JS_MAX_SAFE_INTEGER);
3587 assert!(sequences.insert(marker));
3588 for job_index in jobs {
3589 for line_ordinal in lines {
3590 let sequence =
3591 encode_source_sequence(attempt, job_index, line_ordinal).expect("tuple");
3592 assert!(sequence <= JS_MAX_SAFE_INTEGER);
3593 assert!(sequences.insert(sequence));
3594 assert_ne!(sequence, marker);
3595 }
3596 }
3597 }
3598 assert_eq!(sequences.len(), 10);
3599 assert_eq!(
3600 completion_source_sequence(GithubRunAttempt::new(u64::from(u16::MAX)).unwrap()),
3601 (1_u64 << 48) - 1
3602 );
3603 assert_eq!(
3604 encode_source_sequence(GithubRunAttempt::new(1).unwrap(), u16::MAX, 1),
3605 Err(GithubJobLogError::SourceSequenceOverflow)
3606 );
3607 assert_eq!(
3608 encode_source_sequence(GithubRunAttempt::new(1).unwrap(), 0, maximum_line + 1,),
3609 Err(GithubJobLogError::SourceSequenceOverflow)
3610 );
3611 assert!(GithubRunAttempt::new(u64::from(u16::MAX) + 1).is_err());
3612 }
3613
3614 #[test]
3615 fn projected_store_caps_fail_without_a_completion_marker() {
3616 let mut client = TestClient::default();
3617 client.responses.push_back(response(
3618 200,
3619 Some("application/json"),
3620 None,
3621 [metadata(&job(3), 1)],
3622 ));
3623 queue_job_log(&mut client, 3, vec![b"line\n".to_vec()]);
3624 let mut constrained = limits(1024, 1024);
3625 constrained.projected_bytes = 1;
3626 let mut fetcher = GithubJobLogFetcher::new(client, constrained);
3627 assert_eq!(
3628 fetcher.fetch_attempt(&identity(), &CancellationToken::new()),
3629 Err(GithubJobLogError::ProjectedByteLimitReached)
3630 );
3631
3632 let mut client = TestClient::default();
3633 client.responses.push_back(response(
3634 200,
3635 Some("application/json"),
3636 None,
3637 [metadata(&job(3), 1)],
3638 ));
3639 let mut quote_line = vec![b'"'; MAX_JOB_LOG_LINE_BYTES];
3640 quote_line.push(b'\n');
3641 queue_job_log(&mut client, 3, vec![quote_line]);
3642 let mut fetcher = GithubJobLogFetcher::new(client, limits(8 * 1024, 8 * 1024));
3643 let poll = fetcher
3644 .fetch_attempt(&identity(), &CancellationToken::new())
3645 .expect("oversized JSON projection becomes marker");
3646 assert!(poll.events()[0].unsafe_line());
3647 assert_eq!(poll.events()[0].message(), UNSAFE_JOB_LOG_LINE_MARKER);
3648 assert!(poll.events()[0].projected_store_bytes() <= MAX_PROJECTED_STORE_EVENT_BYTES);
3649 }
3650
3651 #[test]
3652 fn arbitrary_secret_names_urls_ppk_and_workflow_commands_fail_closed() {
3653 let mut client = TestClient::default();
3654 client.responses.push_back(response(
3655 200,
3656 Some("application/json"),
3657 None,
3658 [metadata(&job(3), 1)],
3659 ));
3660 queue_job_log(
3661 &mut client,
3662 3,
3663 vec![
3664 b"MY_TOKEN=my-token-value\nDEPLOY_SECRET=deploy-value\nFOO_PASSWORD=pass-value\nSSH_PRIVATE_KEY=key-value\n"
3665 .to_vec(),
3666 b"https://example.test/private/path-secret\n::add-mask::mask-value\n".to_vec(),
3667 b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ABCD\n"
3668 .to_vec(),
3669 b"PuTTY-User-Key-File-3: ssh-rsa\nEncryption: none\nComment: ppk-secret\nPrivate-Lines: 1\nQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo=\nPrivate-MAC: deadbeef\nafter\n"
3670 .to_vec(),
3671 ],
3672 );
3673 let mut fetcher = GithubJobLogFetcher::new(client, limits(4096, 4096));
3674 let poll = fetcher
3675 .fetch_attempt(&identity(), &CancellationToken::new())
3676 .expect("fetch");
3677 let rendered = format!("{poll:?}");
3678 for secret in [
3679 "my-token-value",
3680 "deploy-value",
3681 "pass-value",
3682 "key-value",
3683 "path-secret",
3684 "mask-value",
3685 "ppk-secret",
3686 "QUJDREV",
3687 ] {
3688 assert!(!rendered.contains(secret));
3689 }
3690 assert!(poll.events()[0].unsafe_line());
3691 assert_eq!(poll.events()[0].message(), UNSAFE_JOB_LOG_LINE_MARKER);
3692 assert!(poll.events()[4].unsafe_line());
3693 assert_eq!(poll.events()[4].message(), UNSAFE_JOB_LOG_LINE_MARKER);
3694 assert!(poll.events()[5].unsafe_line());
3695 assert!(poll.events()[6].unsafe_line());
3696 assert!(
3697 poll.events()[7..13]
3698 .iter()
3699 .all(GithubWorkerLogEvent::unsafe_line)
3700 );
3701 assert_eq!(poll.events()[13].message(), "after");
3702 }
3703
3704 #[test]
3705 fn empty_non_eof_chunks_violate_metadata_and_log_transport_contracts() {
3706 let mut client = TestClient::default();
3707 client.responses.push_back(response(
3708 200,
3709 Some("application/json"),
3710 None,
3711 [Vec::<u8>::new()],
3712 ));
3713 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3714 assert_eq!(
3715 fetcher.fetch_attempt(&identity(), &CancellationToken::new()),
3716 Err(GithubJobLogError::TransportContractViolation)
3717 );
3718
3719 let mut client = TestClient::default();
3720 client.responses.push_back(response(
3721 200,
3722 Some("application/json"),
3723 None,
3724 [metadata(&job(3), 1)],
3725 ));
3726 queue_job_log(&mut client, 3, vec![Vec::new()]);
3727 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3728 assert_eq!(
3729 fetcher.fetch_attempt(&identity(), &CancellationToken::new()),
3730 Err(GithubJobLogError::TransportContractViolation)
3731 );
3732 }
3733
3734 #[test]
3735 fn cancellation_during_get_is_checked_before_response_or_error() {
3736 for fail_during_get in [false, true] {
3737 let cancellation = CancellationToken::new();
3738 let mut client = TestClient {
3739 cancel_during_get: true,
3740 fail_during_get,
3741 ..TestClient::default()
3742 };
3743 client.responses.push_back(response(
3744 200,
3745 Some("application/json"),
3746 None,
3747 [metadata(&job(3), 1)],
3748 ));
3749 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3750 assert_eq!(
3751 fetcher.fetch_attempt(&identity(), &cancellation),
3752 Err(GithubJobLogError::Cancelled)
3753 );
3754 }
3755 }
3756
3757 #[test]
3758 fn cancellation_during_body_read_is_checked_before_eof_or_error() {
3759 for mut metadata_body in [
3760 TestBody::cancelling(Vec::<Vec<u8>>::new()),
3761 TestBody::cancelling_with_error(),
3762 ] {
3763 let cancellation = CancellationToken::new();
3764 assert_eq!(
3765 read_bounded_body(
3766 &mut metadata_body,
3767 1024,
3768 &cancellation,
3769 Deadline::new(Duration::from_secs(1)),
3770 ),
3771 Err(GithubJobLogError::Cancelled)
3772 );
3773 }
3774
3775 for log_body in [
3776 TestBody::cancelling(Vec::<Vec<u8>>::new()),
3777 TestBody::cancelling_with_error(),
3778 ] {
3779 let cancellation = CancellationToken::new();
3780 let fetcher = GithubJobLogFetcher::new(TestClient::default(), limits(1024, 1024));
3781 let mut poll_bytes = 0;
3782 let mut projected_store_bytes = 0;
3783 let mut events = Vec::new();
3784 assert_eq!(
3785 fetcher.project_job_body(
3786 log_body,
3787 &identity(),
3788 BoundJob {
3789 id: GithubActionsJobId::new(3).expect("job id"),
3790 completed_at_ms: 1_579_542_279_123,
3791 },
3792 0,
3793 &cancellation,
3794 Deadline::new(Duration::from_secs(1)),
3795 &mut poll_bytes,
3796 &mut projected_store_bytes,
3797 &mut events,
3798 ),
3799 Err(GithubJobLogError::Cancelled)
3800 );
3801 assert!(events.is_empty());
3802 }
3803 }
3804
3805 #[test]
3806 fn official_missing_attempt_field_and_completed_timestamp_are_strictly_bound() {
3807 assert_eq!(
3808 parse_rfc3339_millis("2020-01-20T17:44:39.123Z"),
3809 Some(1_579_542_279_123)
3810 );
3811 assert_eq!(
3812 parse_rfc3339_millis("2020-01-20T18:44:39.123+01:00"),
3813 Some(1_579_542_279_123)
3814 );
3815 assert_eq!(parse_rfc3339_millis("2021-02-29T00:00:00Z"), None);
3816
3817 let mut client = TestClient::default();
3818 let incomplete = job(3).replace("\"completed\"", "\"in_progress\"");
3819 client.responses.push_back(response(
3820 200,
3821 Some("application/json"),
3822 None,
3823 [metadata(&incomplete, 1)],
3824 ));
3825 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3826 assert_eq!(
3827 fetcher.fetch_attempt(&identity(), &CancellationToken::new()),
3828 Err(GithubJobLogError::JobSetNotTerminal)
3829 );
3830 }
3831
3832 #[test]
3833 fn cancellation_prevents_any_network_request() {
3834 let client = TestClient::default();
3835 let token = CancellationToken::new();
3836 assert!(token.cancel());
3837 let mut fetcher = GithubJobLogFetcher::new(client, limits(1024, 1024));
3838 assert_eq!(
3839 fetcher.fetch_attempt(&identity(), &token),
3840 Err(GithubJobLogError::Cancelled)
3841 );
3842 assert!(fetcher.client().requests.is_empty());
3843 }
3844}