fc_gpui_http_client/
github_download.rs

1use std::{fmt::Write as _, path::Path, pin::Pin, task::Poll};
2
3use anyhow::{Context, Result};
4use async_compression::futures::bufread::GzipDecoder;
5use futures::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, io::BufReader};
6use sha2::{Digest, Sha256};
7
8use crate::{HttpClient, github::AssetKind};
9
10#[derive(serde::Deserialize, serde::Serialize, Debug)]
11pub struct GithubBinaryMetadata {
12    pub metadata_version: u64,
13    pub digest: Option<String>,
14}
15
16impl GithubBinaryMetadata {
17    pub async fn read_from_file(metadata_path: &Path) -> Result<GithubBinaryMetadata> {
18        let metadata_content = async_fs::read_to_string(metadata_path)
19            .await
20            .with_context(|| format!("reading metadata file at {metadata_path:?}"))?;
21        serde_json::from_str(&metadata_content)
22            .with_context(|| format!("parsing metadata file at {metadata_path:?}"))
23    }
24
25    pub async fn write_to_file(&self, metadata_path: &Path) -> Result<()> {
26        let metadata_content = serde_json::to_string(self)
27            .with_context(|| format!("serializing metadata for {metadata_path:?}"))?;
28        async_fs::write(metadata_path, metadata_content.as_bytes())
29            .await
30            .with_context(|| format!("writing metadata file at {metadata_path:?}"))?;
31        Ok(())
32    }
33}
34
35pub async fn download_server_binary(
36    http_client: &dyn HttpClient,
37    url: &str,
38    digest: Option<&str>,
39    destination_path: &Path,
40    asset_kind: AssetKind,
41) -> Result<(), anyhow::Error> {
42    log::info!("downloading github artifact from {url}");
43    let mut response = http_client
44        .get(url, Default::default(), true)
45        .await
46        .with_context(|| format!("downloading release from {url}"))?;
47    let body = response.body_mut();
48    match digest {
49        Some(expected_sha_256) => {
50            let temp_asset_file = tempfile::NamedTempFile::new()
51                .with_context(|| format!("creating a temporary file for {url}"))?;
52            let (temp_asset_file, _temp_guard) = temp_asset_file.into_parts();
53            let mut writer = HashingWriter {
54                writer: async_fs::File::from(temp_asset_file),
55                hasher: Sha256::new(),
56            };
57            futures::io::copy(&mut BufReader::new(body), &mut writer)
58                .await
59                .with_context(|| {
60                    format!("saving archive contents into the temporary file for {url}",)
61                })?;
62            let digest = writer.hasher.finalize();
63            let asset_sha_256 =
64                digest
65                    .iter()
66                    .fold(String::with_capacity(digest.len() * 2), |mut hex, byte| {
67                        let _ = write!(hex, "{byte:02x}");
68                        hex
69                    });
70
71            anyhow::ensure!(
72                asset_sha_256 == expected_sha_256,
73                "{url} asset got SHA-256 mismatch. Expected: {expected_sha_256}, Got: {asset_sha_256}",
74            );
75            writer
76                .writer
77                .seek(std::io::SeekFrom::Start(0))
78                .await
79                .with_context(|| format!("seeking temporary file {destination_path:?}",))?;
80            stream_file_archive(&mut writer.writer, url, destination_path, asset_kind)
81                .await
82                .with_context(|| {
83                    format!("extracting downloaded asset for {url} into {destination_path:?}",)
84                })?;
85        }
86        None => stream_response_archive(body, url, destination_path, asset_kind)
87            .await
88            .with_context(|| {
89                format!("extracting response for asset {url} into {destination_path:?}",)
90            })?,
91    }
92    Ok(())
93}
94
95async fn stream_response_archive(
96    response: impl AsyncRead + Unpin,
97    url: &str,
98    destination_path: &Path,
99    asset_kind: AssetKind,
100) -> Result<()> {
101    match asset_kind {
102        AssetKind::TarGz => extract_tar_gz(destination_path, url, response).await?,
103        AssetKind::Gz => extract_gz(destination_path, url, response).await?,
104        AssetKind::Zip => {
105            util::archive::extract_zip(destination_path, response).await?;
106        }
107    };
108    Ok(())
109}
110
111async fn stream_file_archive(
112    file_archive: impl AsyncRead + AsyncSeek + Unpin,
113    url: &str,
114    destination_path: &Path,
115    asset_kind: AssetKind,
116) -> Result<()> {
117    match asset_kind {
118        AssetKind::TarGz => extract_tar_gz(destination_path, url, file_archive).await?,
119        AssetKind::Gz => extract_gz(destination_path, url, file_archive).await?,
120        #[cfg(not(windows))]
121        AssetKind::Zip => {
122            util::archive::extract_seekable_zip(destination_path, file_archive).await?;
123        }
124        #[cfg(windows)]
125        AssetKind::Zip => {
126            util::archive::extract_zip(destination_path, file_archive).await?;
127        }
128    };
129    Ok(())
130}
131
132async fn extract_tar_gz(
133    destination_path: &Path,
134    url: &str,
135    from: impl AsyncRead + Unpin,
136) -> Result<(), anyhow::Error> {
137    let decompressed_bytes = GzipDecoder::new(BufReader::new(from));
138    let archive = async_tar::Archive::new(decompressed_bytes);
139    archive
140        .unpack(&destination_path)
141        .await
142        .with_context(|| format!("extracting {url} to {destination_path:?}"))?;
143    Ok(())
144}
145
146async fn extract_gz(
147    destination_path: &Path,
148    url: &str,
149    from: impl AsyncRead + Unpin,
150) -> Result<(), anyhow::Error> {
151    let mut decompressed_bytes = GzipDecoder::new(BufReader::new(from));
152    let mut file = async_fs::File::create(&destination_path)
153        .await
154        .with_context(|| {
155            format!("creating a file {destination_path:?} for a download from {url}")
156        })?;
157    futures::io::copy(&mut decompressed_bytes, &mut file)
158        .await
159        .with_context(|| format!("extracting {url} to {destination_path:?}"))?;
160    Ok(())
161}
162
163struct HashingWriter<W: AsyncWrite + Unpin> {
164    writer: W,
165    hasher: Sha256,
166}
167
168impl<W: AsyncWrite + Unpin> AsyncWrite for HashingWriter<W> {
169    fn poll_write(
170        mut self: Pin<&mut Self>,
171        cx: &mut std::task::Context<'_>,
172        buf: &[u8],
173    ) -> Poll<std::result::Result<usize, std::io::Error>> {
174        match Pin::new(&mut self.writer).poll_write(cx, buf) {
175            Poll::Ready(Ok(n)) => {
176                self.hasher.update(&buf[..n]);
177                Poll::Ready(Ok(n))
178            }
179            other => other,
180        }
181    }
182
183    fn poll_flush(
184        mut self: Pin<&mut Self>,
185        cx: &mut std::task::Context<'_>,
186    ) -> Poll<Result<(), std::io::Error>> {
187        Pin::new(&mut self.writer).poll_flush(cx)
188    }
189
190    fn poll_close(
191        mut self: Pin<&mut Self>,
192        cx: &mut std::task::Context<'_>,
193    ) -> Poll<std::result::Result<(), std::io::Error>> {
194        Pin::new(&mut self.writer).poll_close(cx)
195    }
196}
197
198#[cfg(test)]
199mod tests {
200    use async_compression::futures::write::GzipEncoder;
201    use async_tar::{EntryType, Header};
202    use futures::{AsyncWriteExt, io::Cursor};
203
204    use super::extract_tar_gz;
205
206    fn append_entry(
207        archive: &mut Vec<u8>,
208        path: &str,
209        entry_type: EntryType,
210        mode: u32,
211        body: &[u8],
212    ) {
213        let mut header = Header::new_gnu();
214        header.set_path(path).unwrap();
215        header.set_entry_type(entry_type);
216        header.set_mode(mode);
217        header.set_uid(0);
218        header.set_gid(0);
219        header.set_mtime(0);
220        header.set_size(body.len() as u64);
221        header.set_cksum();
222        archive.extend_from_slice(header.as_bytes());
223        archive.extend_from_slice(body);
224        let padding = (512 - body.len() % 512) % 512;
225        archive.extend(std::iter::repeat_n(0, padding));
226    }
227
228    async fn gzip_tar(archive: Vec<u8>) -> Cursor<Vec<u8>> {
229        let mut encoder = GzipEncoder::new(Cursor::new(Vec::new()));
230        encoder.write_all(&archive).await.unwrap();
231        encoder.close().await.unwrap();
232        let mut cursor = encoder.into_inner();
233        cursor.set_position(0);
234        cursor
235    }
236
237    #[test]
238    fn test_extract_tar_gz_preserves_content_and_permissions() {
239        futures::executor::block_on(async {
240            let content: Vec<u8> = (0u16..1024).map(|b| (b % 256) as u8).collect();
241            let mut archive = Vec::new();
242            append_entry(
243                &mut archive,
244                "nested/file.txt",
245                EntryType::Regular,
246                0o755,
247                &content,
248            );
249            archive.extend_from_slice(&[0u8; 1024]);
250            let reader = gzip_tar(archive).await;
251
252            let dir = tempfile::tempdir().unwrap();
253            extract_tar_gz(dir.path(), "test://archive.tar.gz", reader)
254                .await
255                .unwrap();
256
257            let extracted = dir.path().join("nested/file.txt");
258            assert_eq!(std::fs::read(&extracted).unwrap(), content);
259            #[cfg(unix)]
260            {
261                use std::os::unix::fs::PermissionsExt;
262                let mode = std::fs::metadata(&extracted).unwrap().permissions().mode();
263                assert_eq!(mode & 0o777, 0o755);
264            }
265        });
266    }
267
268    #[test]
269    fn test_extract_tar_gz_pax_size_before_gnu_longname() {
270        futures::executor::block_on(async {
271            let mut archive = Vec::new();
272            append_entry(
273                &mut archive,
274                "PaxHeader",
275                EntryType::XHeader,
276                0o644,
277                b"13 size=1024\n",
278            );
279            append_entry(
280                &mut archive,
281                "././@LongLink",
282                EntryType::GNULongName,
283                0o644,
284                b"nested/pax-size.txt\0",
285            );
286            append_entry(
287                &mut archive,
288                "placeholder",
289                EntryType::Regular,
290                0o644,
291                &vec![b'A'; 1024],
292            );
293            // A later entry with its own size. The pax `size` above must not leak
294            // onto this header or body.
295            append_entry(
296                &mut archive,
297                "nested/after.txt",
298                EntryType::Regular,
299                0o644,
300                b"ok",
301            );
302            archive.extend_from_slice(&[0u8; 1024]);
303            let reader = gzip_tar(archive).await;
304
305            let dir = tempfile::tempdir().unwrap();
306            extract_tar_gz(dir.path(), "test://archive.tar.gz", reader)
307                .await
308                .unwrap();
309
310            let extracted = dir.path().join("nested/pax-size.txt");
311            assert!(extracted.exists(), "nested/pax-size.txt not extracted");
312            assert_eq!(std::fs::read(&extracted).unwrap(), vec![b'A'; 1024]);
313            assert!(
314                !dir.path().join("placeholder").exists(),
315                "placeholder file must not be created"
316            );
317            assert_eq!(
318                std::fs::read(dir.path().join("nested/after.txt")).unwrap(),
319                b"ok"
320            );
321        });
322    }
323
324    fn append_symlink(archive: &mut Vec<u8>, path: &str, target: &str) {
325        let mut header = Header::new_gnu();
326        header.set_path(path).unwrap();
327        header.set_link_name(target).unwrap();
328        header.set_entry_type(EntryType::Symlink);
329        header.set_mode(0o777);
330        header.set_uid(0);
331        header.set_gid(0);
332        header.set_mtime(0);
333        header.set_size(0);
334        header.set_cksum();
335        archive.extend_from_slice(header.as_bytes());
336    }
337
338    #[cfg(unix)]
339    #[test]
340    fn test_extract_tar_gz_symlink() {
341        futures::executor::block_on(async {
342            let mut archive = Vec::new();
343            append_entry(
344                &mut archive,
345                "nested/file.txt",
346                EntryType::Regular,
347                0o644,
348                b"target",
349            );
350            append_symlink(&mut archive, "nested/link.txt", "file.txt");
351            archive.extend_from_slice(&[0u8; 1024]);
352            let reader = gzip_tar(archive).await;
353
354            let dir = tempfile::tempdir().unwrap();
355            extract_tar_gz(dir.path(), "test://archive.tar.gz", reader)
356                .await
357                .unwrap();
358
359            let link = dir.path().join("nested/link.txt");
360            assert_eq!(
361                std::fs::read_link(&link).unwrap(),
362                std::path::Path::new("file.txt")
363            );
364            assert_eq!(std::fs::read(&link).unwrap(), b"target");
365        });
366    }
367}
fc_gpui_http_client/github_download.rs

fc_gpui_http_client/
github_download.rs
