instagrab 0.0.1

Scrapes Instagram profile metadata by attaching to a real Chrome over CDP
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

//! Plain-HTTPS download of display_url, idempotent.

use crate::parse::ScrapeResult;
use crate::shutdown;
use anyhow::{Result, bail};
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::time::Duration;

const USER_AGENT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";

pub struct Downloader {
    dir: String,
    agent: ureq::Agent,
}

pub fn new(dir: &str) -> Downloader {
    Downloader {
        dir: dir.to_string(),
        agent: ureq::AgentBuilder::new()
            .timeout(Duration::from_secs(30))
            .build(),
    }
}

impl Downloader {
    /// Downloads display_url for each post in r.recent_posts that has one,
    /// writes <dir>/<username>/<shortcode>.jpg, and sets local_path. Existing
    /// files are kept (idempotent across runs). Per-post errors are appended
    /// to r.errors and don't stop the loop.
    pub fn fill(&self, r: &mut ScrapeResult) {
        if self.dir.is_empty() {
            return;
        }
        let posts = match &mut r.recent_posts {
            Some(p) if !p.is_empty() => p,
            _ => return,
        };
        let udir = Path::new(&self.dir).join(&r.username);
        if let Err(e) = fs::create_dir_all(&udir) {
            r.errors.push(format!("images_mkdir: {e}"));
            return;
        }
        for p in posts.iter_mut() {
            if shutdown::requested() {
                return;
            }
            let display_url = match p.display_url.as_deref() {
                Some(u) if !u.is_empty() => u,
                _ => continue,
            };
            let target = udir.join(format!("{}.jpg", p.shortcode));
            if target.exists() {
                p.local_path = Some(target.to_string_lossy().into_owned());
                continue;
            }
            if let Err(e) = self.fetch_one(display_url, &target) {
                r.errors.push(format!("image {}: {}", p.shortcode, e));
                continue;
            }
            p.local_path = Some(target.to_string_lossy().into_owned());
        }
    }

    fn fetch_one(&self, url: &str, target: &PathBuf) -> Result<()> {
        let resp = match self
            .agent
            .get(url)
            .set("User-Agent", USER_AGENT)
            .set("Referer", "https://www.instagram.com/")
            .set(
                "Accept",
                "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
            )
            .call()
        {
            Ok(resp) => resp,
            Err(ureq::Error::Status(code, resp)) => {
                bail!("http {} {}", code, resp.status_text())
            }
            Err(e) => bail!(e),
        };

        let tmp = target.with_extension("jpg.tmp");
        let mut f = fs::File::create(&tmp)?;
        let mut reader = resp.into_reader();
        if let Err(e) = io::copy(&mut reader, &mut f) {
            drop(f);
            let _ = fs::remove_file(&tmp);
            bail!(e);
        }
        if let Err(e) = f.sync_all() {
            drop(f);
            let _ = fs::remove_file(&tmp);
            bail!(e);
        }
        drop(f);
        fs::rename(&tmp, target)?;
        Ok(())
    }
}