scone/
connectors.rs

1//! Cloud connectors: pull documents out of the services people already
2//! keep their thinking in, and hand them to the engine as episodes.
3//!
4//! Network lives here in the CLI layer, exactly like [`crate::web`];
5//! scone-core stays offline by construction. Credentials live in a
6//! 0600 file next to the database, or in the environment for people who
7//! would rather not have them on disk at all.
8
9use std::io::Read;
10use std::path::{Path, PathBuf};
11use std::time::Duration;
12
13const TIMEOUT: Duration = Duration::from_secs(30);
14const MAX_BODY_BYTES: u64 = 5_000_000;
15const NOTION_VERSION: &str = "2022-06-28";
16
17/// One fetched thing, normalized across services.
18#[derive(Debug, PartialEq)]
19pub struct Document {
20    /// Stable id at the source, so re-syncs land on the same episode.
21    pub id: String,
22    pub title: String,
23    /// Where a human would go to read it.
24    pub url: String,
25    pub body: String,
26    /// Source's own timestamp, RFC3339. Memory is dated by when the
27    /// thing happened, never by when we happened to sync it.
28    pub updated_at: Option<String>,
29}
30
31/// What every connector must do: name itself and produce documents.
32pub trait Connector {
33    fn name(&self) -> &'static str;
34    /// Fetch documents changed since `since` (RFC3339), or everything.
35    fn fetch(&self, since: Option<&str>) -> Result<Vec<Document>, String>;
36}
37
38/// Stored connector credentials and sync state.
39#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
40pub struct Credentials {
41    #[serde(default)]
42    pub providers: std::collections::BTreeMap<String, Provider>,
43}
44
45#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
46pub struct Provider {
47    pub token: String,
48    /// RFC3339 timestamp of the last successful sync, for incrementals.
49    #[serde(default, skip_serializing_if = "Option::is_none")]
50    pub last_sync: Option<String>,
51}
52
53pub fn credentials_path(data_dir: &Path) -> PathBuf {
54    data_dir.join("connectors.toml")
55}
56
57pub fn load_credentials(data_dir: &Path) -> Result<Credentials, String> {
58    let path = credentials_path(data_dir);
59    let raw = std::fs::read_to_string(&path).unwrap_or_default();
60    if raw.trim().is_empty() {
61        return Ok(Credentials::default());
62    }
63    toml::from_str(&raw).map_err(|e| format!("{}: {e}", path.display()))
64}
65
66/// Write credentials readable only by this user. A token in a
67/// world-readable file is a token you have given away.
68pub fn save_credentials(data_dir: &Path, creds: &Credentials) -> Result<(), String> {
69    let path = credentials_path(data_dir);
70    std::fs::create_dir_all(data_dir).map_err(|e| e.to_string())?;
71    let body = toml::to_string_pretty(creds).map_err(|e| e.to_string())?;
72    std::fs::write(&path, body).map_err(|e| format!("{}: {e}", path.display()))?;
73    restrict(&path)
74}
75
76#[cfg(unix)]
77fn restrict(path: &Path) -> Result<(), String> {
78    use std::os::unix::fs::PermissionsExt;
79    std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))
80        .map_err(|e| format!("{}: {e}", path.display()))
81}
82
83#[cfg(not(unix))]
84fn restrict(_path: &Path) -> Result<(), String> {
85    Ok(())
86}
87
88/// Resolve a provider's token: environment first so nothing has to be
89/// written down, then the credentials file.
90pub fn token_for(provider: &str, creds: &Credentials) -> Option<String> {
91    let var = format!("SCONE_{}_TOKEN", provider.to_uppercase().replace('-', "_"));
92    if let Some(v) = std::env::var_os(&var) {
93        let v = v.to_string_lossy().trim().to_owned();
94        if !v.is_empty() {
95            return Some(v);
96        }
97    }
98    creds
99        .providers
100        .get(provider)
101        .map(|p| p.token.clone())
102        .filter(|t| !t.is_empty())
103}
104
105pub fn connector_for(provider: &str, token: String) -> Result<Box<dyn Connector>, String> {
106    match provider {
107        "notion" => Ok(Box::new(Notion { token })),
108        other => Err(format!(
109            "unknown connector {other:?}: known connectors are {}",
110            KNOWN.join(", ")
111        )),
112    }
113}
114
115pub const KNOWN: &[&str] = &["notion"];
116
117/// Notion, via an internal integration token. No OAuth app to register:
118/// create an integration, share the pages with it, paste the token.
119pub struct Notion {
120    pub token: String,
121}
122
123impl Connector for Notion {
124    fn name(&self) -> &'static str {
125        "notion"
126    }
127
128    fn fetch(&self, since: Option<&str>) -> Result<Vec<Document>, String> {
129        let body = serde_json::json!({
130            "filter": {"property": "object", "value": "page"},
131            "sort": {"direction": "descending", "timestamp": "last_edited_time"},
132            "page_size": 100,
133        });
134        let value = self.post("https://api.notion.com/v1/search", body)?;
135        let mut docs = parse_search(&value, since);
136        for doc in &mut docs {
137            doc.body = self.page_text(&doc.id)?;
138        }
139        docs.retain(|d| !d.body.trim().is_empty());
140        Ok(docs)
141    }
142}
143
144impl Notion {
145    fn post(&self, url: &str, body: serde_json::Value) -> Result<serde_json::Value, String> {
146        let mut res = ureq::post(url)
147            .header("authorization", format!("Bearer {}", self.token))
148            .header("notion-version", NOTION_VERSION)
149            .config()
150            .timeout_global(Some(TIMEOUT))
151            .build()
152            .send_json(&body)
153            .map_err(|e| format!("notion: {e}"))?;
154        read_json(res.body_mut().as_reader())
155    }
156
157    fn get(&self, url: &str) -> Result<serde_json::Value, String> {
158        let mut res = ureq::get(url)
159            .header("authorization", format!("Bearer {}", self.token))
160            .header("notion-version", NOTION_VERSION)
161            .config()
162            .timeout_global(Some(TIMEOUT))
163            .build()
164            .call()
165            .map_err(|e| format!("notion: {e}"))?;
166        read_json(res.body_mut().as_reader())
167    }
168
169    fn page_text(&self, page_id: &str) -> Result<String, String> {
170        let url = format!("https://api.notion.com/v1/blocks/{page_id}/children?page_size=100");
171        let value = self.get(&url)?;
172        Ok(blocks_to_text(&value))
173    }
174}
175
176fn read_json(reader: impl Read) -> Result<serde_json::Value, String> {
177    let mut buf = String::new();
178    reader
179        .take(MAX_BODY_BYTES)
180        .read_to_string(&mut buf)
181        .map_err(|e| e.to_string())?;
182    serde_json::from_str(&buf).map_err(|e| format!("unreadable response: {e}"))
183}
184
185/// Pull documents out of a Notion search payload. Pure, so the shape of
186/// their API is tested without touching the network.
187pub fn parse_search(value: &serde_json::Value, since: Option<&str>) -> Vec<Document> {
188    let mut out = Vec::new();
189    let Some(results) = value["results"].as_array() else {
190        return out;
191    };
192    for page in results {
193        let Some(id) = page["id"].as_str() else {
194            continue;
195        };
196        let edited = page["last_edited_time"].as_str().map(str::to_owned);
197        // Incremental: the source's own clock decides, not ours.
198        if let (Some(since), Some(edited)) = (since, edited.as_deref()) {
199            if edited <= since {
200                continue;
201            }
202        }
203        out.push(Document {
204            id: id.to_owned(),
205            title: page_title(page),
206            url: page["url"].as_str().unwrap_or_default().to_owned(),
207            body: String::new(),
208            updated_at: edited,
209        });
210    }
211    out
212}
213
214/// Notion hides the title under whichever property happens to be of
215/// type `title`, which is not always called "title".
216fn page_title(page: &serde_json::Value) -> String {
217    if let Some(props) = page["properties"].as_object() {
218        for value in props.values() {
219            if value["type"] == "title" {
220                let text = rich_text(&value["title"]);
221                if !text.trim().is_empty() {
222                    return text;
223                }
224            }
225        }
226    }
227    "untitled".to_owned()
228}
229
230/// Flatten a Notion block payload into plain text. Unsupported block
231/// types contribute nothing rather than failing the page.
232pub fn blocks_to_text(value: &serde_json::Value) -> String {
233    let Some(results) = value["results"].as_array() else {
234        return String::new();
235    };
236    let mut lines = Vec::new();
237    for block in results {
238        let Some(kind) = block["type"].as_str() else {
239            continue;
240        };
241        let text = rich_text(&block[kind]["rich_text"]);
242        if text.trim().is_empty() {
243            continue;
244        }
245        lines.push(match kind {
246            "heading_1" => format!("# {text}"),
247            "heading_2" => format!("## {text}"),
248            "heading_3" => format!("### {text}"),
249            "bulleted_list_item" | "numbered_list_item" => format!("- {text}"),
250            "to_do" => {
251                let done = block[kind]["checked"].as_bool().unwrap_or(false);
252                format!("- [{}] {text}", if done { "x" } else { " " })
253            }
254            "code" => format!("    {text}"),
255            "quote" => format!("> {text}"),
256            _ => text,
257        });
258    }
259    lines.join("\n")
260}
261
262fn rich_text(value: &serde_json::Value) -> String {
263    value
264        .as_array()
265        .map(|parts| {
266            parts
267                .iter()
268                .filter_map(|p| p["plain_text"].as_str())
269                .collect::<String>()
270        })
271        .unwrap_or_default()
272}
273
274#[cfg(test)]
275mod tests {
276    use super::*;
277
278    fn search_payload() -> serde_json::Value {
279        serde_json::json!({"results": [
280            {
281                "id": "abc",
282                "url": "https://notion.so/abc",
283                "last_edited_time": "2026-09-01T10:00:00.000Z",
284                "properties": {"Name": {"type": "title", "title": [{"plain_text": "Design notes"}]}}
285            },
286            {
287                "id": "old",
288                "url": "https://notion.so/old",
289                "last_edited_time": "2026-08-01T10:00:00.000Z",
290                "properties": {"Name": {"type": "title", "title": [{"plain_text": "Stale"}]}}
291            }
292        ]})
293    }
294
295    #[test]
296    fn search_yields_titled_documents() {
297        let docs = parse_search(&search_payload(), None);
298        assert_eq!(docs.len(), 2);
299        assert_eq!(docs[0].title, "Design notes");
300        assert_eq!(docs[0].url, "https://notion.so/abc");
301        assert_eq!(
302            docs[0].updated_at.as_deref(),
303            Some("2026-09-01T10:00:00.000Z")
304        );
305    }
306
307    /// A resync must not re-fetch what has not changed; the source's own
308    /// edit time decides, so a slow sync cannot skip a page.
309    #[test]
310    fn search_honors_the_incremental_cursor() {
311        let docs = parse_search(&search_payload(), Some("2026-08-15T00:00:00.000Z"));
312        assert_eq!(docs.len(), 1, "only the page edited after the cursor");
313        assert_eq!(docs[0].id, "abc");
314    }
315
316    #[test]
317    fn untitled_pages_still_come_through() {
318        let payload = serde_json::json!({"results": [
319            {"id": "x", "url": "u", "last_edited_time": "2026-09-01T00:00:00.000Z",
320             "properties": {}}
321        ]});
322        assert_eq!(parse_search(&payload, None)[0].title, "untitled");
323    }
324
325    #[test]
326    fn blocks_flatten_to_readable_text() {
327        let payload = serde_json::json!({"results": [
328            {"type": "heading_1", "heading_1": {"rich_text": [{"plain_text": "Title"}]}},
329            {"type": "paragraph", "paragraph": {"rich_text": [{"plain_text": "Body text"}]}},
330            {"type": "bulleted_list_item", "bulleted_list_item": {"rich_text": [{"plain_text": "point"}]}},
331            {"type": "to_do", "to_do": {"rich_text": [{"plain_text": "ship it"}], "checked": true}},
332            {"type": "paragraph", "paragraph": {"rich_text": []}},
333            {"type": "unsupported_block", "unsupported_block": {}}
334        ]});
335        let text = blocks_to_text(&payload);
336        assert_eq!(text, "# Title\nBody text\n- point\n- [x] ship it");
337    }
338
339    #[test]
340    fn missing_fields_never_panic() {
341        assert!(parse_search(&serde_json::json!({}), None).is_empty());
342        assert_eq!(blocks_to_text(&serde_json::json!({})), "");
343    }
344
345    #[test]
346    fn environment_token_wins_over_the_file() {
347        let mut creds = Credentials::default();
348        creds.providers.insert(
349            "notion".into(),
350            Provider {
351                token: "from-file".into(),
352                last_sync: None,
353            },
354        );
355        assert_eq!(token_for("notion", &creds).as_deref(), Some("from-file"));
356        // SAFETY: single-threaded test, no other reader of this var.
357        unsafe { std::env::set_var("SCONE_NOTION_TOKEN", "from-env") };
358        assert_eq!(token_for("notion", &creds).as_deref(), Some("from-env"));
359        unsafe { std::env::remove_var("SCONE_NOTION_TOKEN") };
360    }
361
362    #[test]
363    fn credentials_round_trip_and_stay_private() {
364        let dir = tempfile::tempdir().unwrap();
365        let mut creds = Credentials::default();
366        creds.providers.insert(
367            "notion".into(),
368            Provider {
369                token: "secret".into(),
370                last_sync: Some("2026-09-01T00:00:00Z".into()),
371            },
372        );
373        save_credentials(dir.path(), &creds).unwrap();
374        let back = load_credentials(dir.path()).unwrap();
375        assert_eq!(back.providers["notion"].token, "secret");
376        assert_eq!(
377            back.providers["notion"].last_sync.as_deref(),
378            Some("2026-09-01T00:00:00Z")
379        );
380        #[cfg(unix)]
381        {
382            use std::os::unix::fs::PermissionsExt;
383            let mode = std::fs::metadata(credentials_path(dir.path()))
384                .unwrap()
385                .permissions()
386                .mode();
387            assert_eq!(mode & 0o777, 0o600, "a token file must not be readable");
388        }
389    }
390
391    #[test]
392    fn unknown_connector_is_named_in_the_error() {
393        let err = match connector_for("dropbox", "t".into()) {
394            Err(e) => e,
395            Ok(c) => panic!("dropbox should not resolve, got {}", c.name()),
396        };
397        assert!(err.contains("dropbox") && err.contains("notion"), "{err}");
398    }
399}
scone/connectors.rs

scone/
connectors.rs
