use std::sync::OnceLock;
use crate::sources::providers::open_connector::error::OpenConnectorError;
use crate::sources::providers::open_connector::source_pack::SourcePack;
use super::loader;
static PACK: OnceLock<Result<SourcePack, String>> = OnceLock::new();
pub fn pack() -> Result<&'static SourcePack, OpenConnectorError> {
loader::builtin(
"google_drive.yaml",
include_str!("google_drive.yaml"),
&PACK,
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sources::hierarchy::HierarchyLevel;
use crate::sources::providers::open_connector::action_registry::fingerprint_schema;
use crate::sources::providers::open_connector::json_to_arrow::RowConverter;
use crate::sources::providers::open_connector::pagination::{AbsentCursor, PaginationStrategy};
use crate::sources::providers::open_connector::row_path::RowPath;
use crate::sources::providers::open_connector::source_pack::{FixedValue, SourcePackTable};
use crate::sources::providers::open_connector::testutil::{
EnvVarGuard, MockGateway, MockResponse, boolean, collect, column_values,
convert_first_page, discovery_ok, envelope_err, envelope_ok, execute_inputs,
fingerprint_uncovered_columns, input_keys, utf8,
};
use crate::sources::providers::open_connector::{
OpenConnectorConfig, OpenConnectorGateways, register_open_connector_tables,
register_open_connector_udtfs,
};
use arrow::array::{Array, Int64Array, ListArray, StringArray, TimestampMillisecondArray};
use arrow::record_batch::RecordBatch;
use datafusion::prelude::SessionContext;
use serde_json::{Value, json};
const FILES_PAGE2_TOKEN: &str = "~!!~AI9SyntheticFilesToken2";
fn table(short: &str) -> &'static SourcePackTable {
pack()
.expect("embedded asset is test-pinned to parse")
.tables
.iter()
.find(|t| t.id.rsplit('.').next() == Some(short))
.unwrap_or_else(|| panic!("table {short}"))
}
fn google_drive_discovery(path: &str) -> MockResponse {
let output_schema = if path.ends_with("googledrive.files.list") {
include_str!("fixtures/google_drive/contracts/files.list.json")
} else if path.ends_with("googledrive.drives.list") {
include_str!("fixtures/google_drive/contracts/drives.list.json")
} else if path.ends_with("googledrive.permissions.list") {
include_str!("fixtures/google_drive/contracts/permissions.list.json")
} else {
r#"{"type": "object"}"#
};
MockResponse::ok(&discovery_ok("{}", output_schema, true, None))
}
fn convert_fixture(table: &SourcePackTable, fixture: &str) -> RecordBatch {
let page: Value = serde_json::from_str(fixture).expect("fixture parses");
convert_first_page(table, &page)
}
fn int64<'a>(batch: &'a RecordBatch, name: &str) -> &'a Int64Array {
batch
.column_by_name(name)
.unwrap_or_else(|| panic!("column {name}"))
.as_any()
.downcast_ref()
.expect("Int64 column")
}
fn timestamp<'a>(batch: &'a RecordBatch, name: &str) -> &'a TimestampMillisecondArray {
batch
.column_by_name(name)
.unwrap_or_else(|| panic!("column {name}"))
.as_any()
.downcast_ref()
.expect("Timestamp column")
}
fn utf8_list<'a>(batch: &'a RecordBatch, name: &str) -> &'a ListArray {
batch
.column_by_name(name)
.unwrap_or_else(|| panic!("column {name}"))
.as_any()
.downcast_ref()
.expect("List column")
}
fn list_items(lists: &ListArray, row: usize) -> Vec<Option<String>> {
let values = lists.value(row);
let values = values
.as_any()
.downcast_ref::<StringArray>()
.expect("Utf8 items");
(0..values.len())
.map(|i| values.is_valid(i).then(|| values.value(i).to_string()))
.collect()
}
#[test]
fn files_fixture_converts_every_row_shape() {
let batch = convert_fixture(
table("files"),
include_str!("fixtures/google_drive/files.json"),
);
assert_eq!(batch.num_rows(), 6);
assert_eq!(utf8(&batch, "name").value(0), "notes.txt");
assert_eq!(utf8(&batch, "mime_type").value(0), "text/plain");
assert_eq!(int64(&batch, "size_bytes").value(0), 63);
assert!(utf8(&batch, "drive_id").is_null(0));
assert!(
utf8(&batch, "web_view_link")
.value(0)
.starts_with("https://drive.google.com/file/d/")
);
assert!(timestamp(&batch, "created_time").is_valid(0));
assert!(timestamp(&batch, "modified_time").is_valid(0));
assert_eq!(
list_items(utf8_list(&batch, "parents"), 0),
vec![Some("1SyntheticFolderAAAAAAAAAAAAAAAAA".into())]
);
assert_eq!(
list_items(utf8_list(&batch, "owner_display_names"), 0),
vec![Some("Example User".into())]
);
assert_eq!(
list_items(utf8_list(&batch, "owner_email_addresses"), 0),
vec![Some("user@example.com".into())]
);
assert!(boolean(&batch, "shared").value(0));
assert!(!boolean(&batch, "starred").value(0));
assert!(!boolean(&batch, "trashed").value(0));
assert_eq!(utf8(&batch, "name").value(1), "shared-drive-doc.txt");
assert_eq!(utf8(&batch, "drive_id").value(1), "0ASyntheticDrivePVA");
assert_eq!(
list_items(utf8_list(&batch, "parents"), 1),
vec![Some("0ASyntheticDrivePVA".into())]
);
assert!(utf8_list(&batch, "owner_display_names").is_null(1));
assert!(utf8_list(&batch, "owner_email_addresses").is_null(1));
assert!(boolean(&batch, "shared").is_null(1));
assert!(!boolean(&batch, "starred").value(1));
assert!(!boolean(&batch, "trashed").value(1));
assert_eq!(utf8(&batch, "name").value(2), "obsolete.txt");
assert!(boolean(&batch, "trashed").value(2));
assert_eq!(utf8(&batch, "name").value(3), "Design memo");
assert_eq!(
utf8(&batch, "mime_type").value(3),
"application/vnd.google-apps.document"
);
assert_eq!(int64(&batch, "size_bytes").value(3), 1024);
assert_eq!(utf8(&batch, "id").value(3).len(), 44);
assert!(
utf8(&batch, "web_view_link")
.value(3)
.starts_with("https://docs.google.com/document/d/")
);
assert_eq!(utf8(&batch, "name").value(4), "预算表.csv");
assert!(boolean(&batch, "starred").value(4));
assert_eq!(utf8(&batch, "name").value(5), "corpus-folder");
assert!(utf8(&batch, "mime_type").value(5).ends_with("apps.folder"));
assert!(int64(&batch, "size_bytes").is_null(5));
for absent in ["owners", "kind", "permission_id", "photo_link"] {
assert!(batch.column_by_name(absent).is_none(), "{absent}");
}
}
#[test]
fn drives_fixture_converts_including_the_restriction_flags() {
let batch = convert_fixture(
table("drives"),
include_str!("fixtures/google_drive/drives.json"),
);
assert_eq!(batch.num_rows(), 1);
assert_eq!(utf8(&batch, "id").value(0), "0ASyntheticDrivePVA");
assert_eq!(utf8(&batch, "name").value(0), "example-shared-drive");
assert!(!boolean(&batch, "hidden").value(0));
assert_eq!(utf8(&batch, "color_rgb").value(0), "#e91e63");
assert!(timestamp(&batch, "created_time").is_valid(0));
assert!(utf8(&batch, "org_unit_id").is_null(0));
assert!(utf8(&batch, "theme_id").is_null(0));
assert!(
utf8(&batch, "background_image_link")
.value(0)
.starts_with("https://ssl.gstatic.com/")
);
for flag in [
"admin_managed_restrictions",
"copy_requires_writer_permission",
"domain_users_only",
"drive_members_only",
"sharing_folders_requires_organizer_permission",
] {
assert!(boolean(&batch, flag).is_valid(0), "{flag}");
assert!(!boolean(&batch, flag).value(0), "{flag}");
}
for absent in ["kind", "capabilities", "restrictions"] {
assert!(batch.column_by_name(absent).is_none(), "{absent}");
}
let page = json!({
"drives": [
{
"id": "0ASyntheticMinimlPVA",
"kind": "drive#drive",
"name": "minimal",
"colorRgb": null,
"createdTime": null,
"orgUnitId": null,
"themeId": null,
"backgroundImageLink": null
},
{
"id": "0ASyntheticPartilPVA",
"kind": "drive#drive",
"name": "partial",
"hidden": true,
"colorRgb": null,
"createdTime": null,
"orgUnitId": null,
"themeId": null,
"backgroundImageLink": null,
"restrictions": { "copyRequiresWriterPermission": false }
}
],
"nextPageToken": null
});
let batch = convert_first_page(table("drives"), &page);
assert!(boolean(&batch, "hidden").is_null(0));
for flag in [
"admin_managed_restrictions",
"copy_requires_writer_permission",
"domain_users_only",
"drive_members_only",
"sharing_folders_requires_organizer_permission",
] {
assert!(boolean(&batch, flag).is_null(0), "{flag}");
}
assert!(boolean(&batch, "hidden").value(1));
assert!(
boolean(&batch, "copy_requires_writer_permission").is_valid(1),
"present false is false, not NULL"
);
assert!(!boolean(&batch, "copy_requires_writer_permission").value(1));
for flag in [
"admin_managed_restrictions",
"domain_users_only",
"drive_members_only",
"sharing_folders_requires_organizer_permission",
] {
assert!(boolean(&batch, flag).is_null(1), "{flag}");
}
}
#[test]
fn file_permissions_fixture_converts_every_grant_shape() {
let batch = convert_fixture(
table("file_permissions"),
include_str!("fixtures/google_drive/file_permissions.json"),
);
assert_eq!(batch.num_rows(), 3);
assert_eq!(utf8(&batch, "id").value(0), "anyoneWithLink");
assert_eq!(utf8(&batch, "role").value(0), "reader");
assert_eq!(utf8(&batch, "type").value(0), "anyone");
assert!(utf8(&batch, "email_address").is_null(0));
assert!(utf8(&batch, "display_name").is_null(0));
assert!(utf8(&batch, "domain").is_null(0));
assert!(utf8(&batch, "photo_link").is_null(0));
assert!(boolean(&batch, "allow_file_discovery").is_valid(0));
assert!(!boolean(&batch, "allow_file_discovery").value(0));
assert!(boolean(&batch, "deleted").is_null(0));
assert!(boolean(&batch, "pending_owner").is_null(0));
assert_eq!(utf8(&batch, "type").value(1), "domain");
assert_eq!(utf8(&batch, "domain").value(1), "example.com");
assert_eq!(utf8(&batch, "display_name").value(1), "Example Org");
assert!(utf8(&batch, "email_address").is_null(1));
assert!(boolean(&batch, "allow_file_discovery").value(1));
assert_eq!(utf8(&batch, "role").value(2), "owner");
assert_eq!(utf8(&batch, "type").value(2), "user");
assert_eq!(utf8(&batch, "email_address").value(2), "user@example.com");
assert_eq!(utf8(&batch, "display_name").value(2), "Example User");
assert!(utf8(&batch, "domain").is_null(2));
assert!(utf8(&batch, "photo_link").is_valid(2));
assert!(!boolean(&batch, "deleted").value(2));
assert!(!boolean(&batch, "pending_owner").value(2));
assert!(boolean(&batch, "allow_file_discovery").is_null(2));
for row in 0..3 {
assert!(timestamp(&batch, "expiration_time").is_null(row));
}
let page = json!({
"permissions": [{
"id": "00000000000000000003",
"kind": "drive#permission",
"role": "reader",
"type": "user",
"domain": null,
"deleted": false,
"photoLink": null,
"displayName": null,
"emailAddress": "user@example.com",
"pendingOwner": false,
"expirationTime": "2026-12-31T23:59:59.000Z"
}],
"nextPageToken": null
});
let synthetic = convert_first_page(table("file_permissions"), &page);
assert!(timestamp(&synthetic, "expiration_time").is_valid(0));
for absent in ["kind", "permission_details"] {
assert!(batch.column_by_name(absent).is_none(), "{absent}");
}
}
#[test]
fn wire_nulls_and_empty_strings_convert_without_failing_the_page() {
let page = json!({
"files": [
{
"id": "1SyntheticAllNull",
"name": "n",
"mimeType": "text/plain",
"webViewLink": null,
"createdTime": null,
"modifiedTime": null,
"sizeBytes": null,
"driveId": null
},
{
"id": "1SyntheticAllEmpty",
"name": "",
"mimeType": "",
"webViewLink": "",
"createdTime": null,
"modifiedTime": null,
"sizeBytes": 0,
"driveId": "",
"parents": [],
"owners": []
},
{
"id": "1SyntheticNullPluck",
"name": "n",
"mimeType": "text/plain",
"webViewLink": null,
"createdTime": null,
"modifiedTime": null,
"sizeBytes": null,
"driveId": null,
"owners": [
{ "displayName": null, "emailAddress": "user@example.com" }
]
}
],
"nextPageToken": null
});
let batch = convert_first_page(table("files"), &page);
assert_eq!(batch.num_rows(), 3);
for column in ["web_view_link", "drive_id"] {
assert!(utf8(&batch, column).is_null(0), "{column} must be NULL");
}
assert!(int64(&batch, "size_bytes").is_null(0));
assert!(timestamp(&batch, "created_time").is_null(0));
assert!(timestamp(&batch, "modified_time").is_null(0));
for column in ["parents", "owner_display_names", "owner_email_addresses"] {
assert!(utf8_list(&batch, column).is_null(0), "{column}");
}
for column in ["name", "mime_type", "web_view_link", "drive_id"] {
let values = utf8(&batch, column);
assert!(values.is_valid(1), "{column}: empty is not null");
assert_eq!(values.value(1), "", "{column}");
}
assert_eq!(int64(&batch, "size_bytes").value(1), 0);
for column in ["parents", "owner_display_names", "owner_email_addresses"] {
assert!(utf8_list(&batch, column).is_valid(1), "{column}");
assert_eq!(list_items(utf8_list(&batch, column), 1), vec![], "{column}");
}
assert_eq!(
list_items(utf8_list(&batch, "owner_display_names"), 2),
vec![None]
);
assert_eq!(
list_items(utf8_list(&batch, "owner_email_addresses"), 2),
vec![Some("user@example.com".into())]
);
}
#[test]
fn fixtures_are_redacted_captures_under_a_default_deny_audit() {
fn synthetic_id(s: &str) -> bool {
s.starts_with("1Synthetic")
|| s.starts_with("0ASynthetic")
|| s == "anyoneWithLink"
|| (s.len() == 20 && s.starts_with("0000000000000000000"))
}
fn rfc3339(s: &str) -> bool {
s.len() == 24
&& s.ends_with('Z')
&& s.bytes()
.all(|b| b.is_ascii_digit() || b"-:T.Z".contains(&b))
}
fn synthetic_url(s: &str) -> bool {
for prefix in [
"https://drive.google.com/file/d/",
"https://drive.google.com/drive/folders/",
"https://docs.google.com/document/d/",
] {
if let Some(rest) = s.strip_prefix(prefix) {
let (id, tail) = rest.split_once('/').unwrap_or((rest, ""));
return synthetic_id(id)
&& ["", "view", "view?usp=drivesdk", "edit?usp=drivesdk"].contains(&tail);
}
}
false
}
const REDACTED_NAMES: &[&str] = &[
"notes.txt",
"shared-drive-doc.txt",
"obsolete.txt",
"Design memo",
"预算表.csv",
"corpus-folder",
"example-shared-drive",
"wrong-types.bin",
];
fn audit(fixture: &str, key: &str, value: &Value) {
match value {
Value::String(s) => {
let allowed = match key {
"id" | "driveId" | "parents" | "inheritedFrom" | "permissionId" => {
synthetic_id(s)
}
"name" => REDACTED_NAMES.contains(&s.as_str()),
"displayName" => s == "Example User" || s == "Example Org",
"emailAddress" => s == "user@example.com",
"domain" => s == "example.com",
"photoLink" => s == "https://lh3.googleusercontent.com/a/SYNTHETIC=s64",
"backgroundImageLink" => {
s == "https://ssl.gstatic.com/team_drive_themes/clams_background.jpg"
}
"webViewLink" => synthetic_url(s),
"mimeType" => [
"text/plain",
"text/csv",
"application/vnd.google-apps.document",
"application/vnd.google-apps.folder",
"application/octet-stream",
]
.contains(&s.as_str()),
"createdTime" | "modifiedTime" | "expirationTime" => rfc3339(s),
"kind" => s == "drive#drive" || s == "drive#permission",
"role" => ["owner", "reader", "writer"].contains(&s.as_str()),
"type" | "permissionType" => {
["anyone", "domain", "user", "file"].contains(&s.as_str())
}
"colorRgb" => {
s.len() == 7
&& s.starts_with('#')
&& s[1..].bytes().all(|b| b.is_ascii_hexdigit())
}
"sizeBytes" => !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()),
_ => false,
};
assert!(
allowed,
"{fixture}: unredacted string at key `{key}`: {s:?}"
);
}
Value::Array(items) => {
for item in items {
audit(fixture, key, item);
}
}
Value::Object(map) => {
for (k, v) in map {
audit(fixture, k, v);
}
}
_ => {}
}
}
for (fixture, content) in [
("files", include_str!("fixtures/google_drive/files.json")),
("drives", include_str!("fixtures/google_drive/drives.json")),
(
"file_permissions",
include_str!("fixtures/google_drive/file_permissions.json"),
),
(
"files_type_mismatch",
include_str!("fixtures/google_drive/files_type_mismatch.json"),
),
] {
let page: Value = serde_json::from_str(content).expect("fixture parses");
audit(fixture, "$", &page);
}
}
#[test]
fn empty_page_keeps_schema_stable() {
for (short, row_key, columns) in [
("files", "files", 14),
("drives", "drives", 13),
("file_permissions", "permissions", 11),
] {
let t = table(short);
let batch = convert_first_page(t, &json!({row_key: [], "nextPageToken": null}));
assert_eq!(batch.num_rows(), 0, "{short}");
assert_eq!(
batch.num_columns(),
columns,
"{short} keeps its declared schema"
);
assert!(batch.column_by_name("id").is_some(), "{short}");
}
}
#[test]
fn type_mismatch_fixture_fails_with_the_targeted_error() {
let t = table("files");
let page: Value = serde_json::from_str(include_str!(
"fixtures/google_drive/files_type_mismatch.json"
))
.expect("fixture parses");
let rows = RowPath::parse(t.row_path)
.expect("row path")
.rows(&page, 1)
.expect("row array");
let err = RowConverter::new(t.fields)
.expect("converter")
.convert(rows, 1)
.expect_err("a string in an int64 column must fail the page");
match err {
OpenConnectorError::ConversionFailed {
ref path,
ref column,
page,
row,
ref expected,
ref found,
} => {
assert_eq!(column, "size_bytes");
assert_eq!(path, "$.sizeBytes");
assert_eq!(page, 1);
assert_eq!(row, 0, "the fixture's only row is the failing one");
assert_eq!(expected, "integer");
assert_eq!(found, "a string");
assert!(
!err.to_string().contains("48213"),
"row values never appear in errors"
);
}
other => panic!("expected ConversionFailed, got {other}"),
}
}
#[test]
fn a_row_without_an_id_fails_the_page_rather_than_yielding_a_null_key() {
let t = table("files");
let mut page: Value =
serde_json::from_str(include_str!("fixtures/google_drive/files.json"))
.expect("fixture parses");
page["files"][1]
.as_object_mut()
.expect("row is an object")
.remove("id");
let rows = RowPath::parse(t.row_path)
.expect("row path")
.rows(&page, 3)
.expect("row array");
let err = RowConverter::new(t.fields)
.expect("converter")
.convert(rows, 3)
.expect_err("a missing id must fail the page");
match err {
OpenConnectorError::ConversionFailed {
ref column,
page,
row,
ref found,
..
} => {
assert_eq!(column, "id");
assert_eq!(page, 3, "the page number travels with the error");
assert_eq!(row, 1, "the row index is named");
assert_eq!(found, "missing key");
}
other => panic!("expected ConversionFailed, got {other}"),
}
}
#[test]
fn pinned_fingerprints_match_the_reconciled_contracts() {
for (short, contract) in [
(
"files",
include_str!("fixtures/google_drive/contracts/files.list.json"),
),
(
"drives",
include_str!("fixtures/google_drive/contracts/drives.list.json"),
),
(
"file_permissions",
include_str!("fixtures/google_drive/contracts/permissions.list.json"),
),
] {
let t = table(short);
let schema: Value = serde_json::from_str(contract).expect("contract fixture parses");
let actual = fingerprint_schema(Some(&schema));
assert_eq!(
t.expected_fingerprint,
Some(actual.as_str()),
"{}: pinned {:?}, contract fixture hashes to {actual}",
t.id,
t.expected_fingerprint
);
}
}
#[test]
fn only_the_five_restriction_flags_escape_the_fingerprint_gate() {
for (short, contract) in [
(
"files",
include_str!("fixtures/google_drive/contracts/files.list.json"),
),
(
"file_permissions",
include_str!("fixtures/google_drive/contracts/permissions.list.json"),
),
] {
let t = table(short);
assert_eq!(
fingerprint_uncovered_columns(contract, t.row_path, t.fields),
&[] as &[&str],
"{short}: every column must stay inside the fingerprint gate"
);
}
let drives = table("drives");
assert_eq!(
fingerprint_uncovered_columns(
include_str!("fixtures/google_drive/contracts/drives.list.json"),
drives.row_path,
drives.fields
),
[
"admin_managed_restrictions",
"copy_requires_writer_permission",
"domain_users_only",
"drive_members_only",
"sharing_folders_requires_organizer_permission",
],
"drives: exactly the five restriction flags trade coverage for audit value"
);
}
#[test]
fn every_table_declares_terminating_cursor_pagination() {
for (short, row_path, page_size) in [
("files", "$.files", 1000),
("drives", "$.drives", 100),
("file_permissions", "$.permissions", 100),
] {
let t = table(short);
match t.pagination {
PaginationStrategy::Cursor {
cursor_param,
next_cursor_path,
page_size_param,
page_size: declared,
has_more_path,
absent_cursor,
} => {
assert_eq!(cursor_param, "pageToken", "{short}");
assert_eq!(next_cursor_path, "$.nextPageToken", "{short}");
assert_eq!(page_size_param, Some("pageSize"), "{short}");
assert_eq!(declared, page_size, "{short}");
assert!(has_more_path.is_none(), "{short}");
assert_eq!(absent_cursor, AbsentCursor::EndsTheScan, "{short}");
}
other => panic!("{short} must paginate by cursor, got {other:?}"),
}
assert_eq!(t.row_path, row_path, "{short}");
assert!(t.error_path.is_none(), "{short}");
}
}
#[test]
fn resources_and_fixed_inputs_are_exactly_as_designed() {
let files = table("files");
assert_eq!(
files.fixed_inputs,
&[
("includeItemsFromAllDrives", FixedValue::Bool(true)),
("supportsAllDrives", FixedValue::Bool(true)),
]
);
assert!(files.required_resources.is_empty());
assert_eq!(files.optional_resources, &["driveId", "q"]);
assert!(files.exclusive_resources.is_empty());
let drives = table("drives");
assert!(drives.fixed_inputs.is_empty());
assert!(drives.required_resources.is_empty());
assert_eq!(drives.optional_resources, &["q"]);
let permissions = table("file_permissions");
assert!(permissions.fixed_inputs.is_empty());
assert_eq!(permissions.required_resources, &["fileId"]);
assert!(permissions.optional_resources.is_empty());
for short in ["files", "drives", "file_permissions"] {
assert!(table(short).filters.is_empty(), "{short}");
}
}
#[test]
fn generated_inputs_are_accepted_by_the_captured_input_contracts() {
for (short, contract, declared_never_sent) in [
(
"files",
include_str!("fixtures/google_drive/contracts/inputs/files.list.json"),
vec![
"corpora",
"corpus",
"spaces",
"teamDriveId",
"orderBy",
"includeLabels",
"includePermissionsForView",
],
),
(
"drives",
include_str!("fixtures/google_drive/contracts/inputs/drives.list.json"),
vec!["useDomainAdminAccess"],
),
(
"file_permissions",
include_str!("fixtures/google_drive/contracts/inputs/permissions.list.json"),
vec![
"fields",
"includePermissionsForView",
"useDomainAdminAccess",
"supportsAllDrives",
],
),
] {
let schema: Value =
serde_json::from_str(contract).expect("input contract fixture parses");
let properties = &schema["properties"];
let t = table(short);
assert_eq!(
schema["additionalProperties"],
json!(false),
"{short}: the action's input schema is strict"
);
let mut generated: Vec<&str> = t
.required_resources
.iter()
.chain(t.optional_resources)
.copied()
.collect();
generated.extend(t.fixed_inputs.iter().map(|(key, _)| *key));
if let PaginationStrategy::Cursor {
cursor_param,
page_size_param,
..
} = t.pagination
{
generated.push(cursor_param);
generated.extend(page_size_param);
}
for key in &generated {
assert!(
!properties[*key].is_null(),
"{short}: `{key}` is not declared by the action's input schema"
);
}
if let Some(required) = schema["required"].as_array() {
for entry in required {
let entry = entry.as_str().expect("required entries are strings");
assert!(
generated.contains(&entry),
"{short}: the action requires `{entry}`, which this table never sends"
);
}
}
if let PaginationStrategy::Cursor {
page_size_param: Some(param),
page_size,
..
} = t.pagination
{
let declared = &properties[param];
assert_eq!(
u64::from(page_size),
declared["maximum"].as_u64().expect("declared maximum"),
"{short}: the pinned size is the declared ceiling"
);
assert!(
u64::from(page_size) >= declared["minimum"].as_u64().expect("declared minimum")
);
}
assert_eq!(
properties["pageToken"]["minLength"],
json!(1),
"{short}: the cursor is a non-empty opaque token"
);
let undeclared: &[&str] = if short == "files" {
&[
"fields",
"filter",
"orderby",
"maxResults",
"cursor",
"page",
"perPage",
"limit",
]
} else {
&[
"filter",
"orderby",
"maxResults",
"cursor",
"page",
"perPage",
"limit",
]
};
for absent in undeclared {
assert!(
properties[*absent].is_null(),
"{short}: `{absent}` is not part of this action's input surface"
);
}
for key in declared_never_sent {
assert!(
!properties[key].is_null(),
"{short}: `{key}` should be declared upstream"
);
assert!(
!generated.contains(&key),
"{short}: `{key}` must never be generated"
);
}
}
let permissions: Value = serde_json::from_str(include_str!(
"fixtures/google_drive/contracts/inputs/permissions.list.json"
))
.expect("input contract fixture parses");
assert_eq!(permissions["properties"]["fileId"]["minLength"], json!(1));
assert!(
permissions["required"].is_null(),
"upstream now declares a `required` array ({}) — if it contains \
`fileId`, the executor-layer gap this pack works around is fixed; \
update this assertion and the `fileId` rationale in the yaml",
permissions["required"]
);
assert_eq!(table("file_permissions").required_resources, &["fileId"]);
}
fn google_drive_config(token_env: &str, tables: &str, resource: &str) -> OpenConnectorConfig {
let resource_line = if resource.is_empty() {
String::new()
} else {
format!("resource: {resource}")
};
serde_yaml::from_str(&format!(
r#"
runtime_token_env: {token_env}
bindings:
- name: gdrive
source_pack: google_drive
{resource_line}
tables: [{tables}]
"#
))
.expect("config parses")
}
async fn setup_with_gateway(
gateway: MockGateway,
token_env: &'static str,
tables: &str,
resource: &str,
) -> (MockGateway, SessionContext) {
let _token = EnvVarGuard::set(token_env, "test-token");
let gateways = OpenConnectorGateways::default();
let mut ctx = SessionContext::new();
register_open_connector_tables(
&mut ctx,
"saas",
&gateway.url,
Some(&google_drive_config(token_env, tables, resource)),
false,
HierarchyLevel::Catalog,
Some(&gateways),
)
.await
.expect("gateway registration succeeds");
register_open_connector_udtfs(&ctx, gateways).expect("UDTF registration succeeds");
(gateway, ctx)
}
fn file_row(id: &str) -> Value {
json!({
"id": id,
"name": format!("{id}.txt"),
"mimeType": "text/plain",
"sizeBytes": 12
})
}
#[tokio::test]
async fn files_cursor_scan_pages_with_the_pinned_all_drives_inputs() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.files.list" {
let body: Value = serde_json::from_str(&req.body).unwrap_or_default();
let page = match body["input"].get("pageToken").and_then(Value::as_str) {
None => json!({"files": [file_row("f-1"), file_row("f-2")],
"nextPageToken": FILES_PAGE2_TOKEN}),
Some(token) if token == FILES_PAGE2_TOKEN => {
json!({"files": [file_row("f-3")], "nextPageToken": null})
}
Some(other) => return MockResponse::new(400, format!("bad cursor {other}")),
};
return MockResponse::ok(&envelope_ok(&page.to_string()));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_FILES", "files", "").await;
let batches = collect(&ctx, "SELECT id FROM saas.gdrive.files ORDER BY id").await;
assert_eq!(column_values(&batches, "id"), vec!["f-1", "f-2", "f-3"]);
let inputs = execute_inputs(&gateway, "googledrive.files.list");
assert_eq!(inputs.len(), 2, "two cursor pages");
assert_eq!(
inputs[1]["pageToken"], FILES_PAGE2_TOKEN,
"the opaque cursor verbatim"
);
for (page, (input, expected_keys)) in inputs
.iter()
.zip([
vec!["includeItemsFromAllDrives", "pageSize", "supportsAllDrives"],
vec![
"includeItemsFromAllDrives",
"pageSize",
"pageToken",
"supportsAllDrives",
],
])
.enumerate()
{
assert_eq!(input["pageSize"], 1000, "declared ceiling: {input}");
assert_eq!(input["supportsAllDrives"], true, "{input}");
assert_eq!(input["includeItemsFromAllDrives"], true, "{input}");
assert_eq!(input_keys(input), expected_keys, "page {} keys", page + 1);
}
}
#[tokio::test]
async fn optional_resources_forward_verbatim_and_compose() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.files.list" {
return MockResponse::ok(&envelope_ok(
&json!({"files": [file_row("f-1")], "nextPageToken": null}).to_string(),
));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) = setup_with_gateway(
gateway,
"SKARDI_TEST_OC_GDRIVE_SCOPES",
"files",
"{driveId: 0ASyntheticSharedDriveUk9PVA, q: \"mimeType != 'application/vnd.google-apps.folder' and trashed = false\"}",
)
.await;
let batches = collect(&ctx, "SELECT id FROM saas.gdrive.files").await;
assert_eq!(column_values(&batches, "id"), vec!["f-1"]);
let inputs = execute_inputs(&gateway, "googledrive.files.list");
assert_eq!(inputs.len(), 1);
assert_eq!(inputs[0]["driveId"], "0ASyntheticSharedDriveUk9PVA");
assert_eq!(
inputs[0]["q"],
"mimeType != 'application/vnd.google-apps.folder' and trashed = false"
);
assert_eq!(
input_keys(&inputs[0]),
vec![
"driveId",
"includeItemsFromAllDrives",
"pageSize",
"q",
"supportsAllDrives"
],
"both scopes ride alongside the pins: {}",
inputs[0]
);
}
#[tokio::test]
async fn drives_scan_sends_only_its_page_size_and_terminates_on_null() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.drives.list" {
let body: Value = serde_json::from_str(&req.body).unwrap_or_default();
let page = match body["input"].get("pageToken").and_then(Value::as_str) {
None => json!({"drives": [{"id": "d-1", "name": "Engineering"}],
"nextPageToken": "drives-page-2"}),
Some("drives-page-2") => {
json!({"drives": [{"id": "d-2", "name": "Legal"}],
"nextPageToken": null})
}
Some(other) => return MockResponse::new(400, format!("bad cursor {other}")),
};
return MockResponse::ok(&envelope_ok(&page.to_string()));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_DRIVES", "drives", "").await;
let batches = collect(&ctx, "SELECT id, name FROM saas.gdrive.drives ORDER BY id").await;
assert_eq!(column_values(&batches, "id"), vec!["d-1", "d-2"]);
let inputs = execute_inputs(&gateway, "googledrive.drives.list");
assert_eq!(inputs.len(), 2, "two cursor pages");
assert_eq!(input_keys(&inputs[0]), vec!["pageSize"]);
assert_eq!(
inputs[0]["pageSize"], 100,
"declared ceiling: {}",
inputs[0]
);
assert_eq!(input_keys(&inputs[1]), vec!["pageSize", "pageToken"]);
}
#[tokio::test]
async fn drives_q_resource_forwards_verbatim() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.drives.list" {
return MockResponse::ok(&envelope_ok(
&json!({"drives": [{"id": "d-1", "name": "Engineering"}],
"nextPageToken": null})
.to_string(),
));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) = setup_with_gateway(
gateway,
"SKARDI_TEST_OC_GDRIVE_DRIVES_Q",
"drives",
"{q: \"name contains 'x'\"}",
)
.await;
let batches = collect(&ctx, "SELECT id FROM saas.gdrive.drives").await;
assert_eq!(column_values(&batches, "id"), vec!["d-1"]);
let inputs = execute_inputs(&gateway, "googledrive.drives.list");
assert_eq!(inputs.len(), 1);
assert_eq!(inputs[0]["q"], "name contains 'x'");
assert_eq!(inputs[0]["pageSize"], 100);
assert_eq!(input_keys(&inputs[0]), vec!["pageSize", "q"]);
}
#[tokio::test]
async fn file_permissions_scan_forwards_its_required_file_id_on_every_page() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.permissions.list" {
let body: Value = serde_json::from_str(&req.body).unwrap_or_default();
let page = match body["input"].get("pageToken").and_then(Value::as_str) {
None => json!({"permissions": [{"id": "p-1", "role": "owner"}],
"nextPageToken": "perms-page-2"}),
Some("perms-page-2") => {
json!({"permissions": [{"id": "p-2", "role": "reader"}],
"nextPageToken": null})
}
Some(other) => return MockResponse::new(400, format!("bad cursor {other}")),
};
return MockResponse::ok(&envelope_ok(&page.to_string()));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) = setup_with_gateway(
gateway,
"SKARDI_TEST_OC_GDRIVE_PERMS",
"file_permissions",
"{fileId: 1SyntheticMyDriveDocAAAAAAAAAAAAAAAAAAAAAAA}",
)
.await;
let batches = collect(
&ctx,
"SELECT id, role FROM saas.gdrive.file_permissions ORDER BY id",
)
.await;
assert_eq!(column_values(&batches, "id"), vec!["p-1", "p-2"]);
let inputs = execute_inputs(&gateway, "googledrive.permissions.list");
assert_eq!(inputs.len(), 2, "two cursor pages");
for (page, (input, expected_keys)) in inputs
.iter()
.zip([
vec!["fileId", "pageSize"],
vec!["fileId", "pageSize", "pageToken"],
])
.enumerate()
{
assert_eq!(
input["fileId"], "1SyntheticMyDriveDocAAAAAAAAAAAAAAAAAAAAAAA",
"the bound file on every page: {input}"
);
assert_eq!(input["pageSize"], 100, "declared ceiling: {input}");
assert_eq!(input_keys(input), expected_keys, "page {} keys", page + 1);
}
}
#[tokio::test]
async fn a_permissions_binding_without_a_file_id_fails_before_any_http() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
MockResponse::new(404, "{}")
})
.await;
let _token = EnvVarGuard::set("SKARDI_TEST_OC_GDRIVE_NO_FILE", "test-token");
let mut ctx = SessionContext::new();
let gateways = OpenConnectorGateways::default();
let err = register_open_connector_tables(
&mut ctx,
"saas",
&gateway.url,
Some(&google_drive_config(
"SKARDI_TEST_OC_GDRIVE_NO_FILE",
"file_permissions",
"",
)),
false,
HierarchyLevel::Catalog,
Some(&gateways),
)
.await
.expect_err("a permissions binding with no fileId must fail registration");
assert!(
err.to_string().contains("fileId"),
"the missing resource is named: {err}"
);
assert!(
gateway.requests().iter().all(|r| r.path == "/v1/health"),
"resource enforcement precedes discovery"
);
}
#[tokio::test]
async fn predicates_stay_local_against_a_provider_that_cannot_narrow() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.files.list" {
return MockResponse::ok(&envelope_ok(
&json!({"files": [file_row("keep"), file_row("drop")],
"nextPageToken": null})
.to_string(),
));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_FILTER", "files", "").await;
let batches = collect(&ctx, "SELECT id FROM saas.gdrive.files WHERE id = 'keep'").await;
assert_eq!(column_values(&batches, "id"), vec!["keep"]);
let inputs = execute_inputs(&gateway, "googledrive.files.list");
assert_eq!(inputs.len(), 1);
assert_eq!(
input_keys(&inputs[0]),
vec!["includeItemsFromAllDrives", "pageSize", "supportsAllDrives"],
"a WHERE clause must not invent a wire input: {}",
inputs[0]
);
}
#[tokio::test]
async fn limit_stops_cursor_pagination_early() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.files.list" {
return MockResponse::ok(&envelope_ok(
&json!({"files": [file_row("f-1"), file_row("f-2")],
"nextPageToken": FILES_PAGE2_TOKEN})
.to_string(),
));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_LIMIT", "files", "").await;
let batches = collect(&ctx, "SELECT id FROM saas.gdrive.files LIMIT 1").await;
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
let inputs = execute_inputs(&gateway, "googledrive.files.list");
assert_eq!(inputs.len(), 1, "LIMIT stops before following the cursor");
assert_eq!(
inputs[0]["pageSize"], 1000,
"a full page still crosses the wire: {}",
inputs[0]
);
}
#[tokio::test]
async fn scan_of_an_account_with_no_shared_drives_is_clean() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.drives.list" {
return MockResponse::ok(&envelope_ok(
&json!({"drives": [], "nextPageToken": null}).to_string(),
));
}
MockResponse::new(404, "{}")
})
.await;
let (gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_EMPTY", "drives", "").await;
let batches = collect(&ctx, "SELECT id, name FROM saas.gdrive.drives").await;
assert_eq!(
batches.iter().map(RecordBatch::num_rows).sum::<usize>(),
0,
"an empty collection is an empty result set, not an error"
);
assert_eq!(
execute_inputs(&gateway, "googledrive.drives.list").len(),
1,
"an empty page still costs exactly one request"
);
}
#[tokio::test]
async fn a_repeated_cursor_fails_as_a_pagination_loop() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.files.list" {
return MockResponse::ok(&envelope_ok(
&json!({"files": [file_row("f-1")], "nextPageToken": FILES_PAGE2_TOKEN})
.to_string(),
));
}
MockResponse::new(404, "{}")
})
.await;
let (_gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_LOOP", "files", "").await;
let err = ctx
.sql("SELECT id FROM saas.gdrive.files")
.await
.expect("plan")
.collect()
.await
.expect_err("a repeated cursor must fail the scan");
let message = err.to_string();
assert!(
message.contains("pagination loop") && message.contains(FILES_PAGE2_TOKEN),
"the loop names the cursor the gateway would not advance: {message}"
);
}
#[tokio::test]
async fn provider_errors_surface_through_the_gateway_failure_envelope() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.files.list" {
return MockResponse::new(
403,
envelope_err(
"authorization_failed",
"Connect googledrive with OAuth first.",
),
);
}
MockResponse::new(404, "{}")
})
.await;
let (_gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_AUTHZ", "files", "").await;
let err = ctx
.sql("SELECT id FROM saas.gdrive.files")
.await
.expect("plan")
.collect()
.await
.expect_err("a failure envelope must fail the scan");
let message = err.to_string();
assert!(
message.contains("authorization_failed") && message.contains("googledrive.files.list"),
"the gateway's error code and the dotted action id are named: {message}"
);
assert!(
!message.contains("row path"),
"never the misleading row-path error: {message}"
);
}
#[tokio::test]
async fn udtf_parity_for_files() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return google_drive_discovery(&req.path);
}
if req.method == "POST" && req.path == "/v1/actions/googledrive.files.list" {
return MockResponse::ok(&envelope_ok(
&json!({"files": [file_row("f-1")], "nextPageToken": null}).to_string(),
));
}
MockResponse::new(404, "{}")
})
.await;
let (_gateway, ctx) =
setup_with_gateway(gateway, "SKARDI_TEST_OC_GDRIVE_UDTF", "files", "").await;
let from_table = collect(&ctx, "SELECT id, name, mime_type FROM saas.gdrive.files").await;
let from_udtf = collect(
&ctx,
"SELECT id, name, mime_type \
FROM open_connector_query('saas', 'google_drive.files', '{}')",
)
.await;
assert_eq!(from_table[0].schema(), from_udtf[0].schema());
assert_eq!(
arrow::util::pretty::pretty_format_batches(&from_table)
.unwrap()
.to_string(),
arrow::util::pretty::pretty_format_batches(&from_udtf)
.unwrap()
.to_string()
);
}
#[tokio::test]
async fn drifted_contract_fails_registration_not_the_scan() {
let gateway = MockGateway::start(|req| {
if req.method == "GET" && req.path == "/v1/health" {
return MockResponse::ok("{}");
}
if req.method == "GET" && req.path.starts_with("/v1/actions/") {
return MockResponse::ok(&discovery_ok("{}", r#"{"type": "object"}"#, true, None));
}
MockResponse::new(404, "{}")
})
.await;
let _token = EnvVarGuard::set("SKARDI_TEST_OC_GDRIVE_DRIFT", "test-token");
let mut ctx = SessionContext::new();
let gateways = OpenConnectorGateways::default();
let err = register_open_connector_tables(
&mut ctx,
"saas",
&gateway.url,
Some(&google_drive_config(
"SKARDI_TEST_OC_GDRIVE_DRIFT",
"files",
"",
)),
false,
HierarchyLevel::Catalog,
Some(&gateways),
)
.await
.expect_err("a drifted contract must fail registration");
let message = err.to_string();
assert!(
message.contains("google_drive.files")
&& message.contains("googledrive.files.list")
&& message.contains("fingerprint mismatch"),
"table, action, and cause are named: {message}"
);
}
}