Struct Catalog

pub struct Catalog { /* private fields */ }

User-facing catalog API, accessible through SparkSession.catalog.

Implementations

impl Catalog

pub fn new(spark_session: SparkSession) -> Self

pub async fn current_catalog(self) -> Result<String, SparkError>

Returns the current default catalog in this session

pub async fn set_current_catalog( self, catalog_name: &str, ) -> Result<(), SparkError>

Sets the current default catalog in this session

pub async fn list_catalogs( self, pattern: Option<&str>, ) -> Result<RecordBatch, SparkError>

Returns a list of catalogs in this session

pub async fn current_database(self) -> Result<String, SparkError>

Returns the current default database in this session

pub async fn set_current_database(self, db_name: &str) -> Result<(), SparkError>

Sets the current default database in this session

pub async fn list_databases( self, pattern: Option<&str>, ) -> Result<RecordBatch, SparkError>

Returns a list of databases in this session

pub async fn get_database( self, db_name: &str, ) -> Result<RecordBatch, SparkError>

Get the database with the specified name

pub async fn database_exists(self, db_name: &str) -> Result<bool, SparkError>

Check if the database with the specified name exists.

pub async fn list_tables( self, pattern: Option<&str>, db_name: Option<&str>, ) -> Result<RecordBatch, SparkError>

Returns a list of tables/views in the specific database

pub async fn get_table( self, table_name: &str, ) -> Result<RecordBatch, SparkError>

Get the table or view with the specified name.

pub async fn list_functions( self, db_name: Option<&str>, pattern: Option<&str>, ) -> Result<RecordBatch, SparkError>

Returns a list of functions registered in the specified database.

pub async fn function_exists( self, function_name: &str, db_name: Option<&str>, ) -> Result<bool, SparkError>

Check if the function with the specified name exists.

pub async fn get_function( self, function_name: &str, ) -> Result<RecordBatch, SparkError>

Get the function with the specified name.

pub async fn list_columns( self, table_name: &str, db_name: Option<&str>, ) -> Result<RecordBatch, SparkError>

Returns a list of columns for the given tables/views in the specific database

pub async fn table_exists( self, table_name: &str, db_name: Option<&str>, ) -> Result<bool, SparkError>

Check if the table or view with the specified name exists.

pub async fn drop_temp_view(self, view_name: &str) -> Result<bool, SparkError>

Drops the local temporary view with the given view name in the catalog.

pub async fn drop_global_temp_view( self, view_name: &str, ) -> Result<bool, SparkError>

Drops the global temporary view with the given view name in the catalog.

pub async fn is_cached(self, table_name: &str) -> Result<bool, SparkError>

Returns true if the table is currently cached in-memory.

pub async fn create_table( &self, table_name: &str, path: Option<&str>, source: Option<&str>, description: Option<&str>, schema: Option<StructType>, options: Option<HashMap<String, String>>, ) -> Result<DataFrame, SparkError>

Creates a table based on the dataset in a data source.

pub async fn create_external_table( &self, table_name: &str, path: Option<&str>, source: Option<&str>, schema: Option<StructType>, options: Option<HashMap<String, String>>, ) -> Result<DataFrame, SparkError>

Creates a table based on the dataset in a data source.

pub async fn cache_table( self, table_name: &str, storage_level: Option<StorageLevel>, ) -> Result<(), SparkError>

Caches the specified table in-memory or with given storage level.

pub async fn uncache_table(self, table_name: &str) -> Result<(), SparkError>

Removes the specified table from the in-memory cache.

pub async fn clear_cache(self) -> Result<(), SparkError>

Removes all cached tables from the in-memory cache.

pub async fn refresh_table(self, table_name: &str) -> Result<(), SparkError>

Invalidates and refreshes all the cached data (and the associated metadata) for any DataFrame that contains the given data source path.

pub async fn recover_partitions( self, table_name: &str, ) -> Result<(), SparkError>

Recovers all the partitions of the given table and updates the catalog.

pub async fn refresh_by_path(self, path: &str) -> Result<(), SparkError>

Invalidates and refreshes all the cached data (and the associated metadata) for any DataFrame that contains the given data source path.

Trait Implementations

impl Clone for Catalog

fn clone(&self) -> Catalog

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Catalog

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.