spark_connect_pipeline PySpark Module¶

create_dataflow_graph(
    spark: SparkSession,
    default_catalog: Optional[str],
    default_database: Optional[str],
    sql_conf: Optional[Mapping[str, str]],
) -> str

start_run(
    spark: SparkSession,
    dataflow_graph_id: str,
    full_refresh: Optional[Sequence[str]],
    full_refresh_all: bool,
    refresh: Optional[Sequence[str]],
    dry: bool,
    storage: str,
) -> Iterator[Dict[str, Any]]

handle_pipeline_events(
    iter: Iterator[Dict[str, Any]]
) -> None