pyspark上传数据到hive表 - xd21303/Notebook GitHub Wiki

切换到spark安装目录

指定spark的配置文件

import os
from pyspark import SparkConf, SparkContext
from pyspark.sql import Row, SparkSession
from pyspark.sql import functions as F
from pyspark.sql.functions import col, max, mean, min, pow, udf

os.chdir('/spark/python')
os.environ['USER'] = "root"
os.environ['SPARK_CONF_DIR'] = "/spark/conf"

spark = SparkSession.builder.master("yarn").appName("name").enableHiveSupport().getOrCreate()
spark.createDataFrame(data).createOrReplaceTempView(
    f'tmp_name')
write_sql = f"""
        create table if not exists tmp.table_name as
        SELECT * FROM f'tmp_name
"""
spark.sql(write_sql)