Skip to content

F
ffm-baseline
Commit e146843c authored by 高雅喆's avatar 高雅喆
Browse files
Options

test_df

parent d54bee48
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 0 deletions
eda/smart_rank/gyz_test.py
View file @ e146843c
...@@ -10,6 +10,7 @@ from email.mime.application import MIMEApplication ...@@ -10,6 +10,7 @@ from email.mime.application import MIMEApplication
import redis import redis
import datetime import datetime
from pyspark import SparkConf from pyspark import SparkConf
from pyspark import SparkContext
import time import time
from pyspark.sql import SparkSession from pyspark.sql import SparkSession
import json import json
...@@ -78,6 +79,8 @@ sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true") ...@@ -78,6 +79,8 @@ sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true")
.set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \ .set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \
.set("spark.tispark.pd.addresses", "172.16.40.170:2379").set("spark.io.compression.codec", "lzf") \ .set("spark.tispark.pd.addresses", "172.16.40.170:2379").set("spark.io.compression.codec", "lzf") \
.set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy") .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy")
sc = SparkContex()
sqlContext = SparkSession(sc)
spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate() spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
spark.sparkContext.setLogLevel("WARN") spark.sparkContext.setLogLevel("WARN")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment