test_df

eda/smart_rank/gyz_test.py

@@ -12,7 +12,6 @@ import datetime
 from pyspark import SparkConf
 import time
 from pyspark.sql import SparkSession
-from pyspark.sql import SQLContext
 import json
 import numpy as np
 import pandas as pd
@@ -79,7 +78,6 @@ sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true")
     .set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \
     .set("spark.tispark.pd.addresses", "172.16.40.170:2379").set("spark.io.compression.codec", "lzf") \
     .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy")
-sqlContext = SparkSession(sparkConf)
 
 spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
 spark.sparkContext.setLogLevel("WARN")