Skip to content

F
ffm-baseline
Commit fab30cb9 authored by 高雅喆's avatar 高雅喆
Browse files
Options

test_df

parent f59e4b8a
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 0 deletions
eda/smart_rank/gyz_test.py
View file @ fab30cb9
...@@ -12,6 +12,7 @@ import datetime ...@@ -12,6 +12,7 @@ import datetime
from pyspark import SparkConf from pyspark import SparkConf
import time import time
from pyspark.sql import SparkSession from pyspark.sql import SparkSession
from pyspark.sql import SQLContext
import json import json
import numpy as np import numpy as np
import pandas as pd import pandas as pd
...@@ -78,6 +79,7 @@ sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true") ...@@ -78,6 +79,7 @@ sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true")
.set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \ .set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \
.set("spark.tispark.pd.addresses", "172.16.40.170:2379").set("spark.io.compression.codec", "lzf") \ .set("spark.tispark.pd.addresses", "172.16.40.170:2379").set("spark.io.compression.codec", "lzf") \
.set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy") .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy")
sqlContext = SQLContext(sparkConf)
spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate() spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
spark.sparkContext.setLogLevel("WARN") spark.sparkContext.setLogLevel("WARN")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment