Commit 733ed31e authored by litaolemo's avatar litaolemo

update

parent 8464b112
......@@ -33,6 +33,9 @@ sparkConf.set("spark.debug.maxToStringFields", "100")
sparkConf.set("spark.tispark.plan.allow_index_double_read", False)
sparkConf.set("spark.tispark.plan.allow_index_read", True)
sparkConf.set("spark.hive.mapred.supports.subdirectories", True)
sparkConf.set("spark.sql.adaptive.enabled", True)
sparkConf.set("spark.sql.adaptive.skewedJoin.enabled", True)
sparkConf.set("spark.shuffle.statistics.verbose", True)
sparkConf.set("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive", True)
sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
sparkConf.set("mapreduce.output.fileoutputformat.compress", False)
......@@ -71,9 +74,8 @@ for t in range(0, task_days):
today_str = now.strftime("%Y%m%d")
yesterday_str = (now + datetime.timedelta(days=-1)).strftime("%Y%m%d")
one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d")
sql_distinct_device_id = r"""
SELECT * FROM online.tl_hdfs_doctor_view
"""
sql_distinct_device_id = """
SELECT * FROM online.tl_hdfs_doctor_view"""
print(sql_distinct_device_id)
distinct_device_id_df = spark.sql(sql_distinct_device_id,)
distinct_device_id_df.show(1)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment