修改测试文件

tensnsorflow/test.py

@@ -8,7 +8,6 @@ from pyspark.sql import SparkSession
 # from py4j.java_gateway import java_import
 # import pytispark.pytispark as pti
 import pandas as pd
-import pymysql
 
 def con_sql(db,sql):
     cursor = db.cursor()
@@ -27,7 +26,13 @@ def con_sql(db,sql):
 def test():
         conf = SparkConf().setAppName("My App").set("spark.io.compression.codec", "lzf")
         sc = SparkContext(conf = conf)
+
         hive_context = HiveContext(sc)
+
+        hive_context.sql("SET mapreduce.job.queuename=data")
+        hive_context.sql("SET mapred.input.dir.recursive=true")
+        hive_context.sql("SET hive.mapred.supports.subdirectories=true")
+
         hive_context.sql(''' select device["device_type"] from online.tl_hdfs_maidian_view
         where partition_date = '20181012' and action = "page_view"
         and params["page_name"] = "diary_detail" and params["referrer"] = "home" limit 10 ''').show(6)