Commit 35422345 authored by 张彦钊's avatar 张彦钊

add

parent 6327ad41
...@@ -28,15 +28,15 @@ if __name__ == '__main__': ...@@ -28,15 +28,15 @@ if __name__ == '__main__':
.set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy") .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy")
spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate() spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
sql = "select params['exposure_cards'] from online.ml_community_precise_exposure_detail " \ sql = "select params['exposure_cards'],time_str from online.ml_community_precise_exposure_detail " \
"where action = 'page_precise_exposure' and page_name = 'search_result_welfare' " \ "where action = 'page_precise_exposure' and page_name = 'search_result_welfare' " \
"AND partition_date='20190926' limit 20" "AND partition_date='20190926' limit 20"
df = spark.sql(sql) df = spark.sql(sql)
# df.show(6) # df.show(6)
rdd = df.rdd.map(lambda x:(x[0])).map(lambda x:(eval(x[0]))).map(lambda x:(ctr(x[0]))) rdd = df.rdd.map(lambda x:(x[0],x[1])).map(lambda x:(eval(x[0]),x[1])).map(lambda x:(ctr(x[0]),x[1]))
print(rdd.collect()) print(rdd.collect())
spark.createDataFrame(rdd).toDF("n").show(6) spark.createDataFrame(rdd).show(6)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment