Skip to content

F
ffm-baseline
Commit 4e712687 authored by 王志伟's avatar 王志伟
Browse files
Options

跑波动原因数据

parent 4f8b6803
Show whitespace changes
Inline Side-by-side
Showing with 34 additions and 61 deletions
eda/feededa/src/main/scala/com/gmei/temp_analysis.scala
View file @ 4e712687
...@@ -50,17 +50,10 @@ object temp_analysis { ...@@ -50,17 +50,10 @@ object temp_analysis {
// val ti = new TiContext(sc) // val ti = new TiContext(sc)
sc.sql("use jerry_prod") sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._ import sc.implicits._
val stat_date = GmeiConfig.getMinusNDate(1) // val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date=param.date val stat_date=param.date
//println(param.date) //println(param.date)
val partition_date = stat_date.replace("-","") val partition_date = stat_date.replace("-","")
...@@ -81,25 +74,6 @@ object temp_analysis { ...@@ -81,25 +74,6 @@ object temp_analysis {
) )
agency_id.createOrReplaceTempView("agency_id") agency_id.createOrReplaceTempView("agency_id")
// //每日新用户
// val device_id_newUser = sc.sql(
// s"""
// |select distinct(device_id) as device_id
// |from online.ml_device_day_active_status
// |where active_type != '4'
// |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
// | ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
// | ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
// | ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
// | ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
// | ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
// | ,'promotion_shike','promotion_julang_jl03')
// |and partition_date ='${partition_date}'
// """.stripMargin
// )
// device_id_newUser.createOrReplaceTempView("device_id_new")
val blacklist_id = sc.sql( val blacklist_id = sc.sql(
s""" s"""
|SELECT device_id |SELECT device_id
...@@ -120,48 +94,47 @@ object temp_analysis { ...@@ -120,48 +94,47 @@ object temp_analysis {
final_id.createOrReplaceTempView("final_id") final_id.createOrReplaceTempView("final_id")
val diary_clk_all = sc.sql( // //每日新用户
val device_id_newUser = sc.sql(
s""" s"""
|select ov.partition_date,count(ov.cl_id) as clk_num,count(distinct(ov.cl_id)),count(ov.cl_id)/count(distinct(ov.cl_id)) |select distinct(oms.device_id) as device_id
|from online.tl_hdfs_maidian_view ov left join final_id |from online.ml_device_day_active_status oms left join final_id
|on ov.cl_id = final_id.device_id |on oms.device_id=final_id.device_id
|where ov.action = "page_view" |where oms.active_type != '4'
|and params['page_name']="diary_detail" |and oms.first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
|and ov.cl_id != "NULL" | ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
|and ov.partition_date >='20181201' | ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
| ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
| ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
| ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
| ,'promotion_shike','promotion_julang_jl03')
|and oms.partition_date ='${partition_date}'
|and final_id.device_id is null |and final_id.device_id is null
|group by ov.partition_date
|order by ov.partition_date
""".stripMargin """.stripMargin
) )
diary_clk_all.show(80) device_id_newUser.createOrReplaceTempView("device_id_new")
//日记本点击
val referrer=List("about_me_message_list","all_case_service_comment","all_cases","diary_detail","diary_list"
,"diary_listof_related_service","answer_detail","community_home","conversation_detail","create_diary_title","diary_listof_related_service", val diary_clk_new = sc.sql(
"doctor_all_cases","hospital_all_cases","my_favor","my_order","order_detail","personal_store_diary_list","received_votes",
"topic_detail","welfare_detail","welfare_list","welfare_special","wiki_detail","zone_detail",
"expert_detail","free_activity_detail","home","message_home","my_diary","organization_detail","other_homepage","question_detail",
"search_result_diary","search_result_more","welfare_detail","zone_v3")
for( a <- referrer ){
val diary_clk_temp = sc.sql(
s""" s"""
|select ov.partition_date,count(ov.cl_id) as clk_num,count(distinct(ov.cl_id)),count(ov.cl_id)/count(distinct(ov.cl_id)) |select '${stat_date}' as stat_date,a.device_id,count(a.diary_id)
|from online.tl_hdfs_maidian_view ov left join final_id |(select ov.partition_date,ov.cl_id as device_id,ov.params['diary_id'] as diary_id
|on ov.cl_id = final_id.device_id |from online.tl_hdfs_maidian_view ov inner join device_id_new
|where ov.action = "page_view" |on ov.cl_id = device_id_new.device_id
|and params['page_name']="diary_detail" |where ov.action = 'on_click_diary_card'
|and params['referrer']='${a}' |and ov.params['tab_name'] = '精选'
|and ov.cl_id != "NULL" |and ov.params['page_name'] = 'home'
|and ov.partition_date >='20181201' |and ov.partition_date='${partition_date}'
|and final_id.device_id is null |and agency_id.device_id is null) a
|group by ov.partition_date |group by a.device_id
|order by ov.partition_date
""".stripMargin """.stripMargin
) )
println("来源：",a) diary_clk_new.show(80)
diary_clk_temp.show(80) GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",diary_clk_new, table="temp",SaveMode.Append)
println("写入完成")
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment