Skip to content

F
ffm-baseline
Commit c5317d95 authored by 张彦钊's avatar 张彦钊
Browse files
Options

把刚才修改的文件回滚

parent 680b4c83
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 99 deletions
eda/feededa/src/main/scala/com/gmei/Recommendation_strategy_all.scala
View file @ c5317d95
...@@ -52,7 +52,7 @@ object Recommendation_strategy_all { ...@@ -52,7 +52,7 @@ object Recommendation_strategy_all {
val stat_date = GmeiConfig.getMinusNDate(1) val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = param.date // val stat_date = param.date
//println(param.date) //println(param.date)
val partition_date = stat_date.replace("-","") val partition_date = stat_date.replace("-","")
val decive_id_oldUser = sc.sql( val decive_id_oldUser = sc.sql(
...@@ -83,7 +83,7 @@ object Recommendation_strategy_all { ...@@ -83,7 +83,7 @@ object Recommendation_strategy_all {
|and jd.device_id not in (select device_id from blacklist) |and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${stat_date}' |and jd.stat_date ='${stat_date}'
""".stripMargin """.stripMargin
) )
val imp_count_oldUser_Contrast = sc.sql( val imp_count_oldUser_Contrast = sc.sql(
s""" s"""
...@@ -95,7 +95,7 @@ object Recommendation_strategy_all { ...@@ -95,7 +95,7 @@ object Recommendation_strategy_all {
|and je.device_id not in (select device_id from blacklist) |and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${stat_date}' |and je.stat_date ='${stat_date}'
""".stripMargin """.stripMargin
) )
val clk_count_oldUser_all = sc.sql( val clk_count_oldUser_all = sc.sql(
s""" s"""
...@@ -119,7 +119,7 @@ object Recommendation_strategy_all { ...@@ -119,7 +119,7 @@ object Recommendation_strategy_all {
""".stripMargin """.stripMargin
) )
//获取策略命中用户device_id //获取策略命中用户device_id
val device_id_cover = sc.sql( val device_id_cover = sc.sql(
s""" s"""
|select distinct(device_id) as device_id |select distinct(device_id) as device_id
...@@ -183,13 +183,13 @@ object Recommendation_strategy_all { ...@@ -183,13 +183,13 @@ object Recommendation_strategy_all {
""".stripMargin """.stripMargin
) )
val result1 = clk_count_oldUser_Contrast.join(imp_count_oldUser_Contrast,"stat_date") val result1 = clk_count_oldUser_Contrast.join(imp_count_oldUser_Contrast,"stat_date")
.join(clk_count_oldUser_all,"stat_date") .join(clk_count_oldUser_all,"stat_date")
.join(imp_count_oldUser_all,"stat_date") .join(imp_count_oldUser_all,"stat_date")
.join(clk_count_oldUser_Cover,"stat_date") .join(clk_count_oldUser_Cover,"stat_date")
.join(imp_count_oldUser_Cover,"stat_date") .join(imp_count_oldUser_Cover,"stat_date")
.join(device_num_cover,"stat_date") .join(device_num_cover,"stat_date")
.join(device_num_1_hit,"stat_date") .join(device_num_1_hit,"stat_date")
.join(device_num_hit,"stat_date") .join(device_num_hit,"stat_date")
result1.show() result1.show()
GmeiConfig.writeToJDBCTable(result1, "Recommendation_strategy_temp", SaveMode.Append) GmeiConfig.writeToJDBCTable(result1, "Recommendation_strategy_temp", SaveMode.Append)
...@@ -287,7 +287,7 @@ object Recommendation_strategy_all { ...@@ -287,7 +287,7 @@ object Recommendation_strategy_all {
GmeiConfig.writeToJDBCTable(result2, "strategy_other", SaveMode.Append) GmeiConfig.writeToJDBCTable(result2, "strategy_other", SaveMode.Append)
//统计新用户点击率 //统计新用户点击率
val devicee_id_newUser = sc.sql( val devicee_id_newUser = sc.sql(
s""" s"""
|select distinct(device_id) as device_id |select distinct(device_id) as device_id
...@@ -442,7 +442,7 @@ object Gini_coefficient { ...@@ -442,7 +442,7 @@ object Gini_coefficient {
""".stripMargin """.stripMargin
) )
agency_id.createOrReplaceTempView("agency_id") agency_id.createOrReplaceTempView("agency_id")
//统计次数 //统计次数
val diary_clk_num = sc.sql( val diary_clk_num = sc.sql(
s""" s"""
|select temp1.diary_id as diary_id,count(ov.cl_id) as diary_clk_num |select temp1.diary_id as diary_id,count(ov.cl_id) as diary_clk_num
...@@ -468,3 +468,4 @@ object Gini_coefficient { ...@@ -468,3 +468,4 @@ object Gini_coefficient {
eda/feededa/src/main/scala/com/gmei/temp_analysis.scala
View file @ c5317d95
...@@ -47,14 +47,14 @@ object temp_analysis { ...@@ -47,14 +47,14 @@ object temp_analysis {
val spark_env = GmeiConfig.getSparkSession() val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2 val sc = spark_env._2
// val ti = new TiContext(sc) // val ti = new TiContext(sc)
sc.sql("use jerry_prod") sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list") // ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._ import sc.implicits._
...@@ -80,23 +80,23 @@ object temp_analysis { ...@@ -80,23 +80,23 @@ object temp_analysis {
agency_id.createOrReplaceTempView("agency_id") agency_id.createOrReplaceTempView("agency_id")
// //每日新用户 // //每日新用户
// val device_id_newUser = sc.sql( // val device_id_newUser = sc.sql(
// s""" // s"""
// |select distinct(device_id) as device_id // |select distinct(device_id) as device_id
// |from online.ml_device_day_active_status // |from online.ml_device_day_active_status
// |where active_type != '4' // |where active_type != '4'
// |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3' // |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
// | ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang' // | ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
// | ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1' // | ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
// | ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4' // | ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
// | ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100' // | ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
// | ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ' // | ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
// | ,'promotion_shike','promotion_julang_jl03') // | ,'promotion_shike','promotion_julang_jl03')
// |and partition_date ='${partition_date}' // |and partition_date ='${partition_date}'
// """.stripMargin // """.stripMargin
// ) // )
// device_id_newUser.createOrReplaceTempView("device_id_new") // device_id_newUser.createOrReplaceTempView("device_id_new")
val blacklist_id = sc.sql( val blacklist_id = sc.sql(
s""" s"""
...@@ -135,13 +135,13 @@ object temp_analysis { ...@@ -135,13 +135,13 @@ object temp_analysis {
diary_clk_all.show(80) diary_clk_all.show(80)
//日记本点击 //日记本点击
val referrer=List("about_me_message_list","all_case_service_comment","all_cases","diary_detail","diary_list" val referrer=List("about_me_message_list","all_case_service_comment","all_cases","diary_detail","diary_list"
,"diary_listof_related_service","answer_detail","community_home","conversation_detail","create_diary_title","diary_listof_related_service", ,"diary_listof_related_service","answer_detail","community_home","conversation_detail","create_diary_title","diary_listof_related_service",
"doctor_all_cases","hospital_all_cases","my_favor","my_order","order_detail","personal_store_diary_list","received_votes", "doctor_all_cases","hospital_all_cases","my_favor","my_order","order_detail","personal_store_diary_list","received_votes",
"topic_detail","welfare_detail","welfare_list","welfare_special","wiki_detail","zone_detail", "topic_detail","welfare_detail","welfare_list","welfare_special","wiki_detail","zone_detail",
"expert_detail","free_activity_detail","home","message_home","my_diary","organization_detail","other_homepage","question_detail", "expert_detail","free_activity_detail","home","message_home","my_diary","organization_detail","other_homepage","question_detail",
"search_result_diary","search_result_more","welfare_detail","zone_v3") "search_result_diary","search_result_more","welfare_detail","zone_v3")
for( a <- referrer ){ for( a <- referrer ){
val diary_clk_temp = sc.sql( val diary_clk_temp = sc.sql(
s""" s"""
...@@ -211,14 +211,14 @@ object ARPU_COM { ...@@ -211,14 +211,14 @@ object ARPU_COM {
val spark_env = GmeiConfig.getSparkSession() val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2 val sc = spark_env._2
// val ti = new TiContext(sc) // val ti = new TiContext(sc)
sc.sql("use jerry_prod") sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list") // ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._ import sc.implicits._
...@@ -334,14 +334,14 @@ object hospital_gengmei { ...@@ -334,14 +334,14 @@ object hospital_gengmei {
val spark_env = GmeiConfig.getSparkSession() val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2 val sc = spark_env._2
// val ti = new TiContext(sc) // val ti = new TiContext(sc)
sc.sql("use jerry_prod") sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list") // ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._ import sc.implicits._
...@@ -406,19 +406,19 @@ object meigou_xiaofei_renshu { ...@@ -406,19 +406,19 @@ object meigou_xiaofei_renshu {
val spark_env = GmeiConfig.getSparkSession() val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2 val sc = spark_env._2
// val ti = new TiContext(sc) // val ti = new TiContext(sc)
sc.sql("use jerry_prod") sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list") // ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._ import sc.implicits._
// val stat_date = GmeiConfig.getMinusNDate(1) // val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date=param.date val stat_date=param.date
//println(param.date) //println(param.date)
val partition_date = stat_date.replace("-","") val partition_date = stat_date.replace("-","")
...@@ -460,21 +460,21 @@ object meigou_xiaofei_renshu { ...@@ -460,21 +460,21 @@ object meigou_xiaofei_renshu {
final_id.createOrReplaceTempView("final_id") final_id.createOrReplaceTempView("final_id")
// val meigou_price = sc.sql( // val meigou_price = sc.sql(
// s""" // s"""
// |select md.user_id,sum(md.gengmei_price) as pay_all // |select md.user_id,sum(md.gengmei_price) as pay_all
// |from online.ml_meigou_order_detail md left join final_id // |from online.ml_meigou_order_detail md left join final_id
// |on md.device_id = final_id.device_id // |on md.device_id = final_id.device_id
// |where md.status= 2 // |where md.status= 2
// |and final_id.device_id is null // |and final_id.device_id is null
// |and md.partition_date = '20181223' // |and md.partition_date = '20181223'
// |and md.pay_time is not null // |and md.pay_time is not null
// |and md.validate_time>'2017-01-01 00:00:00.0' // |and md.validate_time>'2017-01-01 00:00:00.0'
// |group by md.user_id // |group by md.user_id
// |order by sum(md.gengmei_price) // |order by sum(md.gengmei_price)
// """.stripMargin // """.stripMargin
// ) // )
// meigou_price.show(80) // meigou_price.show(80)
val meigou_price = sc.sql( val meigou_price = sc.sql(
...@@ -499,9 +499,9 @@ object meigou_xiaofei_renshu { ...@@ -499,9 +499,9 @@ object meigou_xiaofei_renshu {
|order by sum(md.gengmei_price) |order by sum(md.gengmei_price)
""".stripMargin """.stripMargin
) )
// meigou_price.show(80) // meigou_price.show(80)
// GmeiConfig.writeToJDBCTable(meigou_price, "meigou_price", SaveMode.Overwrite) // GmeiConfig.writeToJDBCTable(meigou_price, "meigou_price", SaveMode.Overwrite)
} }
...@@ -548,18 +548,18 @@ object alpha_ctr { ...@@ -548,18 +548,18 @@ object alpha_ctr {
val spark_env = GmeiConfig.getSparkSession() val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2 val sc = spark_env._2
// val ti = new TiContext(sc) // val ti = new TiContext(sc)
sc.sql("use jerry_prod") sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list") // ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._ import sc.implicits._
// val stat_date = GmeiConfig.getMinusNDate(1) // val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date = param.date val stat_date = param.date
//println(param.date) //println(param.date)
val partition_date = stat_date.replace("-","") val partition_date = stat_date.replace("-","")
...@@ -637,7 +637,7 @@ object alpha_ctr { ...@@ -637,7 +637,7 @@ object alpha_ctr {
// GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append) // GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append) GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
...@@ -662,7 +662,7 @@ object alpha_ctr { ...@@ -662,7 +662,7 @@ object alpha_ctr {
) )
val result3=device_num_count.join(duration_device,"stat_date") val result3=device_num_count.join(duration_device,"stat_date")
// GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append) // GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append) GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
...@@ -714,19 +714,19 @@ object copy_database { ...@@ -714,19 +714,19 @@ object copy_database {
val spark_env = GmeiConfig.getSparkSession() val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2 val sc = spark_env._2
// val ti = new TiContext(sc) // val ti = new TiContext(sc)
sc.sql("use jerry_prod") sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist") // ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "tl_hdfs_wiki_item_tag_view") // ti.tidbMapTable(dbName = "jerry_test", tableName = "tl_hdfs_wiki_item_tag_view")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "Knowledge_network") // ti.tidbMapTable(dbName = "jerry_test", tableName = "Knowledge_network")
// ti.tidbMapTable(dbName = "eagle", tableName = "src_mimas_prod_api_diary") // ti.tidbMapTable(dbName = "eagle", tableName = "src_mimas_prod_api_diary")
import sc.implicits._ import sc.implicits._
val stat_date = GmeiConfig.getMinusNDate(6) val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date=param.date // val stat_date=param.date
val partition_date = stat_date.replace("-","") val partition_date = stat_date.replace("-","")
val new_data = sc.sql( val new_data = sc.sql(
...@@ -754,4 +754,3 @@ object copy_database { ...@@ -754,4 +754,3 @@ object copy_database {
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment