Skip to content

F
ffm-baseline
Commit c5317d95 authored by 张彦钊's avatar 张彦钊
Browse files
Options

把刚才修改的文件回滚

parent 680b4c83
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 99 deletions
eda/feededa/src/main/scala/com/gmei/Recommendation_strategy_all.scala
View file @ c5317d95
......@@ -52,7 +52,7 @@ object Recommendation_strategy_all {
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = param.date
// val stat_date = param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
val decive_id_oldUser = sc.sql(
......@@ -83,7 +83,7 @@ object Recommendation_strategy_all {
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
)
val imp_count_oldUser_Contrast = sc.sql(
s"""
......@@ -95,7 +95,7 @@ object Recommendation_strategy_all {
|and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${stat_date}'
""".stripMargin
)
)
val clk_count_oldUser_all = sc.sql(
s"""
......@@ -119,7 +119,7 @@ object Recommendation_strategy_all {
""".stripMargin
)
//获取策略命中用户device_id
//获取策略命中用户device_id
val device_id_cover = sc.sql(
s"""
|select distinct(device_id) as device_id
......@@ -183,13 +183,13 @@ object Recommendation_strategy_all {
""".stripMargin
)
val result1 = clk_count_oldUser_Contrast.join(imp_count_oldUser_Contrast,"stat_date")
.join(clk_count_oldUser_all,"stat_date")
.join(imp_count_oldUser_all,"stat_date")
.join(clk_count_oldUser_Cover,"stat_date")
.join(imp_count_oldUser_Cover,"stat_date")
.join(device_num_cover,"stat_date")
.join(device_num_1_hit,"stat_date")
.join(device_num_hit,"stat_date")
.join(clk_count_oldUser_all,"stat_date")
.join(imp_count_oldUser_all,"stat_date")
.join(clk_count_oldUser_Cover,"stat_date")
.join(imp_count_oldUser_Cover,"stat_date")
.join(device_num_cover,"stat_date")
.join(device_num_1_hit,"stat_date")
.join(device_num_hit,"stat_date")
result1.show()
GmeiConfig.writeToJDBCTable(result1, "Recommendation_strategy_temp", SaveMode.Append)
......@@ -287,7 +287,7 @@ object Recommendation_strategy_all {
GmeiConfig.writeToJDBCTable(result2, "strategy_other", SaveMode.Append)
//统计新用户点击率
//统计新用户点击率
val devicee_id_newUser = sc.sql(
s"""
|select distinct(device_id) as device_id
......@@ -442,7 +442,7 @@ object Gini_coefficient {
""".stripMargin
)
agency_id.createOrReplaceTempView("agency_id")
//统计次数
//统计次数
val diary_clk_num = sc.sql(
s"""
|select temp1.diary_id as diary_id,count(ov.cl_id) as diary_clk_num
......@@ -468,3 +468,4 @@ object Gini_coefficient {
eda/feededa/src/main/scala/com/gmei/temp_analysis.scala
View file @ c5317d95
......@@ -47,14 +47,14 @@ object temp_analysis {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
// val ti = new TiContext(sc)
// val ti = new TiContext(sc)
sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._
......@@ -80,23 +80,23 @@ object temp_analysis {
agency_id.createOrReplaceTempView("agency_id")
// //每日新用户
// val device_id_newUser = sc.sql(
// s"""
// |select distinct(device_id) as device_id
// |from online.ml_device_day_active_status
// |where active_type != '4'
// |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
// | ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
// | ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
// | ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
// | ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
// | ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
// | ,'promotion_shike','promotion_julang_jl03')
// |and partition_date ='${partition_date}'
// """.stripMargin
// )
// device_id_newUser.createOrReplaceTempView("device_id_new")
// //每日新用户
// val device_id_newUser = sc.sql(
// s"""
// |select distinct(device_id) as device_id
// |from online.ml_device_day_active_status
// |where active_type != '4'
// |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
// | ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
// | ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
// | ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
// | ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
// | ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
// | ,'promotion_shike','promotion_julang_jl03')
// |and partition_date ='${partition_date}'
// """.stripMargin
// )
// device_id_newUser.createOrReplaceTempView("device_id_new")
val blacklist_id = sc.sql(
s"""
......@@ -135,13 +135,13 @@ object temp_analysis {
diary_clk_all.show(80)
//日记本点击
//日记本点击
val referrer=List("about_me_message_list","all_case_service_comment","all_cases","diary_detail","diary_list"
,"diary_listof_related_service","answer_detail","community_home","conversation_detail","create_diary_title","diary_listof_related_service",
"doctor_all_cases","hospital_all_cases","my_favor","my_order","order_detail","personal_store_diary_list","received_votes",
"topic_detail","welfare_detail","welfare_list","welfare_special","wiki_detail","zone_detail",
"expert_detail","free_activity_detail","home","message_home","my_diary","organization_detail","other_homepage","question_detail",
"search_result_diary","search_result_more","welfare_detail","zone_v3")
"expert_detail","free_activity_detail","home","message_home","my_diary","organization_detail","other_homepage","question_detail",
"search_result_diary","search_result_more","welfare_detail","zone_v3")
for( a <- referrer ){
val diary_clk_temp = sc.sql(
s"""
......@@ -211,14 +211,14 @@ object ARPU_COM {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
// val ti = new TiContext(sc)
// val ti = new TiContext(sc)
sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._
......@@ -334,14 +334,14 @@ object hospital_gengmei {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
// val ti = new TiContext(sc)
// val ti = new TiContext(sc)
sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._
......@@ -406,19 +406,19 @@ object meigou_xiaofei_renshu {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
// val ti = new TiContext(sc)
// val ti = new TiContext(sc)
sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._
// val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date=param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
......@@ -460,21 +460,21 @@ object meigou_xiaofei_renshu {
final_id.createOrReplaceTempView("final_id")
// val meigou_price = sc.sql(
// s"""
// |select md.user_id,sum(md.gengmei_price) as pay_all
// |from online.ml_meigou_order_detail md left join final_id
// |on md.device_id = final_id.device_id
// |where md.status= 2
// |and final_id.device_id is null
// |and md.partition_date = '20181223'
// |and md.pay_time is not null
// |and md.validate_time>'2017-01-01 00:00:00.0'
// |group by md.user_id
// |order by sum(md.gengmei_price)
// """.stripMargin
// )
// meigou_price.show(80)
// val meigou_price = sc.sql(
// s"""
// |select md.user_id,sum(md.gengmei_price) as pay_all
// |from online.ml_meigou_order_detail md left join final_id
// |on md.device_id = final_id.device_id
// |where md.status= 2
// |and final_id.device_id is null
// |and md.partition_date = '20181223'
// |and md.pay_time is not null
// |and md.validate_time>'2017-01-01 00:00:00.0'
// |group by md.user_id
// |order by sum(md.gengmei_price)
// """.stripMargin
// )
// meigou_price.show(80)
val meigou_price = sc.sql(
......@@ -499,9 +499,9 @@ object meigou_xiaofei_renshu {
|order by sum(md.gengmei_price)
""".stripMargin
)
// meigou_price.show(80)
// meigou_price.show(80)
// GmeiConfig.writeToJDBCTable(meigou_price, "meigou_price", SaveMode.Overwrite)
// GmeiConfig.writeToJDBCTable(meigou_price, "meigou_price", SaveMode.Overwrite)
}
......@@ -548,18 +548,18 @@ object alpha_ctr {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
// val ti = new TiContext(sc)
// val ti = new TiContext(sc)
sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._
// val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date = param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
......@@ -637,7 +637,7 @@ object alpha_ctr {
// GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
// GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
......@@ -662,7 +662,7 @@ object alpha_ctr {
)
val result3=device_num_count.join(duration_device,"stat_date")
// GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append)
// GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
......@@ -714,19 +714,19 @@ object copy_database {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
// val ti = new TiContext(sc)
// val ti = new TiContext(sc)
sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "tl_hdfs_wiki_item_tag_view")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "Knowledge_network")
// ti.tidbMapTable(dbName = "eagle", tableName = "src_mimas_prod_api_diary")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "tl_hdfs_wiki_item_tag_view")
// ti.tidbMapTable(dbName = "jerry_test", tableName = "Knowledge_network")
// ti.tidbMapTable(dbName = "eagle", tableName = "src_mimas_prod_api_diary")
import sc.implicits._
val stat_date = GmeiConfig.getMinusNDate(6)
// val stat_date=param.date
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date=param.date
val partition_date = stat_date.replace("-","")
val new_data = sc.sql(
......@@ -754,4 +754,3 @@ object copy_database {
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment