Skip to content

F
ffm-baseline
Commit 34e8964c authored by 张彦钊's avatar 张彦钊
Browse files
Options

Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline 

change test file
parents ca178971 d5e06565
Hide whitespace changes
Inline Side-by-side
Showing with 43 additions and 24 deletions
eda/feededa/src/main/resources/application.properties
View file @ 34e8964c
......@@ -10,7 +10,6 @@ pre.tidb.jdbcuri=jdbc:mysql://192.168.16.11:4000/eagle?user=root&password=&rewri
pre.tispark.pd.addresses=192.168.16.11:2379
pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas_prod?user=mimas&password=workwork&rewriteBatchedStatements=true
<<<<<<< HEAD
#prod.tidb.jdbcuri=jdbc:mysql://10.66.157.22:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
#prod.gold.jdbcuri=jdbc:mysql://rm-m5e842126ng59jrv6.mysql.rds.aliyuncs.com/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
#prod.mimas.jdbcuri=jdbc:mysql://rm-m5emg41za2w7l6au3.mysql.rds.aliyuncs.com/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true
......@@ -19,7 +18,6 @@ pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas
#prod.redis.host=10.30.50.58
#prod.redis.port=6379
=======
#阿里云线上配置
......@@ -34,7 +32,6 @@ pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas
#prod.jerry.jdbcuri_new=jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
#腾讯云线上配置
>>>>>>> 6bb8533b68efef7c647251ef08479560d5e1216a
prod.gold.jdbcuri=jdbc:mysql://172.16.30.136/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
prod.mimas.jdbcuri=jdbc:mysql://172.16.30.138/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true
prod.gaia.jdbcuri=jdbc:mysql://172.16.30.143/zhengxing?user=work&password=BJQaT9VzDcuPBqkd&rewriteBatchedStatements=true
......
eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala
View file @ 34e8964c
......@@ -3,7 +3,7 @@ package com.gmei
import java.io.Serializable
import com.gmei.WeafareStat.{defaultParams, parser}
import org.apache.spark.sql.{SaveMode}
import org.apache.spark.sql.SaveMode
import org.apache.log4j.{Level, Logger}
import scopt.OptionParser
import com.gmei.lib.AbstractParams
......@@ -82,6 +82,7 @@ object data_feed_exposure_precise {
| and partition_date='${partition_date}'
""".stripMargin
)
result01.show()
result01.createOrReplaceTempView("temp_result")
......@@ -411,10 +412,12 @@ object data_feed_exposure_precise {
)
// GmeiConfig.writeToJDBCTable(result, table = "data_feed_exposure_precise", SaveMode.Append)
print("写入开始")
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="data_feed_exposure_precise",SaveMode.Append)
print("写入结束")
// println("开始写入")
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="data_feed_exposure_precise",SaveMode.Append)
// GmeiConfig.writeToJDBCTable(config.getString("jerry.jdbcuri"),result, table="data_feed_exposure_precise",SaveMode.Append)
// println("写入完成")
}
......
eda/feededa/src/main/scala/com/gmei/find_bug.scala
View file @ 34e8964c
......@@ -55,7 +55,7 @@ object find_bug {
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = param.date
//println(param.date)
println(stat_date)
val partition_date = stat_date.replace("-","")
val decive_id_oldUser = sc.sql(
s"""
......@@ -72,13 +72,14 @@ object find_bug {
|and partition_date ='${partition_date}'
""".stripMargin
)
decive_id_oldUser.show()
decive_id_oldUser.createOrReplaceTempView("device_id_old")
val clk_count_oldUser_Contrast = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as clk_count_oldUser_Contrast
|from data_feed_click jd inner join device_id_old
|from jerry_prod.data_feed_click jd inner join device_id_old
|on jd.device_id = device_id_old.device_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id regexp'1$$'
......@@ -86,11 +87,13 @@ object find_bug {
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
clk_count_oldUser_Contrast.show()
val imp_count_oldUser_Contrast = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as imp_count_oldUser_Contrast
|from data_feed_exposure je inner join device_id_old
|from jerry_prod.data_feed_exposure je inner join device_id_old
|on je.device_id = device_id_old.device_id
|where je.cid_type = 'diary'
|and je.device_id regexp'1$$'
......@@ -98,28 +101,32 @@ object find_bug {
|and je.stat_date ='${stat_date}'
""".stripMargin
)
imp_count_oldUser_Contrast.show()
val clk_count_oldUser_all = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as clk_count_oldUser_all
|from data_feed_click jd inner join device_id_old
|from jerry_prod.data_feed_click jd inner join device_id_old
|on jd.device_id = device_id_old.device_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
clk_count_oldUser_all.show()
val imp_count_oldUser_all = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as imp_count_oldUser_all
|from data_feed_exposure je inner join device_id_old
|from jerry_prod.data_feed_exposure je inner join device_id_old
|on je.device_id = device_id_old.device_id
|where je.cid_type = 'diary'
|and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${stat_date}'
""".stripMargin
)
imp_count_oldUser_all.show()
val result1 = clk_count_oldUser_Contrast.join(imp_count_oldUser_Contrast,"stat_date")
......@@ -141,7 +148,7 @@ object find_bug {
val clk_active_1 = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(jd.cid_id) as clk_active_1
|from data_feed_click jd inner join device_id_old
|from jerry_prod.data_feed_click jd inner join device_id_old
|on jd.device_id = device_id_old.device_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id regexp'1$$'
......@@ -149,12 +156,13 @@ object find_bug {
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
clk_active_1.show()
//device_id尾号1有点击用户日记本曝光数
val imp_active_1 = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(je.cid_id) as imp_active_1
|from data_feed_exposure je inner join device_id_old
|from jerry_prod.data_feed_exposure je inner join device_id_old
|on je.device_id = device_id_old.device_id
|where je.cid_type = 'diary'
|and je.device_id in (select distinct(device_id) from data_feed_click where device_id regexp '1$$' and stat_date = '${stat_date}')
......@@ -162,12 +170,13 @@ object find_bug {
|and je.stat_date ='${stat_date}'
""".stripMargin
)
imp_active_1.show()
//device_id尾号1点击日记本用户数
val clk_diary_device = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(distinct(jd.device_id)) as clk_diary_device
|from data_feed_click jd inner join device_id_old
|from jerry_prod.data_feed_click jd inner join device_id_old
|on jd.device_id = device_id_old.device_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id regexp'1$$'
......@@ -175,24 +184,26 @@ object find_bug {
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
clk_diary_device.show()
//所有有点击用户日记本点击数
val clk_active_all = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(jd.cid_id) as clk_active_all
|from data_feed_click jd inner join device_id_old
|from jerry_prod.data_feed_click jd inner join device_id_old
|on jd.device_id = device_id_old.device_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
clk_active_all.show()
//所有有点击用户日记本曝光数
val imp_active_all = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(je.cid_id) as imp_active_all
|from data_feed_exposure je inner join device_id_old
|from jerry_prod.data_feed_exposure je inner join device_id_old
|on je.device_id = device_id_old.device_id
|where je.cid_type = 'diary'
|and je.device_id in (select distinct(device_id) from data_feed_click where stat_date = '${stat_date}')
......@@ -200,23 +211,27 @@ object find_bug {
|and je.stat_date ='${stat_date}'
""".stripMargin
)
imp_active_all.show()
//策略命中用户点击日记本用户数
val clk_diary_device_cover = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(distinct(device_id)) as clk_diary_device_cover
|from merge_queue_table
|from jerry_prod.merge_queue_table
|where device_id in (select distinct(device_id) from data_feed_click where stat_date = '${stat_date}')
""".stripMargin
)
clk_diary_device_cover.show()
//策略命中用户总数
val device_all_cover = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(distinct(device_id)) as device_all_cover
|from merge_queue_table
|from jerry_prod.merge_queue_table
""".stripMargin
)
device_all_cover.show()
val result2 = clk_active_1.join(imp_active_1,"stat_date")
.join(clk_active_all,"stat_date")
......@@ -254,19 +269,20 @@ object find_bug {
val clk_count_newUser_Contrast = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as clk_count_newUser_Contrast
|from data_feed_click jd inner join device_id_new
|from jerry_prod.data_feed_click jd inner join device_id_new
|on jd.device_id = device_id_new.device_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id regexp'1$$'
|and jd.device_id not in (select device_id from blacklist)
|and jd.device_id not in (select device_id from jerry_prod.blacklist)
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
clk_count_newUser_Contrast.show()
val imp_count_newUser_Contrast = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as imp_count_newUser_Contrast
|from data_feed_exposure je inner join device_id_new
|from jerry_prod.data_feed_exposure je inner join device_id_new
|on je.device_id = device_id_new.device_id
|where je.cid_type = 'diary'
|and je.device_id regexp'1$$'
......@@ -274,28 +290,31 @@ object find_bug {
|and je.stat_date ='${stat_date}'
""".stripMargin
)
imp_count_newUser_Contrast.show()
val clk_count_newUser_all = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as clk_count_newUser_all
|from data_feed_click jd inner join device_id_new
|from jerry_prod.data_feed_click jd inner join device_id_new
|on jd.device_id = device_id_new.device_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id not in (select device_id from blacklist)
|and jd.device_id not in (select device_id from jerry_prod.blacklist)
|and jd.stat_date ='${stat_date}'
""".stripMargin
)
clk_count_newUser_all.show()
val imp_count_newUser_all = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as imp_count_newUser_all
|from data_feed_exposure je inner join device_id_new
|from jerry_prod.data_feed_exposure je inner join device_id_new
|on je.device_id = device_id_new.device_id
|where je.cid_type = 'diary'
|and je.device_id not in (select device_id from blacklist)
|and je.device_id not in (select device_id from jerry_prod.blacklist)
|and je.stat_date ='${stat_date}'
""".stripMargin
)
imp_count_newUser_all.show()
val result3 = clk_count_newUser_Contrast.join(imp_count_newUser_Contrast,"stat_date")
.join(clk_count_newUser_all,"stat_date")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment