Skip to content

F
ffm-baseline
Commit b6289c2d authored by 王志伟's avatar 王志伟
Browse files
Options

补充数据

parent 88978647
Hide whitespace changes
Inline Side-by-side
Showing with 8 additions and 162 deletions
eda/feededa/src/main/scala/com/gmei/find_bug.scala
View file @ b6289c2d
......@@ -51,8 +51,8 @@ object find_bug {
ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date = param.date
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
val decive_id_oldUser = sc.sql(
......@@ -346,8 +346,8 @@ object CTR_precise {
ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date = param.date
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
val decive_id_oldUser = sc.sql(
......
eda/feededa/src/main/scala/com/gmei/temp_analysis.scala
View file @ b6289c2d
......@@ -160,22 +160,6 @@ object temp_analysis {
}
//5.登录人数
val log_device_temp = sc.sql(
s"""
|select oe.stat_date,count(distinct(oe.device_id)) as log_num
|from data_feed_exposure oe left join final_id
|on oe.device_id = final_id.device_id
|and oe.stat_date >='2018-11-01'
|and final_id.device_id is null
|group by oe.stat_date
|order by oe.stat_date
""".stripMargin
)
println("登录人数统计：")
log_device_temp.show(80)
}
......@@ -522,7 +506,7 @@ object meigou_xiaofei_renshu {
object smart_rank_count {
object alpha_ctr {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
......@@ -571,145 +555,7 @@ object smart_rank_count {
//println(param.date)
val partition_date = stat_date.replace("-","")
val agency_id = sc.sql(
s"""
|SELECT DISTINCT(cl_id) as device_id
|FROM online.ml_hospital_spam_pv_day
|WHERE partition_date >= '20180402'
|AND partition_date <= '${partition_date}'
|AND pv_ratio >= 0.95
|UNION ALL
|SELECT DISTINCT(cl_id) as device_id
|FROM online.ml_hospital_spam_pv_month
|WHERE partition_date >= '20171101'
|AND partition_date <= '${partition_date}'
|AND pv_ratio >= 0.95
""".stripMargin
)
agency_id.createOrReplaceTempView("agency_id")
val blacklist_id = sc.sql(
s"""
|SELECT device_id
|from blacklist
""".stripMargin
)
blacklist_id.createOrReplaceTempView("blacklist_id")
val final_id = sc.sql(
s"""
|select device_id
|from agency_id
|UNION ALL
|select device_id
|from blacklist_id
""".stripMargin
)
final_id.createOrReplaceTempView("final_id")
val user_city_meigou_view = sc.sql(
s"""
|select ov.cl_id as device_id,ov.city_id as device_city,ov.params['business_id'] as meigou_id
|from online.tl_hdfs_maidian_view ov left join final_id
|on ov.cl_id = final_id.device_id
|where ov.action = "page_view"
|and ov.params['page_name']="welfare_detail"
|and ov.partition_date >='20181101'
|and ov.partition_date <'20181201'
|and ov.city_id is not null
|and final_id.device_id is null
""".stripMargin
)
user_city_meigou_view.createOrReplaceTempView("user_city_meigou_view")
val meigou_city = sc.sql(
s"""
|select b.id as meigou_id,d.city_id as meigou_city
|from online.tl_meigou_service_view b
|left join online.tl_hdfs_doctor_view c on b.doctor_id=c.id
|left join online.tl_hdfs_hospital_view d on c.hospital_id=d.id
|where b.partition_date='20181228'
|and c.partition_date='20181228'
|and d.partition_date='20181228'
""".stripMargin
)
meigou_city.createOrReplaceTempView("meigou_city")
val meigou_pv_tongcheng = sc.sql(
s"""
|select a.device_id,a.device_city,a.meigou_id,b.meigou_city
|from user_city_meigou_view a
|left join meigou_city b
|on a.meigou_id = b.meigou_id
""".stripMargin
)
meigou_pv_tongcheng.createOrReplaceTempView("meigou_pv_tongcheng")
val meigou_pv_count = sc.sql(
s"""
|select '2018-11' as stat_date,meigou_city,count(device_id) as meigou_pv,count(distinct(device_id)) as meigou_device_num
|from meigou_pv_tongcheng
|where device_city = meigou_city
|group by meigou_city
""".stripMargin
)
meigou_pv_count.createOrReplaceTempView("meigou_pv_count")
//开始计算咨询
val zixun_meigou_view = sc.sql(
s"""
|select ov.cl_id as device_id,ov.city_id as device_city,ov.params['service_id'] as meigou_id
|from online.tl_hdfs_maidian_view ov left join final_id
|on ov.cl_id = final_id.device_id
|where ov.partition_date >= '20181101'
|and ov.partition_date < '20181201'
|and ov.action = 'welfare_detail_click_message'
|and final_id.device_id is null
""".stripMargin
)
zixun_meigou_view.createOrReplaceTempView("zixun_meigou_view")
val zixun_meigou_tongcheng = sc.sql(
s"""
|select a.device_id,a.device_city,a.meigou_id,b.meigou_city
|from zixun_meigou_view a
|left join meigou_city b
|on a.meigou_id=b.meigou_id
""".stripMargin
)
zixun_meigou_tongcheng.createOrReplaceTempView("zixun_meigou_tongcheng")
val zixun_pv_count = sc.sql(
s"""
|select '2018-11' as stat_date,meigou_city,count(device_id) as meigou_zixun,count(distinct(device_id)) as meigou_zixun_device_num
|from zixun_meigou_tongcheng
|where device_city=meigou_city
|group by meigou_city
""".stripMargin
)
zixun_pv_count.createOrReplaceTempView("zixun_pv_count")
//开始计算每个地区每月新增设备
val device_new_count = sc.sql(
s"""
|select first_city,count(distinct(device_id)) as new_device_month
|from online.ml_device_day_active_status
|where active_type != '4'
|and partition_date >='20181101'
|and partition_date <'20181201'
|group by first_city
""".stripMargin
)
device_new_count.createOrReplaceTempView("device_new_count")
//将所有的数据综合一起
val all_count = sc.sql(
val click_count_recommend = sc.sql(
s"""
|select mc.stat_date,mc.meigou_city,mc.meigou_pv,mc.meigou_device_num,zc.meigou_zixun,zc.meigou_zixun_device_num,dc.new_device_month
|from meigou_pv_count mc
......
eda/feededa/src/main/scala/com/gmei/testt.scala
View file @ b6289c2d
......@@ -454,8 +454,8 @@ object diary_clk_card {
ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure_precise")
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date=param.date
// val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date=param.date
val partition_date = stat_date.replace("-","")
//机构id
val blacklist = sc.sql(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment