Skip to content

F
ffm-baseline
Commit 156f1229 authored by 张彦钊's avatar 张彦钊
Browse files
Options

Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline 

修改ip
parents 2106d98b 2c910dda
Show whitespace changes
Inline Side-by-side
Showing with 88 additions and 80 deletions
eda/feededa/src/main/scala/com/gmei/GmeiConfig.scala
View file @ 156f1229
......@@ -38,6 +38,9 @@ object GmeiConfig extends Serializable {
sparkConf.set("spark.sql.crossJoin.enabled", "true")
sparkConf.set("spark.debug.maxToStringFields", "130")
sparkConf.set("spark.sql.broadcastTimeout", "6000")
sparkConf.set("spark.tispark.plan.allow_index_read", "false")
sparkConf.set("spark.hive.mapred.supports.subdirectories","true")
sparkConf.set("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive","true")
val spark = SparkSession
......
eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala
View file @ 156f1229
......@@ -413,8 +413,10 @@ object data_feed_exposure_precise {
// GmeiConfig.writeToJDBCTable(result, table = "data_feed_exposure_precise", SaveMode.Append)
print("写入开始")
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="data_feed_exposure_precise",SaveMode.Append)
print("写入结束")
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="data_feed_exposure_precise",SaveMode.Append)
// print("写入结束")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="data_feed_exposure_precise",SaveMode.Append)
println("写入完成")
// println("开始写入")
// GmeiConfig.writeToJDBCTable(config.getString("jerry.jdbcuri"),result, table="data_feed_exposure_precise",SaveMode.Append)
......
eda/feededa/src/main/scala/com/gmei/find_bug.scala
View file @ 156f1229
......@@ -138,7 +138,8 @@ object find_bug {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_Recommendation_strategy_temp",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="bug_Recommendation_strategy_temp",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="bug_Recommendation_strategy_temp",SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_Recommendation_strategy_temp",SaveMode.Append)
println("写入完成")
......@@ -243,7 +244,9 @@ object find_bug {
// GmeiConfig.writeToJDBCTable(result2, "bug_strategy_other", SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result2, table="bug_strategy_other",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result2, table="bug_strategy_other",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result2, table="bug_strategy_other",SaveMode.Append)
println("写入完成")
......@@ -325,7 +328,9 @@ object find_bug {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_Recommendation_strategy_newUser",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="bug_Recommendation_strategy_newUser",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="bug_Recommendation_strategy_newUser",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_Recommendation_strategy_newUser",SaveMode.Append)
println("写入完成")
......@@ -461,7 +466,10 @@ object CTR_precise {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_precise_Recommendation_strategy_temp",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="bug_precise_Recommendation_strategy_temp",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="bug_precise_Recommendation_strategy_temp",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_precise_Recommendation_strategy_temp",SaveMode.Append)
println("写入完成")
// GmeiConfig.writeToJDBCTable(result1, "bug_precise_Recommendation_strategy_temp", SaveMode.Append)
......@@ -540,7 +548,9 @@ object CTR_precise {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_precise_Recommendation_strategy_newUser",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="bug_precise_Recommendation_strategy_newUser",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="bug_precise_Recommendation_strategy_newUser",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_precise_Recommendation_strategy_newUser",SaveMode.Append)
println("写入完成")
......
eda/feededa/src/main/scala/com/gmei/temp_analysis.scala
View file @ 156f1229
......@@ -2,8 +2,9 @@ package com.gmei
import java.io.Serializable
import com.gmei
import com.gmei.WeafareStat.{defaultParams, parser}
import org.apache.spark.sql.{SaveMode}
import org.apache.spark.sql.SaveMode
//import org.apache.spark.sql.{SaveMode, TiContext}
import org.apache.log4j.{Level, Logger}
import scopt.OptionParser
......@@ -560,8 +561,8 @@ object alpha_ctr {
import sc.implicits._
// val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date = param.date
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
......@@ -642,10 +643,11 @@ object alpha_ctr {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="alpha_ctr",SaveMode.Append)
println("写入完成")
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="alpha_ctr",SaveMode.Append)
// GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
println("写入完成")
......@@ -671,14 +673,14 @@ object alpha_ctr {
val result3=device_num_count.join(duration_device,"stat_date")
// GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="alpha_duration",SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
println("写入完成")
// println("开始写入")
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="alpha_duration",SaveMode.Append)
// println("写入完成")
// GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
......
eda/feededa/src/main/scala/com/gmei/temp_count.scala
View file @ 156f1229
......@@ -168,7 +168,10 @@ object temp_count {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="ffm_diary_ctr",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="ffm_diary_ctr",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="ffm_diary_ctr",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="ffm_diary_ctr",SaveMode.Append)
println("写入完成")
......@@ -344,7 +347,10 @@ object Repeated_content_recommendation {
// GmeiConfig.writeToJDBCTable(df2, table = "Repeated_evaluation_indicator", SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df2, table="Repeated_evaluation_indicator",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df2, table="Repeated_evaluation_indicator",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df2, table="Repeated_evaluation_indicator",SaveMode.Append)
println("写入完成")
// val exp_diary_old = sc.sql(
......@@ -474,7 +480,10 @@ object Repeated_content_recommendation_moreday {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="Repeated_content_recommendation_moreday",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df_result, table="Repeated_content_recommendation_moreday",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df_result, table="Repeated_content_recommendation_moreday",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="Repeated_content_recommendation_moreday",SaveMode.Append)
println("写入完成")
......@@ -648,7 +657,10 @@ object GetHiveSearchData {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="GetHiveSearchData_CTR",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df_result, table="GetHiveSearchData_CTR",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df_result, table="GetHiveSearchData_CTR",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="GetHiveSearchData_CTR",SaveMode.Append)
println("写入完成")
......
eda/feededa/src/main/scala/com/gmei/testt.scala
View file @ 156f1229
......@@ -62,10 +62,6 @@ object testt {
)
blacklist.createOrReplaceTempView("blacklist")
// sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/brickhouse-0.7.1-SNAPSHOT.jar")
// sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar")
// sc.sql("CREATE TEMPORARY FUNCTION json_map AS 'brickhouse.udf.json.JsonMapUDF'")
// sc.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJsonFormatCheck'")
val agency_id = sc.sql(
......@@ -83,7 +79,7 @@ object testt {
|AND pv_ratio >= 0.95
""".stripMargin
)
// agency_id.show()
agency_id.show()
agency_id.createOrReplaceTempView("agency_id")
//每日新用户
val device_id_newUser = sc.sql(
......@@ -124,7 +120,7 @@ object testt {
|and blacklist.device_id is null
""".stripMargin
)
// device_id_oldUser.show()
device_id_oldUser.show()
device_id_oldUser.createOrReplaceTempView("device_id_old")
......@@ -142,6 +138,9 @@ object testt {
""".stripMargin
)
diary_meigou_temp.createOrReplaceTempView("diary_meigou_temp")
diary_meigou_temp.show()
val diary_meigou_device = sc.sql(
s"""
|select dt.device_id
......@@ -151,6 +150,8 @@ object testt {
""".stripMargin
)
diary_meigou_device.createOrReplaceTempView("diary_meigou_device")
diary_meigou_device.show()
//新用户到美购详情页的转化
val diary_meigou_newUser = sc.sql(
s"""
......@@ -160,6 +161,7 @@ object testt {
|where device_id_new.device_id is not null
""".stripMargin
)
diary_meigou_newUser.show()
//老用户到美购详情页的转化
val diary_meigou_oldUser = sc.sql(
s"""
......@@ -169,20 +171,9 @@ object testt {
|where device_id_old.device_id is not null
""".stripMargin
)
diary_meigou_oldUser.show()
// val diary_meigou_count = sc.sql(
// s"""
// |select '${stat_date}' as stat_date, count(page_name) as diary_meigou_count
// |from online.bl_hdfs_page_view_updates ou left join agency_id
// |on ou.cl_id = agency_id.device_id
// |where ou.partition_date = '${partition_date}'
// |and ou.page_name='welfare_detail'
// |and ou.referrer='diary_detail'
// |and agency_id.device_id is null
// |and ou.cl_id not in (select device_id from blacklist)
// """.stripMargin
// )
// diary_meigou_count.show()
//2.日记本点击数
val diary_clk_temp = sc.sql(
......@@ -197,6 +188,7 @@ object testt {
""".stripMargin
)
diary_clk_temp.createOrReplaceTempView("diary_clk_temp")
diary_clk_temp.show()
val diary_clk_device = sc.sql(
s"""
......@@ -208,6 +200,7 @@ object testt {
)
diary_clk_device.createOrReplaceTempView("diary_clk_device")
diary_clk_device.show()
//新用户日记本点击
val diary_clk_newUser = sc.sql(
......@@ -218,6 +211,7 @@ object testt {
|where device_id_new.device_id is not null
""".stripMargin
)
diary_clk_newUser.show()
//老用户日记本点击
val diary_clk_oldUser = sc.sql(
s"""
......@@ -227,19 +221,8 @@ object testt {
|where device_id_old.device_id is not null
""".stripMargin
)
// val diary_clk = sc.sql(
// s"""
// |select '${stat_date}' as stat_date,count(cl_id) as diary_clk
// |from online.tl_hdfs_maidian_view ov left join agency_id
// |on ov.cl_id = agency_id.device_id
// |where ov.action = 'on_click_diary_card'
// |and ov.cl_id != "NULL"
// |and ov.partition_date='${partition_date}'
// |and agency_id.device_id is null
// |and ov.cl_id not in (select device_id from blacklist)
// """.stripMargin
// )
// diary_clk.show()
diary_clk_oldUser.show()
//3.日记本曝光数
val diary_expoure_temp=sc.sql(
......@@ -254,6 +237,7 @@ object testt {
""".stripMargin
)
diary_expoure_temp.createOrReplaceTempView("diary_expoure_temp")
diary_expoure_temp.show()
val diary_expoure_device = sc.sql(
s"""
......@@ -263,6 +247,7 @@ object testt {
|where blacklist.device_id is null
""".stripMargin
)
diary_expoure_device.show()
diary_expoure_device.createOrReplaceTempView("diary_expoure_device")
//新用户日记本曝光
......@@ -274,6 +259,8 @@ object testt {
|where device_id_new.device_id is not null
""".stripMargin
)
diary_exp_newUser.show()
//老用户日记本曝光
val diary_exp_oldUser = sc.sql(
s"""
......@@ -283,20 +270,9 @@ object testt {
|where device_id_old.device_id is not null
""".stripMargin
)
diary_exp_oldUser.show()
// val diary_expoure=sc.sql(
// s"""
// |select '${stat_date}' as stat_date,count(cl_id) as diary_expoure
// |from online.ml_community_exposure_detail od left join agency_id
// |on od.cl_id = agency_id.device_id
// |where od.business_type = "diary"
// |and od.cl_id != "NULL"
// |and od.partition_date='${partition_date}'
// |and agency_id.device_id is null
// |and od.cl_id not in (select device_id from blacklist)
// """.stripMargin
// )
// diary_expoure.show()
//4.搜索次数统计
val search_device_temp = sc.sql(
......@@ -310,6 +286,7 @@ object testt {
""".stripMargin
)
search_device_temp.createOrReplaceTempView("search_device_temp")
search_device_temp.show()
val search_device = sc.sql(
s"""
......@@ -320,6 +297,7 @@ object testt {
""".stripMargin
)
search_device.createOrReplaceTempView("search_device")
search_device.show()
//新用户搜索次数
val search_newUser = sc.sql(
s"""
......@@ -329,6 +307,7 @@ object testt {
|where device_id_new.device_id is not null
""".stripMargin
)
search_newUser.show()
//老用户日搜索次数
val search_oldUser = sc.sql(
s"""
......@@ -338,6 +317,7 @@ object testt {
|where device_id_old.device_id is not null
""".stripMargin
)
search_oldUser.show()
//5.登录人数
val log_device_temp = sc.sql(
......@@ -350,6 +330,7 @@ object testt {
""".stripMargin
)
log_device_temp.createOrReplaceTempView("log_device_temp")
log_device_temp.show()
val log_device = sc.sql(
s"""
......@@ -360,6 +341,7 @@ object testt {
""".stripMargin
)
log_device.createOrReplaceTempView("log_device")
log_device.show()
//新用户登录人数
val log_newUser = sc.sql(
......@@ -370,6 +352,7 @@ object testt {
|where device_id_new.device_id is not null
""".stripMargin
)
log_newUser.show()
//老用户登录人数
val log_oldUser = sc.sql(
s"""
......@@ -379,17 +362,8 @@ object testt {
|where device_id_old.device_id is not null
""".stripMargin
)
// val log_num = sc.sql(
// s"""
// |select '${stat_date}' as stat_date,count(distinct(device_id)) as log_num
// |from data_feed_exposure oe left join agency_id
// |on oe.device_id = agency_id.device_id
// |and oe.stat_date ='${stat_date}'
// |and agency_id.device_id is null
// |and oe.device_id not in (select device_id from blacklist)
// """.stripMargin
// )
// log_num.show()
log_oldUser.show()
val result = diary_meigou_newUser.join(diary_meigou_oldUser,"stat_date")
.join(diary_clk_newUser,"stat_date")
......@@ -405,7 +379,9 @@ object testt {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="diary_meigou_crv",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="diary_meigou_crv",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="diary_meigou_crv",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="diary_meigou_crv",SaveMode.Append)
println("写入完成")
......@@ -776,7 +752,10 @@ object diary_clk_card {
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="on_click_diary_card",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="on_click_diary_card",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="on_click_diary_card",SaveMode.Append)
// println("写入完成")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="on_click_diary_card",SaveMode.Append)
println("写入完成")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment