Commit 5da4b7b6 authored by 张彦钊's avatar 张彦钊

Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

add write_data.sh
parents 40e538c6 7362dcd3
...@@ -699,6 +699,8 @@ object tag_value { ...@@ -699,6 +699,8 @@ object tag_value {
|from level3_id |from level3_id
""".stripMargin """.stripMargin
) )
level_value.createOrReplaceTempView("tagId_value")
// tag_value.show(300) // tag_value.show(300)
println("开始写入") println("开始写入")
...@@ -724,16 +726,34 @@ object tag_value { ...@@ -724,16 +726,34 @@ object tag_value {
val result = level_index_temp.select("tag_id").distinct().rdd.map{x => x.toString().substring(1,x.toString().length - 1)}.zipWithIndex().toDF("level_id","index_id") val result = level_index_temp.select("tag_id").distinct().rdd.map{x => x.toString().substring(1,x.toString().length - 1)}.zipWithIndex().toDF("level_id","index_id")
val test=result.select(result.col("level_id").cast(DoubleType).as("level_id"),result.col("index_id").cast(DoubleType).as("index_id")) val test=result.select(result.col("level_id").cast(DoubleType).as("level_id"),result.col("index_id").cast(DoubleType).as("index_id"))
test.createOrReplaceTempView("tag_level_index")
// val resDf = spark.createDataFrame(rowRdd) // val resDf = spark.createDataFrame(rowRdd)
println("开始写入") println("开始写入")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",test, table="tag_level_index",SaveMode.Overwrite) GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",test, table="tag_level_index",SaveMode.Overwrite)
println("写入完成") println("写入完成")
// zhengxing库里面的数据同步到jerry_prod // zhengxing库里面的数据同步到jerry_prod
val icon_today_train_data=sc.sql(
s"""
|select a.device_id,c.index_id as action,b.value from jerry_prod.icon_train_data a
|left join tagId_value b on a.tag_id=b.level_id
|left join tag_level_index c on a.tag_id=c.level_id
|where c.index_id is not null and b.value is not null
""".stripMargin
)
println("开始写入")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",icon_today_train_data, table="icon_today_train_data_test",SaveMode.Overwrite)
println("写入完成")
......
...@@ -450,7 +450,7 @@ object Repeated_content_recommendation_moreday { ...@@ -450,7 +450,7 @@ object Repeated_content_recommendation_moreday {
val exp_diary = sc.sql( val exp_diary = sc.sql(
s""" s"""
|select stat_date,device_id,concat_ws(',',collect_set(distinct cid_id)) as expoure_diary |select stat_date,device_id,concat_ws(',',collect_set(distinct cid_id)) as expoure_diary
|from data_feed_exposure_precise |from data_feed_exposure
|where cid_type = 'diary' |where cid_type = 'diary'
|and stat_date >='${date}' |and stat_date >='${date}'
|and device_id not in (select device_id from blacklist) |and device_id not in (select device_id from blacklist)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment