Skip to content

F
ffm-baseline
Commit 25b39fd1 authored by 王志伟's avatar 王志伟
Browse files
Options

迁移到腾讯云

parent 4c6c3907
Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 67 deletions
eda/feededa/src/main/resources/application.properties
View file @ 25b39fd1
......@@ -11,10 +11,18 @@ pre.tidb.jdbcuri=jdbc:mysql://192.168.16.11:4000/eagle?user=root&password=&rewri
pre.tispark.pd.addresses=192.168.16.11:2379
pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas_prod?user=mimas&password=workwork&rewriteBatchedStatements=true
prod.tidb.jdbcuri=jdbc:mysql://10.66.157.22:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
prod.gold.jdbcuri=jdbc:mysql://rm-m5e842126ng59jrv6.mysql.rds.aliyuncs.com/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
prod.mimas.jdbcuri=jdbc:mysql://rm-m5emg41za2w7l6au3.mysql.rds.aliyuncs.com/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true
prod.gaia.jdbcuri=jdbc:mysql://rdsfewzdmf0jfjp9un8xj.mysql.rds.aliyuncs.com/zhengxing?user=work&password=BJQaT9VzDcuPBqkd&rewriteBatchedStatements=true
prod.tispark.pd.addresses=10.66.157.22:2379
prod.redis.host=10.30.50.58
prod.redis.port=6379
#prod.tidb.jdbcuri=jdbc:mysql://10.66.157.22:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
#prod.gold.jdbcuri=jdbc:mysql://rm-m5e842126ng59jrv6.mysql.rds.aliyuncs.com/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
#prod.mimas.jdbcuri=jdbc:mysql://rm-m5emg41za2w7l6au3.mysql.rds.aliyuncs.com/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true
#prod.gaia.jdbcuri=jdbc:mysql://rdsfewzdmf0jfjp9un8xj.mysql.rds.aliyuncs.com/zhengxing?user=work&password=BJQaT9VzDcuPBqkd&rewriteBatchedStatements=true
#prod.tispark.pd.addresses=10.66.157.22:2379
#prod.redis.host=10.30.50.58
#prod.redis.port=6379
prod.gold.jdbcuri=jdbc:mysql://172.16.30.136/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
prod.mimas.jdbcuri=jdbc:mysql://172.16.30.138/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true
prod.gaia.jdbcuri=jdbc:mysql://172.16.30.143/zhengxing?user=work&password=BJQaT9VzDcuPBqkd&rewriteBatchedStatements=true
prod.tidb.jdbcuri=jdbc:mysql://172.16.40.158:4000/eagle?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
prod.jerry.jdbcuri=jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
prod.tispark.pd.addresses=172.16.40.158:2379
eda/feededa/src/main/scala/com/gmei/WeafareStat.scala
View file @ 25b39fd1
......@@ -39,10 +39,10 @@ object WeafareStat {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
val ti = new TiContext(sc)
ti.tidbMapTable(dbName = "eagle",tableName = "diary_video")
ti.tidbMapTable(dbName = "jerry_prod",tableName = "data_feed_click")
ti.tidbMapTable(dbName = "eagle",tableName = "feed_diary_boost")
// val ti = new TiContext(sc)
// ti.tidbMapTable(dbName = "eagle",tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod",tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "eagle",tableName = "feed_diary_boost")
import sc.implicits._
......@@ -51,9 +51,9 @@ object WeafareStat {
val video_cids = sc.sql(
s"""
|select distinct(cid_id) as cid_id
|from data_feed_click
|from jerry_prod.data_feed_click
|where cid_type = 'diary'
|and cid_id in (select cid from diary_video where stat_date='${stat_date}')
|and cid_id in (select cid from eagle.diary_video where stat_date='${stat_date}')
|and stat_date ='${stat_date}'
""".stripMargin
)
......@@ -63,9 +63,9 @@ object WeafareStat {
val txt_cids = sc.sql(
s"""
|select distinct(cid_id) as cid_id
|from data_feed_click
|from jerry_prod.data_feed_click
|where cid_type = 'diary'
|and cid_id not in (select cid from diary_video where stat_date='${stat_date}')
|and cid_id not in (select cid from eagle.diary_video where stat_date='${stat_date}')
|and stat_date ='${stat_date}'
""".stripMargin
)
......@@ -101,9 +101,9 @@ object WeafareStat {
val video_clk_count = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as video_clk_count
|from data_feed_click
|from jerry_prod.data_feed_click
|where cid_type = 'diary'
|and cid_id in (select cid from diary_video where stat_date='${stat_date}')
|and cid_id in (select cid from eagle.diary_video where stat_date='${stat_date}')
|and stat_date='${stat_date}'
""".stripMargin
)
......@@ -112,9 +112,9 @@ object WeafareStat {
val txt_clk_count = sc.sql(
s"""
|select '${stat_date}' as stat_date, count(cid_id) as txt_clk_count
|from data_feed_click
|from jerry_prod.data_feed_click
|where cid_type = 'diary'
|and cid_id not in (select cid from diary_video where stat_date='${stat_date}')
|and cid_id not in (select cid from eagle.diary_video where stat_date='${stat_date}')
|and stat_date='${stat_date}'
""".stripMargin
)
......@@ -123,7 +123,7 @@ object WeafareStat {
val video_count = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(distinct(cid)) as video_count
|from diary_video where stat_date='${stat_date}'
|from eagle.diary_video where stat_date='${stat_date}'
""".stripMargin
)
// video_count.show()
......@@ -135,7 +135,7 @@ object WeafareStat {
|where partition_date='${partition_date}'
|and page_name='welfare_detail'
|and referrer='diary_detail'
|and referrer_id in (select distinct(diary_id) from feed_diary_boost)
|and referrer_id in (select distinct(diary_id) from eagle.feed_diary_boost)
""".stripMargin
)
// vlog_meigou_clk_count.show()
......@@ -144,10 +144,10 @@ object WeafareStat {
val vlog_clk_count = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(cid_id) as vlog_clk_num
|from data_feed_click
|from jerry_prod.data_feed_click
|where stat_date='${stat_date}'
|and cid_type = 'diary'
|and cid_id in (select distinct(diary_id) from feed_diary_boost)
|and cid_id in (select distinct(diary_id) from eagle.feed_diary_boost)
""".stripMargin
)
// vlog_clk_count.show()
......@@ -206,7 +206,10 @@ object WeafareStat {
val result5 =result4.withColumn("diary_expoure_meigou_rate",result.col("diary_meigou_count")/result.col("diary_expoure"))
result5.show()
GmeiConfig.writeToJDBCTable(result5, "diary_meigou_cvr", SaveMode.Append)
// GmeiConfig.writeToJDBCTable(result5, "diary_meigou_cvr", SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result5, table="diary_meigou_cvr",SaveMode.Append)
println("写入完成")
sc.stop()
......@@ -246,17 +249,17 @@ object NdDataInput {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
val ti = new TiContext(sc)
ti.tidbMapTable(dbName = "jerry_prod", tableName = "nd_data_meigou_cid")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
ti.tidbMapTable(dbName = "eagle", tableName = "feed_diary_boost")
// val ti = new TiContext(sc)
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "nd_data_meigou_cid")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "eagle", tableName = "feed_diary_boost")
val date8 = GmeiConfig.getMinusNDate(70)
val result00 = sc.sql(
s"""
|SELECT
| split(service_id,'\\\\|')[1] as sid,split(cid,'\\\\|')[1] as cid
|FROM nd_data_meigou_cid
|FROM jerry_prod.nd_data_meigou_cid
|where stat_date > '${date8}'
""".stripMargin
)
......@@ -337,10 +340,10 @@ object ServiceStat {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
val ti = new TiContext(sc)
ti.tidbMapTable(dbName = "jerry_prod", tableName = "nd_data_meigou_cid")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
ti.tidbMapTable(dbName = "eagle", tableName = "feed_diary_boost")
// val ti = new TiContext(sc)
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "nd_data_meigou_cid")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "eagle", tableName = "feed_diary_boost")
val result00 = sc.sql(
s"""
......
eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala
View file @ 25b39fd1
......@@ -57,10 +57,6 @@ object data_feed_exposure_precise {
//println(param.date)
val partition_date = stat_date.replace("-","")
// sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/brickhouse-0.7.1-SNAPSHOT.jar")
// sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar")
// sc.sql("CREATE TEMPORARY FUNCTION json_map AS 'brickhouse.udf.json.JsonMapUDF'")
// sc.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJsonFormatCheck'")
val result01=sc.sql(
s"""
......@@ -414,8 +410,12 @@ object data_feed_exposure_precise {
""".stripMargin
)
GmeiConfig.writeToJDBCTable(result, table = "data_feed_exposure_precise", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="data_feed_exposure_precise",SaveMode.Append)
// GmeiConfig.writeToJDBCTable(result, table = "data_feed_exposure_precise", SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="data_feed_exposure_precise",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="data_feed_exposure_precise",SaveMode.Append)
println("写入完成")
}
......
eda/feededa/src/main/scala/com/gmei/find_bug.scala
View file @ 25b39fd1
......@@ -4,7 +4,7 @@ import java.io.Serializable
import com.gmei.WeafareStat.{defaultParams, parser}
import org.apache.spark.sql.{SaveMode}
//import org.apache.spark.sql.{SaveMode, TiContext}
//import org.apache.spark.sql.{SaveMode}
import org.apache.log4j.{Level, Logger}
import scopt.OptionParser
import com.gmei.lib.AbstractParams
......@@ -128,7 +128,12 @@ object find_bug {
result1.show()
// GmeiConfig.writeToJDBCTable(result1, "bug_Recommendation_strategy_temp", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_Recommendation_strategy_temp",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_Recommendation_strategy_temp",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="bug_Recommendation_strategy_temp",SaveMode.Append)
println("写入完成")
......@@ -221,7 +226,10 @@ object find_bug {
.join(device_all_cover,"stat_date")
result2.show()
GmeiConfig.writeToJDBCTable(result2, "bug_strategy_other", SaveMode.Append)
// GmeiConfig.writeToJDBCTable(result2, "bug_strategy_other", SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result2, table="bug_strategy_other",SaveMode.Append)
println("写入完成")
//统计新用户点击率
......@@ -295,7 +303,11 @@ object find_bug {
result3.show()
// GmeiConfig.writeToJDBCTable(result3, "bug_Recommendation_strategy_newUser", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_Recommendation_strategy_newUser",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_Recommendation_strategy_newUser",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="bug_Recommendation_strategy_newUser",SaveMode.Append)
println("写入完成")
}
......@@ -427,7 +439,11 @@ object CTR_precise {
.join(imp_count_oldUser_all,"stat_date")
result1.show()
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_precise_Recommendation_strategy_temp",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="bug_precise_Recommendation_strategy_temp",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="bug_precise_Recommendation_strategy_temp",SaveMode.Append)
println("写入完成")
// GmeiConfig.writeToJDBCTable(result1, "bug_precise_Recommendation_strategy_temp", SaveMode.Append)
......@@ -502,7 +518,11 @@ object CTR_precise {
result3.show()
// GmeiConfig.writeToJDBCTable(result3, "bug_precise_Recommendation_strategy_newUser", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_precise_Recommendation_strategy_newUser",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="bug_precise_Recommendation_strategy_newUser",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="bug_precise_Recommendation_strategy_newUser",SaveMode.Append)
println("写入完成")
}
......
eda/feededa/src/main/scala/com/gmei/strategy_other.scala
View file @ 25b39fd1
......@@ -3,7 +3,7 @@ package com.gmei
import java.io.Serializable
import com.gmei.WeafareStat.{defaultParams, parser}
import org.apache.spark.sql.{DataFrame, SaveMode, TiContext}
import org.apache.spark.sql.{DataFrame, SaveMode}
import org.apache.log4j.{Level, Logger}
import scopt.OptionParser
import com.gmei.lib.AbstractParams
......@@ -46,13 +46,13 @@ object strategy_other {
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
val ti = new TiContext(sc)
ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
// val ti = new TiContext(sc)
sc.sql("use jerry_prod")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
import sc.implicits._
......@@ -177,11 +177,11 @@ object diary_exposure {
GmeiConfig.setup(param.env)
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
val ti = new TiContext(sc)
ti.tidbMapTable(dbName = "eagle", tableName = "src_mimas_prod_api_diary")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
// val ti = new TiContext(sc)
// ti.tidbMapTable(dbName = "eagle", tableName = "src_mimas_prod_api_diary")
//// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
// ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
val mimas_url ="jdbc:mysql://rr-m5et21lafq1677pid.mysql.rds.aliyuncs.com/mimas_prod"
val mimas_user = "mimas"
......
eda/feededa/src/main/scala/com/gmei/temp_analysis.scala
View file @ 25b39fd1
......@@ -59,6 +59,7 @@ object temp_analysis {
import sc.implicits._
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date=param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
......@@ -638,7 +639,11 @@ object alpha_ctr {
// GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="alpha_ctr",SaveMode.Append)
println("写入完成")
......@@ -663,7 +668,11 @@ object alpha_ctr {
val result3=device_num_count.join(duration_device,"stat_date")
// GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result3, table="alpha_duration",SaveMode.Append)
println("写入完成")
......
eda/feededa/src/main/scala/com/gmei/temp_count.scala
View file @ 25b39fd1
......@@ -166,7 +166,10 @@ object temp_count {
// GmeiConfig.writeToJDBCTable(result1, "ffm_diary_ctr", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="ffm_diary_ctr",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="ffm_diary_ctr",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="ffm_diary_ctr",SaveMode.Append)
println("写入完成")
......@@ -338,7 +341,11 @@ object Repeated_content_recommendation {
val result2=List((stat_date,more_than2_old,all_old,more_than2_new,all_new))
val df2 = sc.createDataFrame(result2).toDF("stat_date","old_rep_count","old_imp_all","new_rep_count","new_imp_all")
GmeiConfig.writeToJDBCTable(df2, table = "Repeated_evaluation_indicator", SaveMode.Append)
// GmeiConfig.writeToJDBCTable(df2, table = "Repeated_evaluation_indicator", SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df2, table="Repeated_evaluation_indicator",SaveMode.Append)
println("写入完成")
// val exp_diary_old = sc.sql(
// s"""
......@@ -464,7 +471,11 @@ object Repeated_content_recommendation_moreday {
val df_result = sc.createDataFrame(result)
// GmeiConfig.writeToJDBCTable(df_result, table = "Repeated_content_recommendation_moreday", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="Repeated_content_recommendation_moreday",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="Repeated_content_recommendation_moreday",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df_result, table="Repeated_content_recommendation_moreday",SaveMode.Append)
println("写入完成")
// exp_diary.show()
......@@ -634,7 +645,11 @@ object GetHiveSearchData {
//
// GmeiConfig.writeToJDBCTable(df_result, table = "GetHiveSearchData_CTR", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="GetHiveSearchData_CTR",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",df_result, table="GetHiveSearchData_CTR",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",df_result, table="GetHiveSearchData_CTR",SaveMode.Append)
println("写入完成")
}
......
eda/feededa/src/main/scala/com/gmei/testt.scala
View file @ 25b39fd1
......@@ -8,7 +8,6 @@ import org.apache.spark.sql.{SaveMode}
import org.apache.log4j.{Level, Logger}
import scopt.OptionParser
import com.gmei.lib.AbstractParams
import com.gmei.GmeiConfig.{getMinusNDate, writeToJDBCTable}
object testt {
......@@ -403,7 +402,11 @@ object testt {
.join(log_oldUser,"stat_date")
// GmeiConfig.writeToJDBCTable(result, "diary_meigou_crv", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="diary_meigou_crv",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="diary_meigou_crv",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="diary_meigou_crv",SaveMode.Append)
println("写入完成")
......@@ -770,7 +773,12 @@ object diary_clk_card {
result1.show()
// GmeiConfig.writeToJDBCTable(result1, "on_click_diary_card", SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="on_click_diary_card",SaveMode.Append)
// GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result1, table="on_click_diary_card",SaveMode.Append)
println("开始写入")
GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result1, table="on_click_diary_card",SaveMode.Append)
println("写入完成")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment