Commit 2ad622f6 authored by 王志伟's avatar 王志伟

更改时间

parent 8b35aba5
...@@ -601,6 +601,8 @@ object icon_train_data { ...@@ -601,6 +601,8 @@ object icon_train_data {
} }
object tag_value { object tag_value {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN) Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
...@@ -653,30 +655,33 @@ object tag_value { ...@@ -653,30 +655,33 @@ object tag_value {
//level_id到权重映射 //level_id到权重映射
val level1_id=sc.sql( val level1_id=sc.sql(
s""" s"""
|select distinct(level1_id) as level_id,'${t1}' as value |select distinct(tag_id) as level_id,'${t1}' as value
|from online.bl_tag_hierarchy_detail |from online.bl_tag_hierarchy_detail
|where partition_date = '${partition_date}' |where partition_date = '${partition_date}'
|and level1_id !='0' |and tag_id !=0
|and tag_type=1
""".stripMargin """.stripMargin
) )
level1_id.createOrReplaceTempView("level1_id") level1_id.createOrReplaceTempView("level1_id")
val level2_id=sc.sql( val level2_id=sc.sql(
s""" s"""
|select distinct(level2_id) as level_id,'${t2}' as value |select distinct(tag_id) as level_id,'${t2}' as value
|from online.bl_tag_hierarchy_detail |from online.bl_tag_hierarchy_detail
|where partition_date = '${partition_date}' |where partition_date = '${partition_date}'
|and level2_id !='0' |and tag_id !=0
|and tag_type=2
""".stripMargin """.stripMargin
) )
level2_id.createOrReplaceTempView("level2_id") level2_id.createOrReplaceTempView("level2_id")
val level3_id=sc.sql( val level3_id=sc.sql(
s""" s"""
|select distinct(level3_id) as level_id,'${t3}' as value |select distinct(tag_id) as level_id,'${t3}' as value
|from online.bl_tag_hierarchy_detail |from online.bl_tag_hierarchy_detail
|where partition_date = '${partition_date}' |where partition_date = '${partition_date}'
|and level3_id !='0' |and tag_id !=0
|and tag_type=3
""".stripMargin """.stripMargin
) )
level3_id.createOrReplaceTempView("level3_id") level3_id.createOrReplaceTempView("level3_id")
...@@ -730,7 +735,7 @@ object tag_value { ...@@ -730,7 +735,7 @@ object tag_value {
// temp1.createOrReplaceTempView("temp1") // temp1.createOrReplaceTempView("temp1")
// import implicit_ // import implicit_
val result = temp1.select("level_id").distinct().rdd.map{x => x.toString().substring(1,x.toString().length - 1)}.zipWithIndex().toDF("level_id","index") val result = temp1.select("tag_id").distinct().rdd.map{x => x.toString().substring(1,x.toString().length - 1)}.zipWithIndex().toDF("level_id","index")
val test=result.select(result.col("level_id").cast(DoubleType).as("level_id"),result.col("index").cast(DoubleType).as("index")) val test=result.select(result.col("level_id").cast(DoubleType).as("level_id"),result.col("index").cast(DoubleType).as("index"))
// val resDf = spark.createDataFrame(rowRdd) // val resDf = spark.createDataFrame(rowRdd)
...@@ -761,7 +766,7 @@ object tag_value { ...@@ -761,7 +766,7 @@ object tag_value {
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",test, table="tag_level_index",SaveMode.Overwrite) GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",test, table="tag_level_index",SaveMode.Overwrite)
println("写入完成") println("写入完成")
// zhengxing库里面的数据同步到jerry_prod
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment