统计icon数据

eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala

 package com.gmei
 
 import java.io.Serializable
+import java.util.Properties
 
 import breeze.linalg.split
 import com.gmei.WeafareStat.{defaultParams, parser}
@@ -1022,7 +1023,7 @@ object app_list_yunying {
 
 
 
-object test_data {
+object icon_data {
 
   Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
   Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
@@ -1065,38 +1066,20 @@ object test_data {
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
+      val properties = new Properties()
+      properties.put("user", "doris")
+      properties.put("password", "o5gbA27hXHHm")
 
-// 百科点击数据
 
-      val wiki_click=sc.sql(
-        s"""
-           |SELECT cl_id as device_id,params['business_id'] as wiki_id
-           |from online.tl_hdfs_maidian_view
-           |where partition_date='${partition_date}'
-           |and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
-       """.stripMargin
-      )
-      wiki_click.createOrReplaceTempView("wiki_click")
+      val device_id=sc
+        .read
+        .option("driver", "com.mysql.jdbc.Driver")
+        .jdbc("jdbc:mysql://172.16.30.136/doris_prod","device_icon_queue",properties)
+        .select("device_id")
+      device_id.show()
 
-      val wiki_tag=sc.sql(
-        s"""
-           |SELECT item_id,tag_id
-           |from online.tl_hdfs_wiki_item_tag_view
-           |where partition_date='${partition_date}'
-       """.stripMargin
-      )
-      wiki_tag.createOrReplaceTempView("wiki_tag")
+      device_id.createTempView("device_id")
 
-      val wiki_device_tag=sc.sql(
-        s"""
-           |SELECT a.device_id,b.tag_id
-           |from wiki_click a
-           |left join wiki_tag b
-           |on a.wiki_id=b.item_id
-           |where b.tag_id is not null
-       """.stripMargin
-      )
-      wiki_device_tag.createOrReplaceTempView("wiki_device_tag")
 
 
     }