Recommendation_strategy_all.scala 19.5 KB
Newer Older
王志伟's avatar
王志伟 committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
package com.gmei

import java.io.Serializable

import com.gmei.WeafareStat.{defaultParams, parser}
import org.apache.spark.sql.{SaveMode, TiContext}
import org.apache.log4j.{Level, Logger}
import scopt.OptionParser
import com.gmei.lib.AbstractParams

object Recommendation_strategy_all {

  Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
  Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)

王志伟's avatar
王志伟 committed
16
  case class Params(env: String = "dev"
王志伟's avatar
王志伟 committed
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
                   ) extends AbstractParams[Params] with Serializable

  val defaultParams = Params()

  val parser = new OptionParser[Params]("Feed_EDA") {
    head("WeafareStat")
    opt[String]("env")
      .text(s"the databases environment you used")
      .action((x, c) => c.copy(env = x))
    note(
      """
        |For example, the following command runs this app on a tidb dataset:
        |
        | spark-submit --class com.gmei.WeafareStat ./target/scala-2.11/feededa-assembly-0.1.jar \
      """.stripMargin +
        s"|   --env ${defaultParams.env}"
    )
  }

  def main(args: Array[String]): Unit = {
    parser.parse(args, defaultParams).map { param =>
      GmeiConfig.setup(param.env)
      val spark_env = GmeiConfig.getSparkSession()
      val sc = spark_env._2

      val ti = new TiContext(sc)
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "diary_video")
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
      ti.tidbMapTable(dbName = "jerry_test", tableName = "bl_device_list")
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
王志伟's avatar
王志伟 committed
48
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
王志伟's avatar
王志伟 committed
49 50 51


      import sc.implicits._
王志伟's avatar
王志伟 committed
52
      val stat_date = GmeiConfig.getMinusNDate(1)
王志伟's avatar
王志伟 committed
53
      println(stat_date)
王志伟's avatar
王志伟 committed
54
      //println(param.date)
王志伟's avatar
王志伟 committed
55
      val partition_date = stat_date.replace("-","")
王志伟's avatar
王志伟 committed
56 57
      val decive_id_oldUser = sc.sql(
        s"""
王志伟's avatar
王志伟 committed
58
           |select distinct(device_id) as device_id
王志伟's avatar
王志伟 committed
59 60 61 62 63 64 65 66 67 68 69 70 71 72
           |from online.ml_device_day_active_status
           |where active_type = '4'
           |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
           |        ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
           |        ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
           |        ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
           |        ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
           |        ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
           |        ,'promotion_shike','promotion_julang_jl03')
           |and partition_date ='${partition_date}'
         """.stripMargin
      )
      decive_id_oldUser.createOrReplaceTempView("device_id_old")

王志伟's avatar
王志伟 committed
73

王志伟's avatar
王志伟 committed
74
      val clk_count_oldUser_Contrast = sc.sql(
王志伟's avatar
王志伟 committed
75
        s"""
王志伟's avatar
王志伟 committed
76
           |select '${stat_date}' as stat_date, count(cid_id) as clk_count_oldUser_Contrast
王志伟's avatar
王志伟 committed
77
           |from data_feed_click jd inner join device_id_old
王志伟's avatar
王志伟 committed
78
           |on jd.device_id = device_id_old.device_id
王志伟's avatar
王志伟 committed
79 80 81 82
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id regexp'1$$'
           |and jd.device_id not in (select device_id from bl_device_list)
           |and jd.device_id not in (select device_id from blacklist)
王志伟's avatar
王志伟 committed
83
           |and jd.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
84
         """.stripMargin
王志伟's avatar
王志伟 committed
85 86
        )

王志伟's avatar
王志伟 committed
87
      val imp_count_oldUser_Contrast = sc.sql(
王志伟's avatar
王志伟 committed
88
        s"""
王志伟's avatar
王志伟 committed
89
           |select '${stat_date}' as stat_date, count(cid_id) as imp_count_oldUser_Contrast
王志伟's avatar
王志伟 committed
90
           |from data_feed_exposure je inner join device_id_old
王志伟's avatar
王志伟 committed
91
           |on je.device_id = device_id_old.device_id
王志伟's avatar
王志伟 committed
92 93 94 95
           |where je.cid_type = 'diary'
           |and je.device_id regexp'1$$'
           |and je.device_id not in (select device_id from bl_device_list)
           |and je.device_id not in (select device_id from blacklist)
王志伟's avatar
王志伟 committed
96
           |and je.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
97
         """.stripMargin
王志伟's avatar
王志伟 committed
98 99
        )

王志伟's avatar
王志伟 committed
100 101
      val clk_count_oldUser_all = sc.sql(
        s"""
王志伟's avatar
王志伟 committed
102
           |select '${stat_date}' as stat_date, count(cid_id) as clk_count_oldUser_all
王志伟's avatar
王志伟 committed
103
           |from data_feed_click jd inner join device_id_old
王志伟's avatar
王志伟 committed
104
           |on jd.device_id = device_id_old.device_id
王志伟's avatar
王志伟 committed
105 106 107
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id not in (select device_id from bl_device_list)
           |and jd.device_id not in (select device_id from blacklist)
王志伟's avatar
王志伟 committed
108
           |and jd.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
109 110 111
         """.stripMargin
      )

王志伟's avatar
王志伟 committed
112 113
      val imp_count_oldUser_all = sc.sql(
        s"""
王志伟's avatar
王志伟 committed
114
           |select '${stat_date}' as stat_date, count(cid_id) as imp_count_oldUser_all
王志伟's avatar
王志伟 committed
115
           |from data_feed_exposure je inner join device_id_old
王志伟's avatar
王志伟 committed
116
           |on je.device_id = device_id_old.device_id
王志伟's avatar
王志伟 committed
117 118 119
           |where je.cid_type = 'diary'
           |and je.device_id not in (select device_id from bl_device_list)
           |and je.device_id not in (select device_id from blacklist)
王志伟's avatar
王志伟 committed
120
           |and je.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
121 122 123
         """.stripMargin
      )

王志伟's avatar
王志伟 committed
124
//获取策略命中用户device_id
王志伟's avatar
王志伟 committed
125 126 127 128 129 130 131 132 133 134 135
      val device_id_cover = sc.sql(
        s"""
           |select distinct(device_id) as device_id
           |from merge_queue_table
         """.stripMargin
      )
      device_id_cover.createOrReplaceTempView("device_id_cover_older")


      val clk_count_oldUser_Cover = sc.sql(
        s"""
王志伟's avatar
王志伟 committed
136
           |select '${stat_date}' as stat_date, count(cid_id) as clk_count_oldUser_Cover
王志伟's avatar
王志伟 committed
137 138 139 140 141
           |from data_feed_click jd inner join device_id_cover_older
           |on jd.device_id = device_id_cover_older.device_id
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id not in (select device_id from bl_device_list)
           |and jd.device_id not in (select device_id from blacklist)
王志伟's avatar
王志伟 committed
142
           |and jd.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
143 144 145 146 147
         """.stripMargin
      )

      val imp_count_oldUser_Cover = sc.sql(
        s"""
王志伟's avatar
王志伟 committed
148
           |select '${stat_date}' as stat_date, count(cid_id) as imp_count_oldUser_Cover
王志伟's avatar
王志伟 committed
149 150 151 152 153
           |from data_feed_exposure je inner join device_id_cover_older
           |on je.device_id = device_id_cover_older.device_id
           |where je.cid_type = 'diary'
           |and je.device_id not in (select device_id from bl_device_list)
           |and je.device_id not in (select device_id from blacklist)
王志伟's avatar
王志伟 committed
154
           |and je.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
155 156
         """.stripMargin
      )
王志伟's avatar
王志伟 committed
157 158 159 160 161 162 163 164 165 166 167 168 169
      //策略覆盖用户数device_num_cover
      val device_num_cover = sc.sql(
        s"""
           |select '${stat_date}' as stat_date,count(distinct(device_id)) as device_num_cover
           |from merge_queue_table
         """.stripMargin
      )

      //尾号1活跃用户数device_num_1_hit
      val device_num_1_hit = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(distinct(je.device_id)) as device_num_1_hit
           |from data_feed_exposure je inner join device_id_old
王志伟's avatar
王志伟 committed
170
           |on je.device_id = device_id_old.device_id
王志伟's avatar
王志伟 committed
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
           |where je.cid_type = 'diary'
           |and je.device_id regexp'1$$'
           |and je.device_id not in (select device_id from bl_device_list)
           |and je.device_id not in (select device_id from blacklist)
           |and je.stat_date ='${stat_date}'
         """.stripMargin
      )

      //策略命中用户数device_num_hit
      val device_num_hit = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(distinct(je.device_id)) as device_num_hit
           |from data_feed_exposure je inner join device_id_cover_older
           |on je.device_id = device_id_cover_older.device_id
           |where je.cid_type = 'diary'
           |and je.device_id not in (select device_id from bl_device_list)
           |and je.device_id not in (select device_id from blacklist)
           |and je.stat_date ='${stat_date}'
         """.stripMargin
      )
191
      val result1 = clk_count_oldUser_Contrast.join(imp_count_oldUser_Contrast,"stat_date")
王志伟's avatar
王志伟 committed
192
          .join(clk_count_oldUser_all,"stat_date")
王志伟's avatar
王志伟 committed
193
          .join(imp_count_oldUser_all,"stat_date")
王志伟's avatar
王志伟 committed
194 195
          .join(clk_count_oldUser_Cover,"stat_date")
          .join(imp_count_oldUser_Cover,"stat_date")
王志伟's avatar
王志伟 committed
196 197 198
          .join(device_num_cover,"stat_date")
          .join(device_num_1_hit,"stat_date")
          .join(device_num_hit,"stat_date")
199 200
      result1.show()

王志伟's avatar
王志伟 committed
201
      GmeiConfig.writeToJDBCTable(result1, "Recommendation_strategy_temp", SaveMode.Append)
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224


      //device_id尾号1有点击用户日记本点击数
      val clk_active_1 = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(jd.cid_id) as clk_active_1
           |from data_feed_click jd inner join device_id_old
           |on jd.device_id = device_id_old.device_id
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id regexp'1$$'
           |and jd.device_id not in (select device_id from bl_device_list)
           |and jd.device_id not in (select device_id from blacklist)
           |and jd.stat_date ='${stat_date}'
         """.stripMargin
      )

      //device_id尾号1有点击用户日记本曝光数
      val imp_active_1 = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(je.cid_id) as imp_active_1
           |from data_feed_exposure je inner join device_id_old
           |on je.device_id = device_id_old.device_id
           |where je.cid_type = 'diary'
225
           |and je.device_id in (select distinct(device_id) from data_feed_click where device_id regexp '1$$' and stat_date = '${stat_date}')
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
           |and je.device_id not in (select device_id from bl_device_list)
           |and je.device_id not in (select device_id from blacklist)
           |and je.stat_date ='${stat_date}'
         """.stripMargin
      )

      //device_id尾号1点击日记本用户数
      val clk_diary_device = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(distinct(jd.device_id)) as clk_diary_device
           |from data_feed_click jd inner join device_id_old
           |on jd.device_id = device_id_old.device_id
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id regexp'1$$'
           |and jd.device_id not in (select device_id from bl_device_list)
           |and jd.device_id not in (select device_id from blacklist)
           |and jd.stat_date ='${stat_date}'
            """.stripMargin
      )

      //所有有点击用户日记本点击数
      val clk_active_all = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(jd.cid_id) as clk_active_all
           |from data_feed_click jd inner join device_id_old
           |on jd.device_id = device_id_old.device_id
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id not in (select device_id from bl_device_list)
           |and jd.device_id not in (select device_id from blacklist)
           |and jd.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
256
          """.stripMargin
257 258 259 260 261 262 263 264 265 266 267 268 269
      )

      //所有有点击用户日记本曝光数
      val imp_active_all = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(je.cid_id) as imp_active_all
           |from data_feed_exposure je inner join device_id_old
           |on je.device_id = device_id_old.device_id
           |where je.cid_type = 'diary'
           |and je.device_id in (select distinct(device_id) from data_feed_click where stat_date = '${stat_date}')
           |and je.device_id not in (select device_id from bl_device_list)
           |and je.device_id not in (select device_id from blacklist)
           |and je.stat_date ='${stat_date}'
王志伟's avatar
王志伟 committed
270
          """.stripMargin
271 272 273 274 275 276 277 278
      )

      //策略命中用户点击日记本用户数
      val clk_diary_device_cover = sc.sql(
        s"""
           |select '${stat_date}' as stat_date,count(distinct(device_id)) as clk_diary_device_cover
           |from merge_queue_table
           |where device_id in (select distinct(device_id) from data_feed_click where stat_date = '${stat_date}')
王志伟's avatar
王志伟 committed
279
          """.stripMargin
280 281 282 283 284 285 286
      )

      //策略命中用户总数
      val device_all_cover = sc.sql(
        s"""
           |select '${stat_date}' as stat_date,count(distinct(device_id)) as device_all_cover
           |from merge_queue_table
王志伟's avatar
王志伟 committed
287
          """.stripMargin
288 289 290 291 292 293 294 295 296 297 298 299
      )

      val result2 = clk_active_1.join(imp_active_1,"stat_date")
        .join(clk_active_all,"stat_date")
        .join(imp_active_all,"stat_date")
        .join(clk_diary_device,"stat_date")
        .join(clk_diary_device_cover,"stat_date")
        .join(device_all_cover,"stat_date")
      result2.show()

      GmeiConfig.writeToJDBCTable(result2, "strategy_other", SaveMode.Append)

王志伟's avatar
王志伟 committed
300

王志伟's avatar
王志伟 committed
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
//统计新用户点击率
      val devicee_id_newUser = sc.sql(
        s"""
           |select distinct(device_id) as device_id
           |from online.ml_device_day_active_status
           |where active_type != '4'
           |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
           |        ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
           |        ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
           |        ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
           |        ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
           |        ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
           |        ,'promotion_shike','promotion_julang_jl03')
           |and partition_date ='${partition_date}'
         """.stripMargin
      )
      devicee_id_newUser.show()
      devicee_id_newUser.createOrReplaceTempView("device_id_new")

      val clk_count_newUser_Contrast = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(cid_id) as clk_count_newUser_Contrast
           |from data_feed_click jd inner join device_id_new
           |on jd.device_id = device_id_new.device_id
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id regexp'1$$'
           |and jd.device_id not in (select device_id from blacklist)
           |and jd.stat_date ='${stat_date}'
         """.stripMargin
      )

      val imp_count_newUser_Contrast = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(cid_id) as imp_count_newUser_Contrast
           |from data_feed_exposure je inner join device_id_new
           |on je.device_id = device_id_new.device_id
           |where je.cid_type = 'diary'
           |and je.device_id regexp'1$$'
           |and je.device_id not in (select device_id from blacklist)
           |and je.stat_date ='${stat_date}'
         """.stripMargin
      )

      val clk_count_newUser_all = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(cid_id) as clk_count_newUser_all
           |from data_feed_click jd inner join device_id_new
           |on jd.device_id = device_id_new.device_id
           |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
           |and jd.device_id not in (select device_id from blacklist)
           |and jd.stat_date ='${stat_date}'
         """.stripMargin
      )

      val imp_count_newUser_all = sc.sql(
        s"""
           |select '${stat_date}' as stat_date, count(cid_id) as imp_count_newUser_all
           |from data_feed_exposure je inner join device_id_new
           |on je.device_id = device_id_new.device_id
           |where je.cid_type = 'diary'
           |and je.device_id not in (select device_id from blacklist)
           |and je.stat_date ='${stat_date}'
         """.stripMargin
      )

      val result3 = clk_count_newUser_Contrast.join(imp_count_newUser_Contrast,"stat_date")
        .join(clk_count_newUser_all,"stat_date")
        .join(imp_count_newUser_all,"stat_date")
      result3.show()

      GmeiConfig.writeToJDBCTable(result3, "Recommendation_strategy_newUser", SaveMode.Append)

373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
    }


  }

}


//以下内容统计信息熵基尼系数
object Gini_coefficient {

  Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
  Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)

  case class Params(env: String = "dev",
                    date: String = "2018-08-01"
                   ) extends AbstractParams[Params] with Serializable

  val defaultParams = Params()

  val parser = new OptionParser[Params]("Feed_EDA") {
    head("Gini_coefficient")
    opt[String]("env")
      .text(s"the databases environment you used")
      .action((x, c) => c.copy(env = x))
    opt[String] ("date")
      .text(s"the date you used")
      .action((x,c) => c.copy(date = x))
    note(
      """
        |For example, the following command runs this app on a tidb dataset:
        |
        | spark-submit --class com.gmei.WeafareStat ./target/scala-2.11/feededa-assembly-0.1.jar \
      """.stripMargin +
        s"|   --env ${defaultParams.env}"
    )
  }
王志伟's avatar
王志伟 committed
410

411 412 413 414 415
  def main(args: Array[String]): Unit = {
    parser.parse(args, defaultParams).map { param =>
      GmeiConfig.setup(param.env)
      val spark_env = GmeiConfig.getSparkSession()
      val sc = spark_env._2
王志伟's avatar
王志伟 committed
416

417 418 419 420 421
      val ti = new TiContext(sc)
      ti.tidbMapTable(dbName = "eagle", tableName = "src_mimas_prod_api_diary")
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
王志伟's avatar
王志伟 committed
422 423


424 425 426 427
      import sc.implicits._
      val stat_date = GmeiConfig.getMinusNDate(1)
      //println(param.date)
      val partition_date = stat_date.replace("-","")
王志伟's avatar
王志伟 committed
428

429 430 431 432 433
      //获取3.5星及以上日记本id
      val diary_id = sc.sql(
        s"""
           |select id as diary_id
           |from src_mimas_prod_api_diary
434
           |where content_level >=3.5
435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
           |and doctor_id is not null
         """.stripMargin
      )
      diary_id.createOrReplaceTempView("temp1")
      //机构id
      val agency_id = sc.sql(
        s"""
           |SELECT DISTINCT(cl_id) as device_id
           |FROM online.ml_hospital_spam_pv_day
           |WHERE partition_date >= '20180402'
           |AND partition_date <= '20181203'
           |AND pv_ratio >= 0.95
           |UNION ALL
           |SELECT DISTINCT(cl_id) as device_id
           |FROM online.ml_hospital_spam_pv_month
           |WHERE partition_date >= '20171101'
           |AND partition_date <= '20181203'
           |AND pv_ratio >= 0.95
         """.stripMargin
      )
      agency_id.createOrReplaceTempView("agency_id")
//统计次数
      val diary_clk_num = sc.sql(
        s"""
459
           |select temp1.diary_id as diary_id,count(ov.cl_id) as diary_clk_num
460 461 462 463 464 465 466 467 468 469 470
           |from online.tl_hdfs_maidian_view ov left join temp1
           |on params["business_id"]= temp1.diary_id
           |where ov.action="page_view"
           |and ov.params["page_name"]="diary_detail"
           |and temp1.diary_id is not null
           |and (ov.params["out"]-ov.params["in"])<7200
           |and ov.partition_date ='${partition_date}'
           |and ov.cl_id not in (select device_id from blacklist)
           |group by temp1.diary_id
         """.stripMargin
      )
471
      GmeiConfig.writeToJDBCTable(diary_clk_num, "Gini_coefficient", SaveMode.Overwrite)
王志伟's avatar
王志伟 committed
472

王志伟's avatar
王志伟 committed
473 474 475 476 477 478
    }


  }

}
479 480 481