Commit 7832deda authored by 王志伟's avatar 王志伟

返回到29号的版本,git版本回退有问题,所以采用这个方法

parent 3b315a1f
package com.gmei package com.gmei
import java.io.Serializable import java.io.Serializable
import java.text.SimpleDateFormat
import java.util.Calendar
import com.gmei.WeafareStat.{defaultParams, parser} import com.gmei.WeafareStat.{defaultParams, parser}
import org.apache.spark.sql.{SaveMode, TiContext} import org.apache.spark.sql.{SaveMode, TiContext}
...@@ -10,15 +8,13 @@ import org.apache.log4j.{Level, Logger} ...@@ -10,15 +8,13 @@ import org.apache.log4j.{Level, Logger}
import scopt.OptionParser import scopt.OptionParser
import com.gmei.lib.AbstractParams import com.gmei.lib.AbstractParams
import scala.collection.mutable.ArrayBuffer
object strategy_clk_imp_oldUser { object strategy_clk_imp_oldUser {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN) Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF) Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
case class Params(env: String = "dev" case class Params(env: String = "dev",
//date: String = "2018-08-01" date: String = "2018-08-01"
) extends AbstractParams[Params] with Serializable ) extends AbstractParams[Params] with Serializable
val defaultParams = Params() val defaultParams = Params()
...@@ -28,9 +24,9 @@ object strategy_clk_imp_oldUser { ...@@ -28,9 +24,9 @@ object strategy_clk_imp_oldUser {
opt[String]("env") opt[String]("env")
.text(s"the databases environment you used") .text(s"the databases environment you used")
.action((x, c) => c.copy(env = x)) .action((x, c) => c.copy(env = x))
// opt[String] ("date") opt[String] ("date")
// .text(s"the date you used") .text(s"the date you used")
//.action((x,c) => c.copy(date = x)) .action((x,c) => c.copy(date = x))
note( note(
""" """
|For example, the following command runs this app on a tidb dataset: |For example, the following command runs this app on a tidb dataset:
...@@ -41,28 +37,6 @@ object strategy_clk_imp_oldUser { ...@@ -41,28 +37,6 @@ object strategy_clk_imp_oldUser {
) )
} }
//获得时间array
def get_date(): ArrayBuffer[String] ={
val startTime = "2018-10-07"
val endTime = "2017-10-10"
val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
val dateFiled = Calendar.DAY_OF_MONTH
var beginDate = dateFormat.parse(startTime)
val endDate = dateFormat.parse(endTime)
val calendar = Calendar.getInstance()
calendar.setTime(beginDate)
val dateArray: ArrayBuffer[String] = ArrayBuffer()
while (beginDate.compareTo(endDate) <= 0) {
dateArray += dateFormat.format(beginDate)
calendar.add(dateFiled, 1)
beginDate = calendar.getTime
}
// println(dateArray)
dateArray
}
def main(args: Array[String]): Unit = { def main(args: Array[String]): Unit = {
parser.parse(args, defaultParams).map { param => parser.parse(args, defaultParams).map { param =>
GmeiConfig.setup(param.env) GmeiConfig.setup(param.env)
...@@ -78,106 +52,102 @@ object strategy_clk_imp_oldUser { ...@@ -78,106 +52,102 @@ object strategy_clk_imp_oldUser {
import sc.implicits._ import sc.implicits._
// val stat_date = GmeiConfig.getMinusNDate(1) // val stat_date = GmeiConfig.getMinusNDate(1)
//println(param.date) println(param.date)
val date_array = get_date() val partition_date = param.date.replace("-","")
date_array.foreach(println) val decive_id_oldUser = sc.sql(
for (elem <- date_array) { s"""
val partition_date = elem.replace("-","") |select distinct(device_id) as decive_id
println(partition_date) |from online.ml_device_day_active_status
val decive_id_oldUser = sc.sql( |where active_type = '4'
|and partition_date ='${partition_date}'
""".stripMargin
)
decive_id_oldUser.show()
decive_id_oldUser.createOrReplaceTempView("device_id_old")
val decive_id_newUser = sc.sql(
s"""
|select distinct(device_id) as decive_id
|from online.ml_device_day_active_status
|where active_type != '4'
|and partition_date ='${partition_date}'
""".stripMargin
)
decive_id_newUser.show()
decive_id_newUser.createOrReplaceTempView("device_id_newUser")
val strategies = Seq("[1|2]$","[3|4]$","[5|6]$","[7|8]$")
for (strategy <- strategies){
val clk_count_oldUser = sc.sql(
s""" s"""
|select distinct(device_id) as decive_id |select '${param.date}' as stat_date, count(cid_id) as get_clk_count_old
|from online.ml_device_day_active_status |from data_feed_click jd inner join device_id_old
|where active_type = '4' |on jd.device_id = device_id_old.decive_id
|and partition_date ='${partition_date}' |where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id regexp'${strategy}'
|and jd.device_id not in (select device_id from bl_device_list)
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${param.date}'
""".stripMargin """.stripMargin
) )
decive_id_oldUser.show()
decive_id_oldUser.createOrReplaceTempView("device_id_old")
val decive_id_newUser = sc.sql( val imp_count_oldUser = sc.sql(
s""" s"""
|select distinct(device_id) as decive_id |select '${param.date}' as stat_date, count(cid_id) as get_imp_count_old
|from online.ml_device_day_active_status |from data_feed_exposure je inner join device_id_old
|where active_type != '4' |on je.device_id = device_id_old.decive_id
|and partition_date ='${partition_date}' |where je.cid_type = 'diary'
|and je.device_id regexp'${strategy}'
|and je.device_id not in (select device_id from bl_device_list)
|and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${param.date}'
""".stripMargin """.stripMargin
) )
decive_id_newUser.show()
decive_id_newUser.createOrReplaceTempView("device_id_newUser")
val clk_count_newUser = sc.sql(
val strategies = Seq("[1|2]$","[3|4]$","[5|6]$","[7|8]$") s"""
for (strategy <- strategies){ |select '${param.date}' as stat_date, count(cid_id) as get_clk_count_newUser
println(strategy) |from data_feed_click jd inner join device_id_newUser
val clk_count_oldUser = sc.sql( |on jd.device_id = device_id_newUser.decive_id
s""" |where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|select '${elem}' as stat_date, count(cid_id) as get_clk_count_old |and jd.device_id regexp'${strategy}'
|from data_feed_click jd inner join device_id_old |and jd.device_id not in (select device_id from bl_device_list)
|on jd.device_id = device_id_old.decive_id |and jd.device_id not in (select device_id from blacklist)
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video') |and jd.stat_date ='${param.date}'
|and jd.device_id regexp'${strategy}'
|and jd.device_id not in (select device_id from bl_device_list)
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${elem}'
""".stripMargin
)
val imp_count_oldUser = sc.sql(
s"""
|select '${elem}' as stat_date, count(cid_id) as get_imp_count_old
|from data_feed_exposure je inner join device_id_old
|on je.device_id = device_id_old.decive_id
|where je.cid_type = 'diary'
|and je.device_id regexp'${strategy}'
|and je.device_id not in (select device_id from bl_device_list)
|and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${elem}'
""".stripMargin
)
val clk_count_newUser = sc.sql(
s"""
|select '${elem}' as stat_date, count(cid_id) as get_clk_count_newUser
|from data_feed_click jd inner join device_id_newUser
|on jd.device_id = device_id_newUser.decive_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id regexp'${strategy}'
|and jd.device_id not in (select device_id from bl_device_list)
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${elem}'
""".stripMargin """.stripMargin
) )
val imp_count_newUser = sc.sql( val imp_count_newUser = sc.sql(
s""" s"""
|select '${elem}' as stat_date, count(cid_id) as get_imp_count_newUser |select '${param.date}' as stat_date, count(cid_id) as get_imp_count_newUser
|from data_feed_exposure je inner join device_id_newUser |from data_feed_exposure je inner join device_id_newUser
|on je.device_id = device_id_newUser.decive_id |on je.device_id = device_id_newUser.decive_id
|where je.cid_type = 'diary' |where je.cid_type = 'diary'
|and je.device_id regexp'${strategy}' |and je.device_id regexp'${strategy}'
|and je.device_id not in (select device_id from bl_device_list) |and je.device_id not in (select device_id from bl_device_list)
|and je.device_id not in (select device_id from blacklist) |and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${elem}' |and je.stat_date ='${param.date}'
""".stripMargin """.stripMargin
) )
imp_count_newUser.show() imp_count_newUser.show()
val result = clk_count_oldUser.join(imp_count_oldUser,"stat_date") val result = clk_count_oldUser.join(imp_count_oldUser,"stat_date")
.join(clk_count_newUser,"stat_date") .join(clk_count_newUser,"stat_date")
.join(imp_count_newUser,"stat_date") .join(imp_count_newUser,"stat_date")
result.show() result.show()
}
} }
} }
} }
} }
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment