Skip to content

F
ffm-baseline
Commit 65110bd4 authored by 王志伟's avatar 王志伟
Browse files
Options

add date

parent 9bbfbfba
Hide whitespace changes
Inline Side-by-side
Showing with 142 additions and 76 deletions
eda/feededa/src/main/scala/com/gmei/strategy_clk_imp_oldUser.scala
View file @ 65110bd4
package com.gmei package com.gmei
import java.io.Serializable import java.io.Serializable
import java.text.SimpleDateFormat
import java.util.Calendar
import com.gmei.WeafareStat.{defaultParams, parser} import com.gmei.WeafareStat.{defaultParams, parser}
import org.apache.spark.sql.{SaveMode, TiContext} import org.apache.spark.sql.{SaveMode, TiContext}
...@@ -8,13 +10,15 @@ import org.apache.log4j.{Level, Logger} ...@@ -8,13 +10,15 @@ import org.apache.log4j.{Level, Logger}
import scopt.OptionParser import scopt.OptionParser
import com.gmei.lib.AbstractParams import com.gmei.lib.AbstractParams
import scala.collection.mutable.ArrayBuffer
object strategy_clk_imp_oldUser { object strategy_clk_imp_oldUser {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN) Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF) Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
case class Params(env: String = "dev", case class Params(env: String = "dev",
date: String = "2018-08-01" //date: String = "2018-08-01"
) extends AbstractParams[Params] with Serializable ) extends AbstractParams[Params] with Serializable
val defaultParams = Params() val defaultParams = Params()
...@@ -53,98 +57,119 @@ object strategy_clk_imp_oldUser { ...@@ -53,98 +57,119 @@ object strategy_clk_imp_oldUser {
import sc.implicits._ import sc.implicits._
// val stat_date = GmeiConfig.getMinusNDate(1) // val stat_date = GmeiConfig.getMinusNDate(1)
println(param.date) //println(param.date)
val partition_date = param.date.replace("-","") //获得时间array
val decive_id_oldUser = sc.sql( def get_date(): ArrayBuffer[String] ={
s""" val startTime = "2018-10-07"
|select distinct(device_id) as decive_id val endTime = "2017-10-10"
|from online.ml_device_day_active_status val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
|where active_type = '4' val dateFiled = Calendar.DAY_OF_MONTH
|and partition_date ='${partition_date}' var beginDate = dateFormat.parse(startTime)
""".stripMargin val endDate = dateFormat.parse(endTime)
) val calendar = Calendar.getInstance()
decive_id_oldUser.show() calendar.setTime(beginDate)
decive_id_oldUser.createOrReplaceTempView("device_id_old") val dateArray: ArrayBuffer[String] = ArrayBuffer()
while (beginDate.compareTo(endDate) <= 0) {
val decive_id_newUser = sc.sql( dateArray += dateFormat.format(beginDate)
s""" calendar.add(dateFiled, 1)
|select distinct(device_id) as decive_id beginDate = calendar.getTime
|from online.ml_device_day_active_status }
|where active_type != '4' //println(dateArray)
|and partition_date ='${partition_date}' dateArray
""".stripMargin }
) val date_array = get_date()
decive_id_newUser.show() for (elem <- date_array) {
decive_id_newUser.createOrReplaceTempView("device_id_newUser")
val strategies = Seq("[1|2]$","[3|4]$","[5|6]$","[7|8]$") val partition_date = elem.replace("-","")
for (strategy <- strategies){ val decive_id_oldUser = sc.sql(
val clk_count_oldUser = sc.sql(
s""" s"""
|select '${param.date}' as stat_date, count(cid_id) as get_clk_count_old |select distinct(device_id) as decive_id
|from data_feed_click jd inner join device_id_old |from online.ml_device_day_active_status
|on jd.device_id = device_id_old.decive_id |where active_type = '4'
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video') |and partition_date ='${partition_date}'
|and jd.device_id regexp'${strategy}'
|and jd.device_id not in (select device_id from bl_device_list)
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${param.date}'
""".stripMargin """.stripMargin
) )
decive_id_oldUser.show()
decive_id_oldUser.createOrReplaceTempView("device_id_old")
val imp_count_oldUser = sc.sql( val decive_id_newUser = sc.sql(
s""" s"""
|select '${param.date}' as stat_date, count(cid_id) as get_imp_count_old |select distinct(device_id) as decive_id
|from data_feed_exposure je inner join device_id_old |from online.ml_device_day_active_status
|on je.device_id = device_id_old.decive_id |where active_type != '4'
|where je.cid_type = 'diary' |and partition_date ='${partition_date}'
|and je.device_id regexp'${strategy}'
|and je.device_id not in (select device_id from bl_device_list)
|and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${param.date}'
""".stripMargin """.stripMargin
) )
decive_id_newUser.show()
decive_id_newUser.createOrReplaceTempView("device_id_newUser")
val clk_count_newUser = sc.sql(
s""" val strategies = Seq("[1|2]$","[3|4]$","[5|6]$","[7|8]$")
|select '${param.date}' as stat_date, count(cid_id) as get_clk_count_newUser for (strategy <- strategies){
|from data_feed_click jd inner join device_id_newUser val clk_count_oldUser = sc.sql(
|on jd.device_id = device_id_newUser.decive_id s"""
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video') |select '${elem}' as stat_date, count(cid_id) as get_clk_count_old
|and jd.device_id regexp'${strategy}' |from data_feed_click jd inner join device_id_old
|and jd.device_id not in (select device_id from bl_device_list) |on jd.device_id = device_id_old.decive_id
|and jd.device_id not in (select device_id from blacklist) |where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.stat_date ='${param.date}' |and jd.device_id regexp'${strategy}'
|and jd.device_id not in (select device_id from bl_device_list)
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${elem}'
""".stripMargin """.stripMargin
) )
val imp_count_newUser = sc.sql( val imp_count_oldUser = sc.sql(
s""" s"""
|select '${param.date}' as stat_date, count(cid_id) as get_imp_count_newUser |select '${elem}' as stat_date, count(cid_id) as get_imp_count_old
|from data_feed_exposure je inner join device_id_newUser |from data_feed_exposure je inner join device_id_old
|on je.device_id = device_id_newUser.decive_id |on je.device_id = device_id_old.decive_id
|where je.cid_type = 'diary' |where je.cid_type = 'diary'
|and je.device_id regexp'${strategy}' |and je.device_id regexp'${strategy}'
|and je.device_id not in (select device_id from bl_device_list) |and je.device_id not in (select device_id from bl_device_list)
|and je.device_id not in (select device_id from blacklist) |and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${param.date}' |and je.stat_date ='${elem}'
""".stripMargin """.stripMargin
) )
imp_count_newUser.show()
val clk_count_newUser = sc.sql(
s"""
|select '${elem}' as stat_date, count(cid_id) as get_clk_count_newUser
|from data_feed_click jd inner join device_id_newUser
|on jd.device_id = device_id_newUser.decive_id
|where (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
|and jd.device_id regexp'${strategy}'
|and jd.device_id not in (select device_id from bl_device_list)
|and jd.device_id not in (select device_id from blacklist)
|and jd.stat_date ='${elem}'
""".stripMargin
)
val imp_count_newUser = sc.sql(
s"""
|select '${elem}' as stat_date, count(cid_id) as get_imp_count_newUser
|from data_feed_exposure je inner join device_id_newUser
|on je.device_id = device_id_newUser.decive_id
|where je.cid_type = 'diary'
|and je.device_id regexp'${strategy}'
|and je.device_id not in (select device_id from bl_device_list)
|and je.device_id not in (select device_id from blacklist)
|and je.stat_date ='${elem}'
""".stripMargin
)
imp_count_newUser.show()
val result = clk_count_oldUser.join(imp_count_oldUser,"stat_date") val result = clk_count_oldUser.join(imp_count_oldUser,"stat_date")
.join(clk_count_newUser,"stat_date") .join(clk_count_newUser,"stat_date")
.join(imp_count_newUser,"stat_date") .join(imp_count_newUser,"stat_date")
result.show() result.show()
}
} }
} }
......
eda/feededa/src/main/scala/com/gmei/testt.scala 0 → 100644
View file @ 65110bd4
package com.gmei
import java.text.SimpleDateFormat
import java.util.{Calendar, Date}
import java.text.SimpleDateFormat
import java.util.Calendar
import scala.collection.mutable.ArrayBuffer
object testt {
def main(args: Array[String]): Unit ={
val dateArray2 = get_date()
println(dateArray2(0))
for (elem <- dateArray2) {
println(elem)
}
}
def get_date(): ArrayBuffer[String] ={
val startTime = "2017-12-01"
val endTime = "2017-12-10"
val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
val dateFiled = Calendar.DAY_OF_MONTH
var beginDate = dateFormat.parse(startTime)
val endDate = dateFormat.parse(endTime)
val calendar = Calendar.getInstance()
calendar.setTime(beginDate)
val dateArray: ArrayBuffer[String] = ArrayBuffer()
while (beginDate.compareTo(endDate) <= 0) {
dateArray += dateFormat.format(beginDate)
calendar.add(dateFiled, 1)
beginDate = calendar.getTime
}
//println(dateArray)
dateArray
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment