Commit 041fd6aa authored by 高雅喆's avatar 高雅喆

add GetLevelCount

parent 4df9c820
......@@ -225,7 +225,6 @@ object EsmmData {
}
object EsmmPredData {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
......@@ -432,7 +431,6 @@ object EsmmPredData {
}
object GetDiaryPortrait {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
......@@ -603,4 +601,65 @@ object GetDevicePortrait {
}
}
}
object GetLevelCount {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
case class Params(env: String = "dev",
path: String = null
) extends AbstractParams[Params] with Serializable
val defaultParams = Params()
val parser = new OptionParser[Params]("Feed_EDA") {
head("EsmmData")
opt[String]("env")
.text(s"the databases environment you used")
.action((x, c) => c.copy(env = x))
opt[String]("path")
.text(s"the path you used")
.action((x,c) => c.copy(path = x))
note(
"""
|For example, the following command runs this app on a tidb dataset:
|
| spark-submit --class com.gmei.EsmmData ./target/scala-2.11/feededa-assembly-0.1.jar \
""".stripMargin +
s"| --env ${defaultParams.env}"
)
}
def main(args: Array[String]): Unit = {
parser.parse(args, defaultParams).map { param =>
GmeiConfig.setup(param.env)
val spark_env = GmeiConfig.getSparkSession()
val sc = spark_env._2
val ti = new TiContext(sc)
ti.tidbMapTable(dbName = "jerry_prod",tableName = "data_feed_click")
ti.tidbMapTable(dbName = "jerry_prod",tableName = "diary_feat")
val diary_queue = sc.read.json(param.path)
val diary_level1 = sc.sql(
s"""
|select diary_id,explode(split(level1_ids,';')) from diary_feat
|where diary_id in "${diary_queue}"
""".stripMargin
)
diary_level1.show()
println(diary_level1.count())
val level1_count = diary_level1.rdd.map(x => (x(1).toString)).map(level1 => (level1,1)).reduceByKey((a,b) => a+b)
level1_count.collect()
sc.stop()
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment