Merge branch 'master' of http://git.wanmeizhensuo.com/ML/ffm-baseline

eda/esmm/Model_pipline/sort_and_2sql.py

@@ -19,7 +19,7 @@ def con_sql(sql):
     return result
 
 def set_join(lst):
-    return ','.join([str(i) for i in set(lst)])
+    return ','.join([str(i) for i in list(lst)])
 
 def main():
 

eda/esmm/Model_pipline/submit.sh

@@ -27,7 +27,7 @@ echo "Bayes Error Rate" : $((repe_feat*100/all_sample))%
 echo "split data"
 split -l $((`wc -l < ${DATA_PATH}/tr.csv`/15)) ${DATA_PATH}/tr.csv -d -a 4 ${DATA_PATH}/tr/tr_ --additional-suffix=.csv
 split -l $((`wc -l < ${DATA_PATH}/va.csv`/5)) ${DATA_PATH}/va.csv -d -a 4 ${DATA_PATH}/va/va_ --additional-suffix=.csv
-split -l $((`wc -l < ${DATA_PATH}/native.csv`/5)) ${DATA_PATH}/native.csv -d -a 4 ${DATA_PATH}/native/native_ --additional-suffix=.csv
+split -l $((`wc -l < ${DATA_PATH}/native.csv`/15)) ${DATA_PATH}/native.csv -d -a 4 ${DATA_PATH}/native/native_ --additional-suffix=.csv
 split -l $((`wc -l < ${DATA_PATH}/nearby.csv`/5)) ${DATA_PATH}/nearby.csv -d -a 4 ${DATA_PATH}/nearby/nearby_ --additional-suffix=.csv
 
 echo "csv to tfrecord"

eda/feededa/src/main/scala/com/gmei/EsmmData.scala

@@ -310,7 +310,7 @@ object EsmmPredData {
       // native_data
       val native_data = sc.sql(
         s"""
-           |select distinct a.device_id,a.city_id,b.native_queue from data_feed_click a
+           |select distinct a.device_id,a.city_id,b.native_queue from data_feed_exposure a
            |left join (select if(city_id='world','worldwide',city_id) city_id,native_queue from biz_feed_diary_queue) b
            |on a.city_id = b.city_id
            |where a.stat_date='${yesteday_have_seq}' and b.native_queue != ""
@@ -417,9 +417,21 @@ object EsmmPredData {
            |and d.partition_date='${yesteday}'
          """.stripMargin
       )
+      union_data_scity_id.createOrReplaceTempView("union_data_scity_id")
+
+      val union_data_scity_id2 = sc.sql(
+        s"""
+           |select device_id,cid_id,first(stat_date) stat_date,first(ucity_id) ucity_id,first(label) label,first(diary_service_id)diary_service_id,first(y) y,
+           |first(z) z,first(clevel1_id) clevel1_id,first(slevel1_id) slevel1_id,first(ccity_name) ccity_name,first(scity_id) scity_id
+           |from union_data_scity_id
+           |group by device_id,cid_id
+         """.stripMargin
+      )
+
+
       //      union_data_scity_id.createOrReplaceTempView("union_data_scity_id")
-      println(union_data_scity_id.count())
-      GmeiConfig.writeToJDBCTable("jdbc:mysql://10.66.157.22:4000/jerry_test?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",union_data_scity_id, table="esmm_pre_data",SaveMode.Overwrite)
+      println(union_data_scity_id2.count())
+      GmeiConfig.writeToJDBCTable("jdbc:mysql://10.66.157.22:4000/jerry_test?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",union_data_scity_id2, table="esmm_pre_data",SaveMode.Overwrite)