change test file

eda/esmm/Model_pipline/submit.sh

@@ -4,11 +4,10 @@ PYTHON_PATH=/srv/envs/esmm/bin/python
 MODEL_PATH=/srv/apps/ffm-baseline_git/eda/esmm/Model_pipline
 LOCAL_PATH=/home/gmuser/esmm
 HDFS_PATH=hdfs://172.16.32.4:8020/strategy/esmm
-
+export CLASSPATH=$(hadoop classpath --glob)
 
 echo "rm model file"
 rm -r ${LOCAL_PATH}/model_ckpt/DeepCvrMTL/20*
 
 echo "train..."
-CLASSPATH="$(hadoop classpath --glob)" ${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH} --task_type=train
-
+CLASSPATH="$(hadoop classpath --glob)" ${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH} --task_type=train
\ No newline at end of file

eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala

@@ -622,6 +622,38 @@ object icon_train_data {
 
       temp_meigou.createOrReplaceTempView("temp_meigou")
 
+      // 百科点击数据
+      val wiki_click=sc.sql(
+        s"""
+           |SELECT cl_id as device_id,params['business_id'] as wiki_id
+           |from online.tl_hdfs_maidian_view
+           |where partition_date='${partition_date}'
+           |and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
+       """.stripMargin
+      )
+      wiki_click.createOrReplaceTempView("wiki_click")
+
+      val wiki_tag=sc.sql(
+        s"""
+           |SELECT item_id,tag_id
+           |from online.tl_hdfs_wiki_item_tag_view
+           |where partition_date='${partition_date}'
+       """.stripMargin
+      )
+      wiki_tag.createOrReplaceTempView("wiki_tag")
+
+      val wiki_device_tag=sc.sql(
+        s"""
+           |SELECT a.device_id,b.tag_id
+           |from wiki_click a
+           |left join wiki_tag b
+           |on a.wiki_id=b.item_id
+           |where b.tag_id is not null
+       """.stripMargin
+      )
+      wiki_device_tag.createOrReplaceTempView("wiki_device_tag")
+
+
       val final_train = sc.sql(
         s"""
            |select *
@@ -629,13 +661,16 @@ object icon_train_data {
            |union all
            |select *
            |from temp_meigou
+           |union all
+           |select *
+           |from wiki_device_tag
        """.stripMargin
       )
       final_train.show()
 
 
       println("开始写入")
-      GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",final_train, table="icon_train_data_test",SaveMode.Append)
+      GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",final_train, table="icon_train_data",SaveMode.Append)
       println("写入完成")
 
 
@@ -853,79 +888,38 @@ object test_data {
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
-//美购加车数据
-      val meigou_jiache=sc.sql(
-        s"""
-           |SELECT T.partition_date
-           |T5_1.service_id AS service_id,
-           |       T5_2.device_id AS device_id,
-           |  FROM (SELECT DISTINCT T.partition_date,T.person_id,
-           |                        T.service_id AS service_id
-           |          FROM online.tl_meigou_shopcart_view T
-           |         WHERE T.partition_date = '${partition_date}'
-           |           AND from_unixtime(unix_timestamp(to_date(T.added_at), 'yyyy-MM-dd'), 'yyyyMMdd') = '${partition_date}') T5_1 --加入购物车的时间
-           |  LEFT JOIN (SELECT T.person_id    AS person_id,
-           |                    T5_3.device_id AS device_id
-           |               FROM online.ml_user_updates T
-           |            LATERAL VIEW explode(T.device_list) T5_3 AS device_id
-           |              WHERE T.partition_date = '${partition_date}') T5_2
-           |    ON T5_1.person_id = T5_2.person_id
-       """.stripMargin
-      )
-//      meigou_jiache.show()
 
-//美购订单数据
-      val megou_order=sc.sql(
-        s"""
-           |SELECT partition_date,device_id,service_id
-           |from online.ml_meigou_order_detail
-           |where partition_date='${partition_date}'
-           |and device_id is not null
-       """.stripMargin
-      )
-//      megou_order.show()
+// 百科点击数据
 
-//咨询美购数据
-      val zixun_meigou=sc.sql(
+      val wiki_click=sc.sql(
         s"""
-           |SELECT cl_id as device_id,params['service_id'] as service_id
+           |SELECT cl_id as device_id,params['business_id'] as wiki_id
            |from online.tl_hdfs_maidian_view
            |where partition_date='${partition_date}'
-           |and (action='welfare_detail_click_phone' or action='welfare_detail_click_consult' or action='welfare_detail_click_message' or action='welfare_detail_click_curearea_contact')
+           |and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
        """.stripMargin
       )
-      zixun_meigou.show()
+      wiki_click.createOrReplaceTempView("wiki_click")
 
-// 百科点击数据
-
-      val wiki_click=sc.sql(
+      val wiki_tag=sc.sql(
         s"""
-           |SELECT cl_id,params['business_id'] as wiki_id
-           |from online.tl_hdfs_maidian_view
-           |where partition_date='20190523'
-           |and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
+           |SELECT item_id,tag_id
+           |from online.tl_hdfs_wiki_item_tag_view
+           |where partition_date='${partition_date}'
        """.stripMargin
       )
-      wiki_click.show()
-
-//      补单美购数据
+      wiki_tag.createOrReplaceTempView("wiki_tag")
 
-      val budan_click=sc.sql(
+      val wiki_device_tag=sc.sql(
         s"""
-           |SELECT cl_id,params['business_id'] as wiki_id
-           |from online.tl_hdfs_maidian_view
-           |where partition_date='20190523'
-           |and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
+           |SELECT a.device_id,b.tag_id
+           |from wiki_click a
+           |left join wiki_tag b
+           |on a.wiki_id=b.item_id
+           |where b.tag_id is not null
        """.stripMargin
       )
-      wiki_click.show()
-
-
-
-
-
-
-
+      wiki_device_tag.createOrReplaceTempView("wiki_device_tag")
 
 
     }