Skip to content

F
ffm-baseline
Commit 9cd2e00f authored by 张彦钊's avatar 张彦钊
Browse files
Options

change test file

parents 4bc3522e 36e93b36
Hide whitespace changes
Inline Side-by-side
Showing with 56 additions and 62 deletions
eda/esmm/Model_pipline/submit.sh
View file @ 9cd2e00f
......@@ -4,11 +4,10 @@ PYTHON_PATH=/srv/envs/esmm/bin/python
MODEL_PATH=/srv/apps/ffm-baseline_git/eda/esmm/Model_pipline
LOCAL_PATH=/home/gmuser/esmm
HDFS_PATH=hdfs://172.16.32.4:8020/strategy/esmm
export CLASSPATH=$(hadoop classpath --glob)
echo "rm model file"
rm -r ${LOCAL_PATH}/model_ckpt/DeepCvrMTL/20*
echo "train..."
CLASSPATH="$(hadoop classpath --glob)" ${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH} --task_type=train
CLASSPATH="$(hadoop classpath --glob)" ${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH} --task_type=train
\ No newline at end of file
eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala
View file @ 9cd2e00f
......@@ -622,6 +622,38 @@ object icon_train_data {
temp_meigou.createOrReplaceTempView("temp_meigou")
// 百科点击数据
val wiki_click=sc.sql(
s"""
|SELECT cl_id as device_id,params['business_id'] as wiki_id
|from online.tl_hdfs_maidian_view
|where partition_date='${partition_date}'
|and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
""".stripMargin
)
wiki_click.createOrReplaceTempView("wiki_click")
val wiki_tag=sc.sql(
s"""
|SELECT item_id,tag_id
|from online.tl_hdfs_wiki_item_tag_view
|where partition_date='${partition_date}'
""".stripMargin
)
wiki_tag.createOrReplaceTempView("wiki_tag")
val wiki_device_tag=sc.sql(
s"""
|SELECT a.device_id,b.tag_id
|from wiki_click a
|left join wiki_tag b
|on a.wiki_id=b.item_id
|where b.tag_id is not null
""".stripMargin
)
wiki_device_tag.createOrReplaceTempView("wiki_device_tag")
val final_train = sc.sql(
s"""
|select *
......@@ -629,13 +661,16 @@ object icon_train_data {
|union all
|select *
|from temp_meigou
|union all
|select *
|from wiki_device_tag
""".stripMargin
)
final_train.show()
println("开始写入")
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",final_train, table="icon_train_data_test",SaveMode.Append)
GmeiConfig.writeToJDBCTable("jdbc:mysql://172.16.40.158:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",final_train, table="icon_train_data",SaveMode.Append)
println("写入完成")
......@@ -853,79 +888,38 @@ object test_data {
//println(param.date)
val partition_date = stat_date.replace("-","")
//美购加车数据
val meigou_jiache=sc.sql(
s"""
|SELECT T.partition_date
|T5_1.service_id AS service_id,
| T5_2.device_id AS device_id,
| FROM (SELECT DISTINCT T.partition_date,T.person_id,
| T.service_id AS service_id
| FROM online.tl_meigou_shopcart_view T
| WHERE T.partition_date = '${partition_date}'
| AND from_unixtime(unix_timestamp(to_date(T.added_at), 'yyyy-MM-dd'), 'yyyyMMdd') = '${partition_date}') T5_1 --加入购物车的时间
| LEFT JOIN (SELECT T.person_id AS person_id,
| T5_3.device_id AS device_id
| FROM online.ml_user_updates T
| LATERAL VIEW explode(T.device_list) T5_3 AS device_id
| WHERE T.partition_date = '${partition_date}') T5_2
| ON T5_1.person_id = T5_2.person_id
""".stripMargin
)
// meigou_jiache.show()
//美购订单数据
val megou_order=sc.sql(
s"""
|SELECT partition_date,device_id,service_id
|from online.ml_meigou_order_detail
|where partition_date='${partition_date}'
|and device_id is not null
""".stripMargin
)
// megou_order.show()
// 百科点击数据
//咨询美购数据
val zixun_meigou=sc.sql(
val wiki_click=sc.sql(
s"""
|SELECT cl_id as device_id,params['service_id'] as service_id
|SELECT cl_id as device_id,params['business_id'] as wiki_id
|from online.tl_hdfs_maidian_view
|where partition_date='${partition_date}'
|and (action='welfare_detail_click_phone' or action='welfare_detail_click_consult' or action='welfare_detail_click_message' or action='welfare_detail_click_curearea_contact')
|and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
""".stripMargin
)
zixun_meigou.show()
wiki_click.createOrReplaceTempView("wiki_click")
// 百科点击数据
val wiki_click=sc.sql(
val wiki_tag=sc.sql(
s"""
|SELECT cl_id,params['business_id'] as wiki_id
|from online.tl_hdfs_maidian_view
|where partition_date='20190523'
|and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
|SELECT item_id,tag_id
|from online.tl_hdfs_wiki_item_tag_view
|where partition_date='${partition_date}'
""".stripMargin
)
wiki_click.show()
// 补单美购数据
wiki_tag.createOrReplaceTempView("wiki_tag")
val budan_click=sc.sql(
val wiki_device_tag=sc.sql(
s"""
|SELECT cl_id,params['business_id'] as wiki_id
|from online.tl_hdfs_maidian_view
|where partition_date='20190523'
|and (action='Search_result_wiki_click_item' or action='search_result_click_infomation_item' or action='search_result_wiki_click_recommend_wiki')
|SELECT a.device_id,b.tag_id
|from wiki_click a
|left join wiki_tag b
|on a.wiki_id=b.item_id
|where b.tag_id is not null
""".stripMargin
)
wiki_click.show()
wiki_device_tag.createOrReplaceTempView("wiki_device_tag")
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment