Merge branch 'master' of http://git.wanmeizhensuo.com/ML/ffm-baseline

eda/esmm/.gitignore

+*.class
+*.log
+Model_pipline/model_ckpt/*
+data/*
+__pycache__/
+*.py[cod]
+metastore_db/*
+*.idea
\ No newline at end of file

eda/esmm/Feature_pipline/get_tfrecord.py

+#!/usr/bin/env python
+#coding=utf-8
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+import os
+import glob
+
+import tensorflow as tf
+import numpy as np
+import re
+from multiprocessing import Pool as ThreadPool
+
+flags = tf.app.flags
+FLAGS = flags.FLAGS
+LOG = tf.logging
+
+tf.app.flags.DEFINE_string("input_dir", "./", "input dir")
+tf.app.flags.DEFINE_string("output_dir", "./", "output dir")
+tf.app.flags.DEFINE_integer("threads", 16, "threads num")
+
+#保证顺序以及字段数量
+#User_Fileds = set(['101','109_14','110_14','127_14','150_14','121','122','124','125','126','127','128','129'])
+#Ad_Fileds = set(['205','206','207','210','216'])
+#Context_Fileds = set(['508','509','702','853','301'])
+#Common_Fileds = {'1':'1','2':'2','3':'3','4':'4','5':'5','6':'6','7':'7','8':'8','9':'9','10':'10','11':'11','12':'12','13':'13','14':'14','15':'15','16':'16','17':'17','18':'18','19':'19','20':'20','21':'21','22':'22','23':'23','24':'24','25':'25','26':'26','27':'27','28':'28','29':'29','30':'30'}
+Common_Fileds = {'1':'1','2':'2','3':'3','4':'4','5':'5','6':'6','7':'7','8':'8','9':'9','10':'10','11':'11'}
+UMH_Fileds      = {'109_14':('u_cat','12'),'110_14':('u_shop','13'),'127_14':('u_brand','14'),'150_14':('u_int','15')}      #user multi-hot feature
+Ad_Fileds       = {'206':('a_cat','16'),'207':('a_shop','17'),'210':('a_int','18'),'216':('a_brand','19')}                  #ad feature for DIN
+
+#40362692,0,0,216:9342395:1.0 301:9351665:1.0 205:7702673:1.0 206:8317829:1.0 207:8967741:1.0 508:9356012:2.30259 210:9059239:1.0 210:9042796:1.0 210:9076972:1.0 210:9103884:1.0 210:9063064:1.0 127_14:3529789:2.3979 127_14:3806412:2.70805
+def gen_tfrecords(in_file):
+    basename = os.path.basename(in_file) + ".tfrecord"
+    out_file = os.path.join(FLAGS.output_dir, basename)
+    tfrecord_out = tf.python_io.TFRecordWriter(out_file)
+    with open(in_file) as fi:
+        for line in fi:
+            line = line.strip().split('\t')[-1]
+            fields = line.strip().split(',')
+            if len(fields) != 4:
+                continue
+            #1 label
+            y = [float(fields[1])]
+            z = [float(fields[2])]
+            feature = {
+                "y": tf.train.Feature(float_list = tf.train.FloatList(value=y)),
+                "z": tf.train.Feature(float_list = tf.train.FloatList(value=z))
+             }
+
+            splits = re.split('[ :]', fields[3])
+            ffv = np.reshape(splits,(-1,3))
+            #common_mask = np.array([v in Common_Fileds for v in ffv[:,0]])
+            #af_mask = np.array([v in Ad_Fileds for v in ffv[:,0]])
+            #cf_mask = np.array([v in Context_Fileds for v in ffv[:,0]])
+
+            #2 不需要特殊处理的特征
+            feat_ids = np.array([])
+            #feat_vals = np.array([])
+            for f, def_id in Common_Fileds.items():
+                if f in ffv[:,0]:
+                    mask = np.array(f == ffv[:,0])
+                    feat_ids = np.append(feat_ids, ffv[mask,1])
+                    #np.append(feat_vals,ffv[mask,2].astype(np.float))
+                else:
+                    feat_ids = np.append(feat_ids, def_id)
+                    #np.append(feat_vals,1.0)
+            feature.update({"feat_ids": tf.train.Feature(int64_list=tf.train.Int64List(value=feat_ids.astype(np.int)))})
+                            #"feat_vals": tf.train.Feature(float_list=tf.train.FloatList(value=feat_vals))})
+
+            #3 特殊字段单独处理
+            for f, (fname, def_id) in UMH_Fileds.items():
+                if f in ffv[:,0]:
+                    mask = np.array(f == ffv[:,0])
+                    feat_ids = ffv[mask,1]
+                    feat_vals= ffv[mask,2]
+                else:
+                    feat_ids = np.array([def_id])
+                    feat_vals = np.array([1.0])
+                feature.update({fname+"ids": tf.train.Feature(int64_list=tf.train.Int64List(value=feat_ids.astype(np.int))),
+                                fname+"vals": tf.train.Feature(float_list=tf.train.FloatList(value=feat_vals.astype(np.float)))})
+
+            for f, (fname, def_id) in Ad_Fileds.items():
+                if f in ffv[:,0]:
+                    mask = np.array(f == ffv[:,0])
+                    feat_ids = ffv[mask,1]
+                else:
+                    feat_ids = np.array([def_id])
+                feature.update({fname+"ids": tf.train.Feature(int64_list=tf.train.Int64List(value=feat_ids.astype(np.int)))})
+
+            # serialized to Example
+            example = tf.train.Example(features = tf.train.Features(feature = feature))
+            serialized = example.SerializeToString()
+            tfrecord_out.write(serialized)
+            #num_lines += 1
+            #if num_lines % 10000 == 0:
+            #    print("Process %d" % num_lines)
+    tfrecord_out.close()
+
+def main(_):
+    if not os.path.exists(FLAGS.output_dir):
+        os.mkdir(FLAGS.output_dir)
+    file_list = glob.glob(os.path.join(FLAGS.input_dir, "*.csv"))
+    print("total files: %d" % len(file_list))
+
+    pool = ThreadPool(FLAGS.threads) # Sets the pool size
+    pool.map(gen_tfrecords, file_list)
+    pool.close()
+    pool.join()
+
+
+if __name__ == "__main__":
+    tf.logging.set_verbosity(tf.logging.INFO)
+    tf.app.run()
\ No newline at end of file

eda/esmm/Model_pipline/send_mail.py

+# -*- coding: utf-8 -*-
+
+import smtplib
+from email.mime.text import MIMEText
+from email.utils import formataddr
+import datetime
+
+my_sender='gaoyazhe@igengmei.com'
+my_pass = 'VCrKTui99a7ALhiK'
+my_user='gaoyazhe@igengmei.com'
+def mail():
+    ret=True
+    try:
+        with open('/home/gaoyazhe/data/submit.log') as f:
+            stat_data = f.read()
+            msg=MIMEText(stat_data,'plain','utf-8')
+            msg['From']=formataddr(["高雅喆",my_sender])
+            msg['To']=formataddr(["高雅喆",my_user])
+            msg['Subject']= str(datetime.date.today())+"-esmm多目标模型训练指标统计"
+            server=smtplib.SMTP_SSL("smtp.exmail.qq.com", 465)
+            server.login(my_sender, my_pass)
+            server.sendmail(my_sender,[my_user,],msg.as_string())
+            server.quit()
+    except Exception:
+        ret=False
+    return ret
+
+ret=mail()
+if ret:
+    print("邮件发送成功")
+else:
+    print("邮件发送失败")
\ No newline at end of file

eda/esmm/Model_pipline/sort_and_2sql.py

+from sqlalchemy import create_engine
+import pandas as pd
+import pymysql
+import MySQLdb
+import time
+
+def con_sql(sql):
+    """
+    :type sql : str
+    :rtype : tuple
+    """
+    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    cursor = db.cursor()
+    cursor.execute(sql)
+    result = cursor.fetchall()
+    db.close()
+    return result
+
+def set_join(lst):
+    return ','.join(set(lst))
+
+def main():
+    sql = "select device_id,city_id,cid from esmm_data2ffm_infer_native"
+    result = con_sql(sql)
+    dct = {"uid":[],"city":[],"cid_id":[]}
+    for i in result:
+        dct["uid"].append(i[0])
+        dct["city"].append(i[1])
+        dct["cid_id"].append(i[2])
+
+    df1 = pd.read_csv("/home/gaoyazhe/data/native/pred.txt",sep='\t',header=None,names=["ctr","cvr"])
+    df2 = pd.DataFrame(dct)
+    df2["ctr"],df2["cvr"] = df1["ctr"],df1["cvr"]
+    df3 = df2.groupby(by=["uid","city"]).apply(lambda x: x.sort_values(by="cvr",ascending=False)).reset_index(drop=True).groupby(by=["uid","city"]).agg({'cid_id':set_join}).reset_index(drop=False)
+    ctime = int(time.time())
+    df3["time"] = ctime
+    df3.columns = ["device_id","city_id","native_queue","time"]
+    print("native_device_count",df3.shape)
+
+
+    sql_nearby = "select device_id,city_id,cid from esmm_data2ffm_infer_nearby"
+    result = con_sql(sql_nearby)
+    dct = {"uid":[],"city":[],"cid_id":[]}
+    for i in result:
+        dct["uid"].append(i[0])
+        dct["city"].append(i[1])
+        dct["cid_id"].append(i[2])
+
+    df1 = pd.read_csv("/home/gaoyazhe/data/nearby/pred.txt",sep='\t',header=None,names=["ctr","cvr"])
+    df2 = pd.DataFrame(dct)
+    df2["ctr"],df2["cvr"] = df1["ctr"],df1["cvr"]
+    df4 = df2.groupby(by=["uid","city"]).apply(lambda x: x.sort_values(by="cvr",ascending=False)).reset_index(drop=True).groupby(by=["uid","city"]).agg({'cid_id':set_join}).reset_index(drop=False)
+    df4.columns = ["device_id","city_id","nearby_queue"]
+    print("nearby_device_count",df4.shape)
+
+    #union
+    df_all = pd.merge(df3,df4,on=['device_id','city_id'],how='outer').fillna("")
+    print("union_device_count",df_all.shape)
+
+
+
+    host='10.66.157.22'
+    port=4000
+    user='root'
+    password='3SYz54LS9#^9sBvC'
+    db='jerry_test'
+    charset='utf8'
+
+    engine = create_engine(str(r"mysql+mysqldb://%s:" + '%s' + "@%s:%s/%s") % (user, password, host, port, db))
+
+    try:
+        df_all.to_sql('esmm_device_diary_queue',con=engine,if_exists='replace',index=False)
+    except Exception as e:
+        print(e)
+
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

eda/esmm/Model_pipline/submit.sh

+#! /bin/bash
+PYTHON_PATH=/home/gaoyazhe/miniconda3/bin/python
+MODEL_PATH=/srv/apps/ffm-baseline/eda/esmm
+DATA_PATH=/home/gaoyazhe/data
+
+echo "start timestamp"
+current=$(date "+%Y-%m-%d %H:%M:%S")
+timeStamp=$(date -d "$current" +%s)
+currentTimeStamp=$((timeStamp*1000+`date "+%N"`/1000000))
+echo $currentTimeStamp
+
+echo "rm leave tfrecord"
+rm ${DATA_PATH}/tr/*
+rm ${DATA_PATH}/va/*
+rm ${DATA_PATH}/native/*
+rm ${DATA_PATH}/nearby/*
+
+echo "mysql to csv"
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_train" > ${DATA_PATH}/tr.csv
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_cv" > ${DATA_PATH}/va.csv
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_infer_native" > ${DATA_PATH}/native.csv
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_infer_nearby" > ${DATA_PATH}/nearby.csv
+
+echo "split data"
+split -l $((`wc -l < ${DATA_PATH}/tr.csv`/15)) ${DATA_PATH}/tr.csv -d -a 4 ${DATA_PATH}/tr/tr_ --additional-suffix=.csv
+split -l $((`wc -l < ${DATA_PATH}/va.csv`/5)) ${DATA_PATH}/va.csv -d -a 4 ${DATA_PATH}/va/va_ --additional-suffix=.csv
+split -l $((`wc -l < ${DATA_PATH}/native.csv`/5)) ${DATA_PATH}/native.csv -d -a 4 ${DATA_PATH}/native/native_ --additional-suffix=.csv
+split -l $((`wc -l < ${DATA_PATH}/nearby.csv`/5)) ${DATA_PATH}/nearby.csv -d -a 4 ${DATA_PATH}/nearby/nearby_ --additional-suffix=.csv
+
+echo "csv to tfrecord"
+${PYTHON_PATH} ${MODEL_PATH}/Feature_pipline/get_tfrecord.py --input_dir=${DATA_PATH}/tr/ --output_dir=${DATA_PATH}/tr/
+${PYTHON_PATH} ${MODEL_PATH}/Feature_pipline/get_tfrecord.py --input_dir=${DATA_PATH}/va/ --output_dir=${DATA_PATH}/va/
+${PYTHON_PATH} ${MODEL_PATH}/Feature_pipline/get_tfrecord.py --input_dir=${DATA_PATH}/native/ --output_dir=${DATA_PATH}/native/
+${PYTHON_PATH} ${MODEL_PATH}/Feature_pipline/get_tfrecord.py --input_dir=${DATA_PATH}/nearby/ --output_dir=${DATA_PATH}/nearby/
+
+cat ${DATA_PATH}/tr/*.tfrecord > ${DATA_PATH}/tr/tr.tfrecord
+cat ${DATA_PATH}/va/*.tfrecord > ${DATA_PATH}/va/va.tfrecord
+cat ${DATA_PATH}/native/*.tfrecord > ${DATA_PATH}/native/native.tfrecord
+cat ${DATA_PATH}/nearby/*.tfrecord > ${DATA_PATH}/nearby/nearby.tfrecord
+
+rm ${DATA_PATH}/tr/tr_*
+rm ${DATA_PATH}/va/va_*
+rm ${DATA_PATH}/native/native_*
+rm ${DATA_PATH}/nearby/nearby_*
+
+echo "data transform timestamp"
+current=$(date "+%Y-%m-%d %H:%M:%S")
+timeStamp=$(date -d "$current" +%s)
+currentTimeStamp=$((timeStamp*1000+`date "+%N"`/1000000))
+echo $currentTimeStamp
+
+echo "train..."
+${PYTHON_PATH} ${MODEL_PATH}/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir="${DATA_PATH}" --task_type="train"
+
+echo "train timestamp"
+current=$(date "+%Y-%m-%d %H:%M:%S")
+timeStamp=$(date -d "$current" +%s)
+currentTimeStamp=$((timeStamp*1000+`date "+%N"`/1000000))
+echo $currentTimeStamp
+
+${PYTHON_PATH} ${MODEL_PATH}/Model_pipline/send_mail.py
+
+echo "infer native..."
+${PYTHON_PATH} ${MODEL_PATH}/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir="${DATA_PATH}/native" --task_type="infer" > ${DATA_PATH}/infer.log
+
+
+echo "infer nearby..."
+${PYTHON_PATH} ${MODEL_PATH}/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir="${DATA_PATH}/nearby" --task_type="infer" > ${DATA_PATH}/infer.log
+
+echo "sort and 2sql"
+${PYTHON_PATH} ${MODEL_PATH}/Model_pipline/sort_and_2sql.py
+
+echo "infer and sort and 2sql timestamp"
+current=$(date "+%Y-%m-%d %H:%M:%S")
+timeStamp=$(date -d "$current" +%s)
+currentTimeStamp=$((timeStamp*1000+`date "+%N"`/1000000))
+echo $currentTimeStamp
\ No newline at end of file

eda/feededa/src/main/scala/com/gmei/Data2FFM.scala

@@ -52,10 +52,11 @@ object Data2FFM {
       ti.tidbMapTable(dbName = "jerry_test", tableName = "esmm_pre_data")
 
 
-//      val yesteday_have_seq = GmeiConfig.getMinusNDate(5)
+      val train_sep_date = GmeiConfig.getMinusNDate(10)
       val esmm_data = sc.sql(
         s"""
            |select device_id,y,z,stat_date,ucity_id,cid_id,clevel1_id,ccity_name from esmm_train_data
+           |where stat_date > '${train_sep_date}'
          """.stripMargin
       ).repartition(200).na.drop()
       val column_list = esmm_data.columns.filter(x => x != "y" && x != "z")
@@ -114,28 +115,42 @@ object Data2FFM {
 
       val esmm_pre_data = sc.sql(
         s"""
-           |select device_id,y,z,stat_date,ucity_id,cid_id,clevel1_id,ccity_name
+           |select device_id,y,z,stat_date,ucity_id,cid_id,clevel1_id,ccity_name,label
            |from esmm_pre_data
         """.stripMargin
       ).repartition(200).na.drop()
-
+      esmm_pre_data.persist()
       val esmm_pre_cids = esmm_pre_data.select("cid_id").distinct().collect().map(
         s => s(0).toString
       )
       val esmm_pre_city = esmm_pre_data.select("ucity_id").distinct().collect().map(
-        s => s(0).toString
-      )
+        s => s(0).toString)
+      val esmm_pre_device = esmm_pre_data.select("device_id").distinct().collect().map(
+        s => s(0).toString)
 
       val esmm_join_cids = esmm_pre_cids.intersect(column_number("cid_id"))
       val esmm_join_city = esmm_pre_city.intersect(column_number("ucity_id"))
+      val esmm_join_device = esmm_pre_device.intersect(column_number("device_id"))
 
       val rdd_pre = esmm_pre_data.rdd.repartition(200)
         .map(x => (x(0).toString,x(1).toString,x(2).toString,x(3).toString,
           x(4).toString,x(5).toString,x(6).toString,
-          x(7).toString)).filter(x => esmm_join_cids.indexOf(x._6) != -1)
-        .filter(x => esmm_join_city.indexOf(x._5) != -1)
+          x(7).toString,x(8).toString)).filter(x => esmm_join_cids.indexOf(x._6) != -1)
+        .filter(x => esmm_join_city.indexOf(x._5) != -1).filter(x => esmm_join_device.indexOf(x._1) != -1)
+
+      val native_pre = rdd_pre.filter(x => x._9 == "0").map(x => (x._1,x._2,x._3,column_number("device_id").indexOf(x._1),
+        column_number("stat_date").indexOf(x._4), column_number("ucity_id").indexOf(x._5),
+        column_number("cid_id").indexOf(x._6), column_number("clevel1_id").indexOf(x._7),
+        column_number("ccity_name").indexOf(x._8),x._5,x._6))
+        .map(x => ((new util.Random).nextInt(2147483647),x._2,x._3,"1:%d:1.0 2:%d:1.0 3:%d:1.0 4:%d:1.0 5:%d:1.0 6:%d:1.0".
+          format(x._4,x._5,x._6,x._7,x._8,x._9),x._1,x._10,x._11)).zipWithIndex()
+        .map(x => (x._1._1,x._2,x._1._2,x._1._3,x._1._4,x._1._5,x._1._6,x._1._7))
+        .map(x => (x._1,x._2+","+x._3+","+x._4+","+x._5,x._6,x._7,x._8)).toDF("number","data","device_id","city_id","cid")
+      println("pre")
+      native_pre.show(6)
+      GmeiConfig.writeToJDBCTable(jdbcuri, native_pre, "esmm_data2ffm_infer_native", SaveMode.Overwrite)
 
-      val pre = rdd_pre.map(x => (x._1,x._2,x._3,column_number("device_id").indexOf(x._1),
+      val nearby_pre = rdd_pre.filter(x => x._9 == "1").map(x => (x._1,x._2,x._3,column_number("device_id").indexOf(x._1),
         column_number("stat_date").indexOf(x._4), column_number("ucity_id").indexOf(x._5),
         column_number("cid_id").indexOf(x._6), column_number("clevel1_id").indexOf(x._7),
         column_number("ccity_name").indexOf(x._8),x._5,x._6))
@@ -144,8 +159,8 @@ object Data2FFM {
         .map(x => (x._1._1,x._2,x._1._2,x._1._3,x._1._4,x._1._5,x._1._6,x._1._7))
         .map(x => (x._1,x._2+","+x._3+","+x._4+","+x._5,x._6,x._7,x._8)).toDF("number","data","device_id","city_id","cid")
       println("pre")
-      pre.show(6)
-      GmeiConfig.writeToJDBCTable(jdbcuri, pre, "esmm_data2ffm_infer", SaveMode.Overwrite)
+      nearby_pre.show(6)
+      GmeiConfig.writeToJDBCTable(jdbcuri, nearby_pre, "esmm_data2ffm_infer_nearby", SaveMode.Overwrite)
 
 
       sc.stop()