Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

update test file

diary-training.py

+import datetime
+import pymysql
+import pandas as pd
+from sklearn.utils import shuffle
+import numpy as np
+import xlearn as xl
+# 从数据库的表里获取数据，并转化成df格式
+def con_sql(sql):
+    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    cursor = db.cursor()
+    cursor.execute(sql)
+    result = cursor.fetchall()
+    df = pd.DataFrame(list(result)).dropna()
+    db.close()
+    return df
+
+# 获取点击表里的device_id
+sql = "select distinct device_id from data_feed_click where cid_type = 'diary'"
+click_device_id = con_sql(sql)[0].values.tolist()
+print("成功获取点击表里的device_id")
+
+# 获取点击表里的数据
+sql = "select cid,device_id,time from data_feed_click where cid_type = 'diary'"
+click = con_sql(sql)
+click = click.rename(columns={0:"cid",1:"device_id",2:"time"})
+print("成功获取点击表里的数据")
+
+# 获取曝光表里的数据
+sql = "select cid,device_id,time from data_feed_exposure where cid_type = 'diary'"
+exposure = con_sql(sql)
+exposure = exposure.rename(columns={0:"cid",1:"device_id",2:"time"})
+print("成功获取曝光表里的数据")
+
+# 求曝光表和点击表的差集合
+exposure.append(click)
+exposure.append(click)
+subset = click.columns.tolist()
+exposure = exposure.drop_duplicates(subset=subset,keep=False)
+print("成功完成曝光表和点击表的差集合")
+exposure = exposure.loc[exposure["device_id"].isin(click_device_id)]
+# 打标签
+click["y"] = 1
+exposure["y"] = 0
+print("成功获取正负样本")
+
+# 合并点击表和曝光表
+data = click.append(exposure)
+print("done 合并点击表和曝光表")
+print(data.head(2))
+# 从time特征中抽取hour、weekday
+data["hour"] = data["time"].apply(lambda x:datetime.datetime.fromtimestamp(x).hour)
+data["weekday"] = data["time"].apply(lambda x:datetime.datetime.fromtimestamp(x).weekday())
+# 数值是0的特征会被ffm格式删除，经过下面的处理后，没有数值是0的特征
+data.loc[data["hour"]==0] = 24
+data.loc[data["weekday"]==0] = 7
+data["hour"] = data["hour"].astype("category")
+data["weekday"] = data["weekday"].astype("category")
+data = data.drop("time",axis=1)
+print("成功从time特征中抽取hour、weekday")
+print(data.head(2))
+
+data = shuffle(data)
+
+print("start ffm transform")
+# ffm 格式转换函数、类
+class FFMFormatPandas:
+    def __init__(self):
+        self.field_index_ = None
+        self.feature_index_ = None
+        self.y = None
+
+    def fit(self, df, y=None):
+        self.y = y
+        df_ffm = df[df.columns.difference([self.y])]
+        if self.field_index_ is None:
+            self.field_index_ = {col: i for i, col in enumerate(df_ffm)}
+
+        if self.feature_index_ is not None:
+            last_idx = max(list(self.feature_index_.values()))
+
+        if self.feature_index_ is None:
+            self.feature_index_ = dict()
+            last_idx = 0
+
+        for col in df.columns:
+            vals = df[col].unique()
+            for val in vals:
+                if pd.isnull(val):
+                    continue
+                name = '{}_{}'.format(col, val)
+                if name not in self.feature_index_:
+                    self.feature_index_[name] = last_idx
+                    last_idx += 1
+            self.feature_index_[col] = last_idx
+            last_idx += 1
+        return self
+
+    def fit_transform(self, df, y=None):
+        self.fit(df, y)
+        return self.transform(df)
+
+    def transform_row_(self, row, t):
+        ffm = []
+        if self.y != None:
+            ffm.append(str(row.loc[row.index == self.y][0]))
+        if self.y is None:
+            ffm.append(str(0))
+
+        for col, val in row.loc[row.index != self.y].to_dict().items():
+            col_type = t[col]
+            name = '{}_{}'.format(col, val)
+            if col_type.kind ==  'O':
+                ffm.append('{}:{}:1'.format(self.field_index_[col], self.feature_index_[name]))
+            elif col_type.kind == 'i':
+                ffm.append('{}:{}:{}'.format(self.field_index_[col], self.feature_index_[col], val))
+        return ' '.join(ffm)
+
+    def transform(self, df):
+        t = df.dtypes.to_dict()
+        return pd.Series({idx: self.transform_row_(row, t) for idx, row in df.iterrows()})
+
+ffm_train = FFMFormatPandas()
+data = ffm_train.fit_transform(data, y='y')
+print("done transform ffm")
+
+n = np.rint(data.shape[0]/8)
+m = np.rint(data.shape[0]*(3/8))
+# 1/8的数据集用来做测试集
+data.loc[:n].to_csv("/home/zhangyanzhao/test.csv",index = False,header = None)
+# 1/4的数据集用来做验证集
+data.loc[n+1:m].to_csv("/home/zhangyanzhao/validation.csv",index = False,header = None)
+# 剩余的数据集用来做验证集
+data.loc[m+1:].to_csv("/home/zhangyanzhao/train.csv",index = False,header = None)
+# 销毁data，目的是为了节省内存
+data = data.drop(data.index.tolist())
+
+print("start training")
+ffm_model = xl.create_ffm()
+ffm_model.setTrain("/home/zhangyanzhao/train.csv")
+ffm_model.setValidate("/home/zhangyanzhao/validation.csv")
+
+param = {'task':'binary', 'lr':0.2,
+         'lambda':0.002, 'metric':'auc'}
+
+ffm_model.fit(param, '/home/zhangyanzhao/model.out')
+
+ffm_model.setTest("/home/zhangyanzhao/test.csv")
+ffm_model.setSigmoid()
+
+ffm_model.predict("/home/zhangyanzhao/model.out", "/home/zhangyanzhao/output.txt")
+print("end")
+
+
+
+
+
+
+
+
+
+
+
+

diaryTestSet.py

+import pymysql
+import pandas as pd
+db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+
+# 从数据库获取数据，并将数据转化成DataFrame
+def get_data(sql):
+    cursor = db.cursor()
+    cursor.execute(sql)
+    data = cursor.fetchall()
+    data = pd.DataFrame(list(data)).dropna()
+    return data
+
+# 获取全国点击量TOP2000日记
+sql = "select city_id,cid where cid_type = 'diary' order by click_count_choice desc limit 2000"
+allCitiesTop2000 = get_data(sql)
+allCitiesTop2000 = allCitiesTop2000.rename(columns={0:"city_id",1:"cid"})
+allCitiesTop2000.to_csv("\home\zhangyanzhao\diaryTestSet\allCitiesTop2000.csv")
+print("成功获取全国日记点击量TOP2000")
+
+# 获取全国城市列表
+sql = "select distinct city_id from data_feed_click"
+cityList = get_data(sql)
+cityList.to_csv("\home\zhangyanzhao\diaryTestSet\cityList.csv")
+cityList = cityList[0].values.tolist()
+print("成功获取城市列表")
+
+# 获取每个城市点击量TOP2000日记，如果数量小于2000，用全国点击量TOP2000日记补充
+for i in cityList:
+    sql = "select city_id,cid from data_feed_click " \
+          "where cid_type = 'diary' and city_id = {0} " \
+          "order by click_count_choice desc limit 2000".format(i)
+    data = get_data(sql)
+    data = data.rename(columns={0:"city_id",1:"cid"})
+
+    if data.shape[0]<2000:
+        n = 2000-data.shape[0]
+        # 全国点击量TOP2000日记中去除该城市的日记
+        temp = allCitiesTop2000[allCitiesTop2000["city_id"]!=i].loc[:n-1]
+        data = data.append(temp)
+    else:
+        pass
+
+    file_name = "\home\zhangyanzhao\diaryTestSet\{0}DiaryTop2000.csv".format(i)
+    data.to_csv(file_name)
+
+print("end")
+
+
+
+

eda/ml_tools/rate.csv

@@ -5,3 +5,4 @@ date,answer_imp_all,answer_imp_ios,answer_imp_android,diary_imp_all,diary_imp_io
 20180820,0.0374,0.0504,0.032,0.7443,0.8829,0.6866,0.0449,0.068,0.0281,55.88,84.81,0.0157,0.0192,0.0118,0.1142,0.149,0.0846,0.1284,0.1641,0.0981
 20180821,0.0376,0.0509,0.0322,0.7392,0.8783,0.6831,0.0429,0.0658,0.0274,58.85,90.43,0.0174,0.0185,0.0162,0.1162,0.1551,0.0846,0.1316,0.1709,0.0998
 20180822,0.028,0.0391,0.0226,0.7481,0.8792,0.6837,0.0426,0.0614,0.0267,59.29,80.53,0.0155,0.0151,0.0162,0.1212,0.1513,0.0899,0.1363,0.1659,0.1056,0.0223,0.0267,0.0201,0.0714,0.1077,0.0459
+20180823,0.0134,0.0244,0.008,0.7531,0.8862,0.6873,0.0439,0.0664,0.0262,56.91,80.93,0.0154,0.0153,0.0156,0.1277,0.1577,0.0963,0.1447,0.175,0.113,0.0368,0.0412,0.0347,0.0709,0.1053,0.0458