Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

add ctr

ctr_search.py

+# -*- coding: UTF-8 -*-
+import pymysql
+import datetime
+import pandas as pd
+
+DIRECTORY_PATH="/data2/ffm/"
+
+def get_yesterday_date():
+    today = datetime.date.today()
+    yesterday = today - datetime.timedelta(days=1)
+    yesterday = yesterday.strftime("%Y-%m-%d")
+    print(yesterday)
+    return yesterday
+
+
+def get_data():
+    conn2db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='eagle')
+    cursor = conn2db.cursor()
+    sql = "select device_id from search_queue where device_id regexp '[7|8]$'"
+    cursor.execute(sql)
+    result = cursor.fetchall()
+    device_id = tuple(pd.DataFrame(list(result))[0].values.tolist())
+    cursor.close()
+    print(device_id[0:2])
+
+    return device_id
+
+
+def ctr_all():
+    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_prod')
+    cursor = db.cursor()
+    sql_active = "select distinct device_id from data_feed_exposure where cid_type = 'diary'" \
+                 "and device_id regexp'[7|8]$' and stat_date = '{}';".format(date)
+    cursor.execute(sql_active)
+    result = cursor.fetchall()
+    tail56 = tuple(pd.DataFrame(list(result))[0].values.tolist())
+    print("当天尾号7或8活跃用户数：")
+    print(len(tail56))
+    cover = len(set(device_id) & set(tail56))
+    print("当天尾号7或8活跃用户覆盖数：")
+    print(cover)
+    cover_percent = cover / len(set(device_id))
+    print("当天尾号7或8活跃用户覆盖率：")
+    print(cover_percent)
+    sql_click = "select count(cid) from data_feed_click " \
+                "where cid_type = 'diary'" \
+                "and stat_date = '{}' and device_id regexp '[7|8]$';".format(date)
+    cursor.execute(sql_click)
+    click = cursor.fetchone()[0]
+    print("点击数：" + str(click))
+    sql_exp = "select count(cid) from data_feed_exposure " \
+              "where cid_type = 'diary' and stat_date = '{}' and " \
+              "device_id regexp '[7|8]$'".format(date)
+    cursor.execute(sql_exp)
+    exp = cursor.fetchone()[0]
+    print("曝光数：" + str(exp))
+    print("点击率：" + str(click / exp))
+    return len(tail56),cover,cover_percent,click,exp,click / exp
+
+
+def ctr():
+    sql_click = "select count(cid) from data_feed_click " \
+              "where cid_type = 'diary'" \
+                "and stat_date = '{}' and device_id in {};".format(date,device_id)
+    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_prod')
+    cursor = db.cursor()
+    cursor.execute(sql_click)
+    click = cursor.fetchone()[0]
+    print("实验用户点击数："+str(click))
+    sql_exp = "select count(cid) from data_feed_exposure " \
+              "where cid_type = 'diary' and stat_date = '{}' and " \
+              "device_id in {}".format(date,device_id)
+    cursor.execute(sql_exp)
+    exp = cursor.fetchone()[0]
+    print("实验用户曝光数："+str(exp))
+    print("实验用户点击率："+str(click/exp))
+
+    return click,exp,click/exp
+
+
+def rate2file():
+    output_path = DIRECTORY_PATH + "search_ctr.csv"
+    with open(output_path,'a+') as f:
+        line = date.replace('-', '')+','+str(temp_data[0])+','+str(temp_data[1])+','+str(temp_data[2])+\
+               ","+str(data[0])+","+str(data[1])+","+str(data[2])+","+str(data[3])+","+str(data[4])\
+               +","+str(data[5])+'\n'
+        f.write(line)
+
+if __name__ == "__main__":
+    date = get_yesterday_date()
+    device_id = get_data()
+    temp_data = ctr()
+    data = ctr_all()
+    rate2file()
\ No newline at end of file

eda/gray_stat/ctr.py

+# -*- coding: UTF-8 -*-
+from utils import con_sql,get_yesterday_date,get_between_day
+import time
+import sys
+
+
+OUTPUT_PATH = "/data2/models/eda/node2vec/"
+
+num = sys.argv[1]
+if num == "3|4":
+    OUTPUT_PATH = "/data2/models/eda/node2vec/3|4"
+    table_name = "jerry_prod.nd_device_cid_similarity_matrix_tmp"
+elif num == "5|6":
+    OUTPUT_PATH = "/data2/models/eda/node2vec/5|6"
+    table_name = "eagle.ffm_diary_queue_temp"
+elif num == "7|8":
+    OUTPUT_PATH = "/data2/models/eda/node2vec/7|8"
+    table_name = "eagle.search_queue"
+else:
+    print("参数必须是 '3|4' 或 '5|6' 或 '7|8'")
+    print("程序异常停止，请重新输入参数运行")
+    sys.exit()
+
+class GrayStat(object):
+	def __init__(self, cid_type, uid_type, ndays=get_yesterday_date()):
+		"""
+		cid_type : diary，answer，question
+		uid_type : 8：_8结尾；6：_6结尾；6|8：_6或者_8结尾;^68：不是6或者8结尾的
+		ndays : '2018-08-30'....
+		"""
+		self.cid_type = cid_type
+		self.uid_type = uid_type
+		self.ndays = ndays
+
+	def get_uid_count(self):
+		sql = "select count(distinct(device_id)) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id in \
+				(select device_id \
+				from {3}) \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type,table_name)
+		uid_count = con_sql(sql)[0][0]
+		return uid_count
+
+	def get_uid_clk_times(self):
+		sql = "select count(device_id) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id in \
+				(select device_id \
+				from {3}) \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type,table_name)
+		uid_clk_times = con_sql(sql)[0][0]
+		return uid_clk_times
+
+	def get_uid_imp_times(self):
+		sql = "select count(device_id) from data_feed_exposure \
+			where stat_date='{0}' \
+			and cid_type='{1}' \
+			and device_id regexp '[{2}]$' \
+			and device_id in \
+				(select device_id \
+				from {3}) \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type,table_name)
+		uid_imp_times = con_sql(sql)[0][0]
+		return uid_imp_times
+
+
+
+class AllStat(object):
+	def __init__(self, cid_type, uid_type, ndays=get_yesterday_date()):
+		"""
+		cid_type : diary，answer，question
+		uid_type : 8：_8结尾；6：_6结尾；6|8：_6或者_8结尾;^68：不是6或者8结尾的
+		ndays : '2018-08-30'....
+		"""
+		self.cid_type = cid_type
+		self.uid_type = uid_type
+		self.ndays = ndays
+
+	def get_uid_count(self):
+		sql = "select count(distinct(device_id)) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_count = con_sql(sql)[0][0]
+		return uid_count
+
+	def get_uid_clk_times(self):
+		sql = "select count(device_id) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_clk_times = con_sql(sql)[0][0]
+		return uid_clk_times
+
+	def get_uid_imp_times(self):
+		sql = "select count(device_id) from data_feed_exposure \
+			where stat_date='{0}' \
+			and cid_type='{1}' \
+			and device_id regexp '[{2}]$' \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_imp_times = con_sql(sql)[0][0]
+		return uid_imp_times
+
+
+def main():
+	output = OUTPUT_PATH + "ctr.csv"
+	with open(output,"a+") as f:
+		print("stat" + "  " + get_yesterday_date())
+		g_class = GrayStat("diary",num)
+		a_class = AllStat("diary",num)
+		line1 = str(g_class.get_uid_count())+"\t"+str(g_class.get_uid_imp_times())+"\t"+str(g_class.get_uid_clk_times())
+		line2 = str(a_class.get_uid_count())+"\t"+str(a_class.get_uid_imp_times())+"\t"+str(a_class.get_uid_clk_times())
+		g_ctr = g_class.get_uid_clk_times()/g_class.get_uid_imp_times()
+		a_ctr = a_class.get_uid_clk_times()/a_class.get_uid_imp_times()
+		growth_rate = (g_ctr-a_ctr)/a_ctr
+		line = get_yesterday_date() + "\t" + str(round(g_ctr*100,2))+'%' + "\t" + str(round(a_ctr*100,2))+'%' + "\t" + \
+		str(round(growth_rate*100,2))+'%' + "\t" + line1 + "\t" + line2 + "\n"
+		f.write(line)
+
+
+if __name__ == '__main__':
+	main()
\ No newline at end of file

eda/gray_stat/node2vec_ctr.py

 # -*- coding: UTF-8 -*-
-import pymysql
-import datetime
-import pandas as pd
-
-DIRECTORY_PATH="/data2/ffm/"
-
-def get_yesterday_date():
-    #自动获取昨天的日期,如"2018-08-08"
-    """
-    :rtype : str
-    """
-    today = datetime.date.today()
-    yesterday = today - datetime.timedelta(days=1)
-    yesterday = yesterday.strftime("%Y-%m-%d")
-    print(yesterday)
-    return yesterday
-    #today = datetime.date.today().strftime("%Y%m%d")
-    #return today
-
-
-def get_data():
-    conn2db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_prod')
-    cursor = conn2db.cursor()
-    sql = "select device_id from nd_device_cid_similarity_matrix where device_id regexp '[3|4]$'"
-    cursor.execute(sql)
-    result = cursor.fetchall()
-    device_id = tuple(pd.DataFrame(list(result))[0].values.tolist())
-    cursor.close()
-    return device_id
-
-
-def ctr(date):
-    device_id = get_data()
-    sql_click = "select count(cid) from data_feed_click " \
-              "where cid_type = 'diary' " \
-                "and stat_date = '{}' and device_id in {};".format(date,device_id)
-    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_prod')
-    cursor = db.cursor()
-    cursor.execute(sql_click)
-    click = cursor.fetchone()[0]
-    print("点击数："+str(click))
-    sql_exp = "select count(cid) from data_feed_exposure " \
-              "where cid_type = 'diary' and stat_date = '{}' and " \
-              "device_id in {}".format(date,device_id)
-    cursor.execute(sql_exp)
-    exp = cursor.fetchone()[0]
-    print("曝光数："+str(exp))
-    if exp != 0:
-        print("点击率："+str(click/exp))
-
-    return click,exp,click/exp
-
-def rate2file():
-    output_path = DIRECTORY_PATH + "node2vec_ctr.csv"
-    with open(output_path,'a+') as f:
-        line = get_yesterday_date().replace('-', '')+','+str(temp_data[0])+','+str(temp_data[1])+','+str(temp_data[2])+'\n'
-        f.write(line)
-
-if __name__ == "__main__":
-    #ctr(date)
-    date = get_yesterday_date()
-    temp_data = ctr(date)
-    rate2file()
+from utils import con_sql,get_yesterday_date,get_between_day
+import time
+
+OUTPUT_PATH = "/data2/models/eda/node2vec/"
+
+
+class GrayStat(object):
+	def __init__(self, cid_type, uid_type, ndays=get_yesterday_date()):
+		"""
+		cid_type : diary，answer，question
+		uid_type : 8：_8结尾；6：_6结尾；6|8：_6或者_8结尾;^68：不是6或者8结尾的
+		ndays : '2018-08-30'....
+		"""
+		self.cid_type = cid_type
+		self.uid_type = uid_type
+		self.ndays = ndays
+
+	def get_uid_count(self):
+		sql = "select count(distinct(device_id)) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id in \
+				(select device_id \
+				from nd_device_cid_similarity_matrix_tmp) \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_count = con_sql(sql)[0][0]
+		return uid_count
+
+	def get_uid_clk_times(self):
+		sql = "select count(device_id) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id in \
+				(select device_id \
+				from nd_device_cid_similarity_matrix_tmp) \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_clk_times = con_sql(sql)[0][0]
+		return uid_clk_times
+
+	def get_uid_imp_times(self):
+		sql = "select count(device_id) from data_feed_exposure \
+			where stat_date='{0}' \
+			and cid_type='{1}' \
+			and device_id regexp '[{2}]$' \
+			and device_id in \
+				(select device_id \
+				from nd_device_cid_similarity_matrix_tmp) \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_imp_times = con_sql(sql)[0][0]
+		return uid_imp_times
+
+
+
+class AllStat(object):
+	def __init__(self, cid_type, uid_type, ndays=get_yesterday_date()):
+		"""
+		cid_type : diary，answer，question
+		uid_type : 8：_8结尾；6：_6结尾；6|8：_6或者_8结尾;^68：不是6或者8结尾的
+		ndays : '2018-08-30'....
+		"""
+		self.cid_type = cid_type
+		self.uid_type = uid_type
+		self.ndays = ndays
+
+	def get_uid_count(self):
+		sql = "select count(distinct(device_id)) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_count = con_sql(sql)[0][0]
+		return uid_count
+
+	def get_uid_clk_times(self):
+		sql = "select count(device_id) from data_feed_click \
+			where stat_date='{0}' \
+			and (cid_type='{1}' or cid_type='diary_video') \
+			and device_id regexp '[{2}]$' \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_clk_times = con_sql(sql)[0][0]
+		return uid_clk_times
+
+	def get_uid_imp_times(self):
+		sql = "select count(device_id) from data_feed_exposure \
+			where stat_date='{0}' \
+			and cid_type='{1}' \
+			and device_id regexp '[{2}]$' \
+			and device_id not in (select distinct(device_id) from jerry_test.bl_device_list) \
+			and device_id not in (select distinct(device_id) from jerry_prod.blacklist)".format(self.ndays,\
+				self.cid_type,self.uid_type)
+		uid_imp_times = con_sql(sql)[0][0]
+		return uid_imp_times
+
+
+def main():
+	output = OUTPUT_PATH + "ctr.csv"
+	with open(output,"a+") as f:
+		print("stat" + "  " + get_yesterday_date())
+		g_class = GrayStat("diary","3|4")
+		a_class = AllStat("diary","3|4")
+		line1 = str(g_class.get_uid_count())+","+str(g_class.get_uid_imp_times())+","+str(g_class.get_uid_clk_times())
+		line2 = str(a_class.get_uid_count())+","+str(a_class.get_uid_imp_times())+","+str(a_class.get_uid_clk_times())
+		g_ctr = g_class.get_uid_clk_times()/g_class.get_uid_imp_times()
+		a_ctr = a_class.get_uid_clk_times()/a_class.get_uid_imp_times()
+		growth_rate = (g_ctr-a_ctr)/a_ctr
+		line = get_yesterday_date() + "," + str(round(g_ctr*100,2))+'%' + "," + str(round(a_ctr*100,2))+'%' + "," + \
+		str(round(growth_rate*100,2))+'%' + "," + line1 + "," + line2 + "\n"
+		f.write(line)
+
+
+if __name__ == '__main__':
+	main()
\ No newline at end of file