Skip to content

M
meta_base_code
Commit 0c081a62 authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent c1226b15
Hide whitespace changes
Inline Side-by-side
Showing with 45 additions and 49 deletions
task/search_meigou_ctr.py
View file @ 0c081a62
...@@ -20,13 +20,10 @@ import time ...@@ -20,13 +20,10 @@ import time
from pyspark import SparkConf from pyspark import SparkConf
from pyspark.sql import SparkSession, DataFrame from pyspark.sql import SparkSession, DataFrame
# from pyspark.sql.functions import lit # from pyspark.sql.functions import lit
# import pytispark.pytispark as pti # import pytispark.pytispark as pti
db = pymysql.connect(host='172.16.40.158', port=4000, user='st_user', passwd='aqpuBLYzEV7tML5RPsN1pntUzFy',
db='jerry_prod')
cursor = db.cursor()
def con_sql(sql): def con_sql(sql):
# 从数据库的表里获取数据 # 从数据库的表里获取数据
...@@ -297,57 +294,56 @@ and (dev.device_id is null or dev.device_id='') ...@@ -297,57 +294,56 @@ and (dev.device_id is null or dev.device_id='')
GROUP by D.DEVICE_OS_TYPE, GROUP by D.DEVICE_OS_TYPE,
D.ACTIVE_TYPE D.ACTIVE_TYPE
""".format(partition_day=yesterday_str,end_date=today_str ) """.format(partition_day=yesterday_str, end_date=today_str)
print(sql_search_ctr) print(sql_search_ctr)
search_ctr_df = spark.sql(sql_search_ctr) search_ctr_df = spark.sql(sql_search_ctr)
# spam_pv_df.createOrReplaceTempView("dev_view") # spam_pv_df.createOrReplaceTempView("dev_view")
search_ctr_df.show(1) search_ctr_df.show(1)
sql_res = search_ctr_df.collect() sql_res = search_ctr_df.collect()
res_dict = {
"新增": {
"ios": {"click_num": 0, "exposure": 0},
"android": {"click_num": 0, "exposure": 0}
},
"老活": {
"ios": {"click_num": 0, "exposure": 0},
"android": {"click_num": 0, "exposure": 0}
}
}
print("-------------------------------") print("-------------------------------")
db = pymysql.connect(host='172.16.40.158', port=4000, user='st_user', passwd='aqpuBLYzEV7tML5RPsN1pntUzFy',
db='jerry_prod')
cursor = db.cursor()
for res in sql_res: for res in sql_res:
print(res) # print(res)
print("-------------------------------") if res.active_type:
if res.active_type in (1, 2):
# for res in sql_res: res_dict["新增"][res.device_os_type]["click_num"] += res.click_num
# # print(res) res_dict["新增"][res.device_os_type]["exposure"] += res.exposure
# day_id = res.day_id else:
# device_os_type = res.device_os_type res_dict["老活"][res.device_os_type]["click_num"] += res.click_num
# active_type = res.active_type res_dict["老活"][res.device_os_type]["exposure"] += res.exposure
# grey_type = res.grey_type
# page_name = res.page_name for active_type in res_dict:
# content_pv = res.content_pv for device_os_type in res_dict[active_type]:
# content_uv = res.content_uv day_id = yesterday_str
# wel_exp_pv = res.wel_exp_pv pid = hashlib.md5((day_id + device_os_type + active_type).encode("utf8")).hexdigest()
# content_exp_pv = res.content_exp_pv click_num = res_dict[active_type][device_os_type]["click_num"]
# meigou_ctr=res.meigou_ctr exposure = res_dict[active_type][device_os_type]["exposure"]
# if not meigou_ctr: meigou_ctr = 0 try:
# grey_meigou_ctr=res.grey_meigou_ctr search_ctr = round(click_num / exposure, 5)
# neirong_ctr=res.neirong_ctr except:
# if not neirong_ctr: neirong_ctr = 0 search_ctr = 0
# grey_neirong_ctr=res.grey_neirong_ctr instert_sql = """replace into search_meigou_ctr(
# day_id,device_os_type,active_type,pid,click_num,exposure,search_ctr) VALUES('{day_id}','{device_os_type}','{active_type}','{pid}',{click_num},{exposure},{search_ctr});""".format(
# wel_click_pv = res.wel_click_pv day_id=day_id, device_os_type=device_os_type, active_type=active_type, pid=pid, click_num=click_num,
# content_click_pv = res.content_click_pv exposure=exposure, search_ctr=search_ctr
# slide_wel_click_pv = res.slide_wel_click_pv )
# self_wel_click_pv = res.self_wel_click_pv print(instert_sql)
# partition_day = res.PARTITION_DAY # cursor.execute("set names 'UTF8'")
# pid = hashlib.md5((day_id + device_os_type + active_type + grey_type + page_name).encode("utf8")).hexdigest() res = cursor.execute(instert_sql)
# instert_sql = """replace into conent_detail_page_grayscale_ctr( db.commit()
# day_id,device_os_type,active_type,grey_type,page_name,content_pv,content_uv,wel_exp_pv, print(res)
# content_exp_pv,wel_click_pv,content_click_pv,slide_wel_click_pv,self_wel_click_pv,partition_day,pid,meigou_ctr,neirong_ctr, # cursor.executemany()
# grey_meigou_ctr,grey_neirong_ctr) VALUES('{day_id}','{device_os_type}','{active_type}','{grey_type}','{page_name}',{content_pv},{content_uv},
# {wel_exp_pv},{content_exp_pv},{wel_click_pv},{content_click_pv},{slide_wel_click_pv},{self_wel_click_pv},'{partition_day}','{pid}',{meigou_ctr},{neirong_ctr},{grey_meigou_ctr},{grey_neirong_ctr});""".format(
# day_id=day_id,device_os_type=device_os_type,active_type=active_type,grey_type=grey_type,page_name=page_name,
# content_pv=content_pv,content_uv=content_uv,wel_exp_pv=wel_exp_pv,content_exp_pv=content_exp_pv,wel_click_pv=wel_click_pv,
# content_click_pv=content_click_pv,slide_wel_click_pv=slide_wel_click_pv,self_wel_click_pv=self_wel_click_pv,meigou_ctr=meigou_ctr,neirong_ctr=neirong_ctr,
# partition_day=partition_day, pid=pid,grey_neirong_ctr=grey_neirong_ctr,grey_meigou_ctr=grey_meigou_ctr
# )
# print(instert_sql)
# # cursor.execute("set names 'UTF8'")
# res = cursor.execute(instert_sql)
# db.commit()
# print(res)
# # cursor.executemany()
db.close() db.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment