update

task/search_meigou_ctr.py

@@ -20,13 +20,10 @@ import time
 from pyspark import SparkConf
 from pyspark.sql import SparkSession, DataFrame
 
+
 # from pyspark.sql.functions import lit
 # import pytispark.pytispark as pti
 
-db = pymysql.connect(host='172.16.40.158', port=4000, user='st_user', passwd='aqpuBLYzEV7tML5RPsN1pntUzFy',
-                     db='jerry_prod')
-cursor = db.cursor()
-
 
 def con_sql(sql):
     # 从数据库的表里获取数据
@@ -297,57 +294,56 @@ and (dev.device_id is null or dev.device_id='')
 
     GROUP by D.DEVICE_OS_TYPE,
         D.ACTIVE_TYPE
-    """.format(partition_day=yesterday_str,end_date=today_str )
+    """.format(partition_day=yesterday_str, end_date=today_str)
 
     print(sql_search_ctr)
     search_ctr_df = spark.sql(sql_search_ctr)
     # spam_pv_df.createOrReplaceTempView("dev_view")
     search_ctr_df.show(1)
     sql_res = search_ctr_df.collect()
+    res_dict = {
+        "新增": {
+            "ios": {"click_num": 0, "exposure": 0},
+            "android": {"click_num": 0, "exposure": 0}
+        },
+        "老活": {
+            "ios": {"click_num": 0, "exposure": 0},
+            "android": {"click_num": 0, "exposure": 0}
+        }
+    }
     print("-------------------------------")
-
+    db = pymysql.connect(host='172.16.40.158', port=4000, user='st_user', passwd='aqpuBLYzEV7tML5RPsN1pntUzFy',
+                         db='jerry_prod')
+    cursor = db.cursor()
     for res in sql_res:
-        print(res)
-    print("-------------------------------")
-
-#     for res in sql_res:
-#         # print(res)
-#         day_id = res.day_id
-#         device_os_type = res.device_os_type
-#         active_type = res.active_type
-#         grey_type = res.grey_type
-#         page_name = res.page_name
-#         content_pv = res.content_pv
-#         content_uv = res.content_uv
-#         wel_exp_pv = res.wel_exp_pv
-#         content_exp_pv = res.content_exp_pv
-#         meigou_ctr=res.meigou_ctr
-#         if not meigou_ctr: meigou_ctr = 0
-#         grey_meigou_ctr=res.grey_meigou_ctr
-#         neirong_ctr=res.neirong_ctr
-#         if not neirong_ctr: neirong_ctr = 0
-#         grey_neirong_ctr=res.grey_neirong_ctr
-#
-#         wel_click_pv = res.wel_click_pv
-#         content_click_pv = res.content_click_pv
-#         slide_wel_click_pv = res.slide_wel_click_pv
-#         self_wel_click_pv = res.self_wel_click_pv
-#         partition_day = res.PARTITION_DAY
-#         pid = hashlib.md5((day_id + device_os_type + active_type + grey_type + page_name).encode("utf8")).hexdigest()
-#         instert_sql = """replace into conent_detail_page_grayscale_ctr(
-#         day_id,device_os_type,active_type,grey_type,page_name,content_pv,content_uv,wel_exp_pv,
-#         content_exp_pv,wel_click_pv,content_click_pv,slide_wel_click_pv,self_wel_click_pv,partition_day,pid,meigou_ctr,neirong_ctr,
-#         grey_meigou_ctr,grey_neirong_ctr) VALUES('{day_id}','{device_os_type}','{active_type}','{grey_type}','{page_name}',{content_pv},{content_uv},
-#         {wel_exp_pv},{content_exp_pv},{wel_click_pv},{content_click_pv},{slide_wel_click_pv},{self_wel_click_pv},'{partition_day}','{pid}',{meigou_ctr},{neirong_ctr},{grey_meigou_ctr},{grey_neirong_ctr});""".format(
-#             day_id=day_id,device_os_type=device_os_type,active_type=active_type,grey_type=grey_type,page_name=page_name,
-#             content_pv=content_pv,content_uv=content_uv,wel_exp_pv=wel_exp_pv,content_exp_pv=content_exp_pv,wel_click_pv=wel_click_pv,
-#             content_click_pv=content_click_pv,slide_wel_click_pv=slide_wel_click_pv,self_wel_click_pv=self_wel_click_pv,meigou_ctr=meigou_ctr,neirong_ctr=neirong_ctr,
-#             partition_day=partition_day, pid=pid,grey_neirong_ctr=grey_neirong_ctr,grey_meigou_ctr=grey_meigou_ctr
-#         )
-#         print(instert_sql)
-#         # cursor.execute("set names 'UTF8'")
-#         res = cursor.execute(instert_sql)
-#         db.commit()
-#         print(res)
-# # cursor.executemany()
+        # print(res)
+        if res.active_type:
+            if res.active_type in (1, 2):
+                res_dict["新增"][res.device_os_type]["click_num"] += res.click_num
+                res_dict["新增"][res.device_os_type]["exposure"] += res.exposure
+            else:
+                res_dict["老活"][res.device_os_type]["click_num"] += res.click_num
+                res_dict["老活"][res.device_os_type]["exposure"] += res.exposure
+
+    for active_type in res_dict:
+        for device_os_type in res_dict[active_type]:
+            day_id = yesterday_str
+            pid = hashlib.md5((day_id + device_os_type + active_type).encode("utf8")).hexdigest()
+            click_num = res_dict[active_type][device_os_type]["click_num"]
+            exposure = res_dict[active_type][device_os_type]["exposure"]
+            try:
+                search_ctr = round(click_num / exposure, 5)
+            except:
+                search_ctr = 0
+            instert_sql = """replace into search_meigou_ctr(
+            day_id,device_os_type,active_type,pid,click_num,exposure,search_ctr) VALUES('{day_id}','{device_os_type}','{active_type}','{pid}',{click_num},{exposure},{search_ctr});""".format(
+                day_id=day_id, device_os_type=device_os_type, active_type=active_type, pid=pid, click_num=click_num,
+                exposure=exposure, search_ctr=search_ctr
+            )
+            print(instert_sql)
+            # cursor.execute("set names 'UTF8'")
+            res = cursor.execute(instert_sql)
+            db.commit()
+            print(res)
+# cursor.executemany()
 db.close()