update

task/search_tractate_ctr.py

@@ -73,7 +73,7 @@ spark.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJso
 spark.sql("CREATE TEMPORARY FUNCTION arrayMerge AS 'com.gmei.hive.common.udf.UDFArryMerge'")
 
 task_list = []
-task_days = 3
+task_days = 60
 for t in range(1, task_days):
     day_num = 0 - t
     now = (datetime.datetime.now() + datetime.timedelta(days=day_num))
@@ -117,6 +117,59 @@ for t in range(1, task_days):
               group by partition_day,card_content_type,device_id
             )t1
             LEFT JOIN
+            (
+                SELECT partition_date,device_id
+                FROM 
+                (--找出user_id当天活跃的第一个设备id
+                    SELECT user_id,partition_date,
+                            if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
+                      FROM online.ml_user_updates
+                      WHERE partition_date>='{partition_day}' AND partition_date<'{end_date}'
+                )t1
+                JOIN
+                (  --医生账号
+                    SELECT distinct user_id
+                    FROM online.tl_hdfs_doctor_view
+                    WHERE partition_date = '{partition_day}'
+            
+                    --马甲账号/模特用户
+                    UNION ALL
+                    SELECT user_id
+                    FROM ml.ml_c_ct_ui_user_dimen_d
+                    WHERE partition_day = '{partition_day}'
+                    AND (is_puppet = 'true' or is_classifyuser = 'true')
+            
+                    UNION ALL
+                    --公司内网覆盖用户
+                    select distinct user_id
+                    from dim.dim_device_user_staff
+            
+                    UNION ALL
+                    --登陆过医生设备
+                    SELECT distinct t1.user_id
+                    FROM
+                    (
+                        SELECT user_id, v.device_id as device_id
+                        FROM online.ml_user_history_detail
+                            LATERAL VIEW EXPLODE(device_history_list) v AS device_id
+                        WHERE partition_date = '{partition_day}'
+                    )t1
+                    JOIN
+                    (
+                        SELECT device_id
+                        FROM online.ml_device_history_detail
+                        WHERE partition_date = '{partition_day}'
+                        AND is_login_doctor = '1'
+                    )t2
+                        ON t1.device_id = t2.device_id
+                )t2
+                on t1.user_id=t2.user_id
+                group by partition_date,device_id
+            )dev 
+            on t1.device_id=dev.device_id
+            WHERE (t1.device_id IS NULL or t1.device_id = '')
+            and (dev.device_id is null or dev.device_id='')
+            LEFT JOIN
             (--搜索结果页卡片点击
               SELECT cl_id,partition_date
                   ,sum(CASE WHEN card_content_type='service' THEN click_pv END) as service_click_pv
@@ -174,7 +227,6 @@ for t in range(1, task_days):
         )dev_channel
           on dev_channel.device_id = exp.device_id
             AND dev_channel.partition_date = exp.partition_date
-
         GROUP BY exp.partition_date,active_type,device_os_type
     """.format(partition_day=yesterday_str, end_date=today_str)
 
@@ -186,30 +238,30 @@ for t in range(1, task_days):
 
     print("-------------------------------")
 
-
-    for active_type in sql_res:
-        print(active_type)
-#         for device_os_type in sql_res[active_type]:
-#             partition_date = yesterday_str
-#             pid = hashlib.md5((partition_date + device_os_type + active_type).encode("utf8")).hexdigest()
-#             click_num = sql_res[active_type][device_os_type]["click_num"]
-#             exposure = sql_res[active_type][device_os_type]["exposure"]
-#             try:
-#                 search_ctr = round(click_num / exposure, 5)
-#             except:
-#                 search_ctr = 0
-#             instert_sql = """replace into search_tractate_ctr(
-#             partition_date,device_os_type,active_type,pid,click_num,exposure,search_ctr) VALUES('{partition_date}','{device_os_type}','{active_type}','{pid}',{click_num},{exposure},{search_ctr});""".format(
-#                 partition_date=partition_date, device_os_type=device_os_type, active_type=active_type, pid=pid, click_num=click_num,
-#                 exposure=exposure, search_ctr=search_ctr
-#             )
-#             print(instert_sql)
-#             # cursor.execute("set names 'UTF8'")
-#             db = pymysql.connect(host='172.16.40.158', port=4000, user='st_user', passwd='aqpuBLYzEV7tML5RPsN1pntUzFy',
-#                                  db='jerry_prod')
-#             cursor = db.cursor()
-#             res = cursor.execute(instert_sql)
-#             db.commit()
-#             print(res)
-# # cursor.executemany()
-# db.close()
+    for res in sql_res:
+        print(res)
+        device_os_type = res.device_os_type
+        active_type = res.active_type
+        partition_date = yesterday_str
+        pid = hashlib.md5((partition_date + device_os_type + active_type).encode("utf8")).hexdigest()
+        click_num = res.neirong_click_pv
+        exposure = res.neirong_exp_pv
+        try:
+            search_ctr = round(click_num / exposure, 5)
+        except:
+            search_ctr = 0
+        instert_sql = """replace into search_tractate_ctr(
+        partition_date,device_os_type,active_type,pid,click_num,exposure,search_ctr) VALUES('{partition_date}','{device_os_type}','{active_type}','{pid}',{click_num},{exposure},{search_ctr});""".format(
+            partition_date=partition_date, device_os_type=device_os_type, active_type=active_type, pid=pid, click_num=click_num,
+            exposure=exposure, search_ctr=search_ctr
+        )
+        print(instert_sql)
+        # cursor.execute("set names 'UTF8'")
+        db = pymysql.connect(host='172.16.40.158', port=4000, user='st_user', passwd='aqpuBLYzEV7tML5RPsN1pntUzFy',
+                             db='jerry_prod')
+        cursor = db.cursor()
+        res = cursor.execute(instert_sql)
+        db.commit()
+        print(res)
+# cursor.executemany()
+db.close()