update

task/spark_test.py

@@ -90,13 +90,15 @@ for t in range(2, task_days):
     yesterday_str_format = (now + datetime.timedelta(days=-1)).strftime("%Y-%m-%d")
     one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d")
     new_urser_device_id_sql = r"""
-SELECT *  FROM online.bl_hdfs_maidian_updates
-                         WHERE partition_date >= '{partition_day}'
-                           AND partition_date < '{end_date}'
-                           AND ((action in ('on_click_topic_card','on_click_diary_card','search_result_click_infomation_item')
-                               AND page_name in ('search_result_more','search_result_diary','search_result_post'))
-                               or (action = 'on_click_card' AND params['card_content_type'] in ('answer','diary') AND page_name in ('search_result_more','search_result_diary','search_result_question_answer')))
-    """.format(partition_day="20210224", end_date="20210225", tomorrow_str=tomorrow_str)
+    SELECT *
+                  FROM online.bl_hdfs_maidian_updates
+                  WHERE partition_date >= '{partition_day}'
+                      AND partition_date < '{end_date}'
+                  AND ((action in ('search_result_click_recommend_item','search_result_welfare_click_item')
+                      AND page_name in ('search_result_more','search_result_welfare'))
+                   or (action = 'goto_welfare_detail' AND params ['from'] = 'search_result_welfare_recommend')
+                   or (action = 'on_click_card' AND params['card_content_type'] in ('service') AND page_name in ('search_result_more','search_result_welfare')))
+       """.format(partition_day="20210224", end_date="20210225", tomorrow_str=tomorrow_str)
 
     #   print(new_urser_device_id_sql)
     new_urser_device_id_df = spark.sql(new_urser_device_id_sql)
@@ -106,16 +108,16 @@ SELECT *  FROM online.bl_hdfs_maidian_updates
     res_list = []
     for res in sql_res:
         print(res)
-        query = res.params.get("query","")
-        card_name = res.params.get("card_name","")
+        query = res.params.get("query","").replace("\r","").replace("\n","")
+        card_name = res.params.get("card_name","").replace("\r","").replace("\n","")
         card_id = res.params.get("card_id","")
-        user_id = res.user_id
+        cl_id = res.cl_id
         time_str = res.time_str
         page_name = res.page_name
         res_list.append({"query": query,
                          "card_name": card_name,
                          "card_id": card_id,
-                         "user_id": user_id,
+                         "cl_id": cl_id,
                          "time_str": time_str,
                          "page_name": page_name
                          })