update

task/tractate_analysis_in_7000.py

@@ -77,7 +77,7 @@ spark.sql("CREATE TEMPORARY FUNCTION arrayMerge AS 'com.gmei.hive.common.udf.UDF
 sql = """
 select card_id from strategy_content_exposure_index where card_content_type="user_post" and preciseexposure_num>=50 and ctr>=0.05 and avg_page_stay>=20 and create_day="2020-09-17"
 """
-second_demands_count_dict, tags_v3_count_dict, second_demands_card_id_list,tags_v3_card_id_list = get_user_post_from_mysql(sql)
+second_demands_count_dict, tags_v3_count_dict, second_demands_card_id_list,tags_v3_card_id_list,second_demands_tractate_dict,tags_v3_tractate_dict = get_user_post_from_mysql(sql)
 print(second_demands_count_dict,tags_v3_count_dict,second_demands_card_id_list,tags_v3_card_id_list)
 
 time.sleep(20)
@@ -281,9 +281,9 @@ for res in sql_res:
         session_pv = res.session_pv
         # card_id_set.update(card_id)
         if cl_id in second_demands_tag_dict:
-            if int(card_id) in second_demands_card_id_list:
-                print(cl_id, second_demands_tag_dict[cl_id])
-                for tag_id in second_demands_tag_dict[cl_id]:
+            if int(card_id) in second_demands_tractate_dict:
+                # print(cl_id, second_demands_tag_dict[card_id])
+                for tag_id in second_demands_tractate_dict[card_id]:
 
                     if tag_id in second_demands_id_count:
                         second_demands_id_count[tag_id][int(card_id)] = 1
@@ -297,9 +297,9 @@ for res in sql_res:
                 #     baoguang_dict[tag_id] = session_pv
 
         if cl_id in projects_demands_tag_dict:
-            if int(card_id) in tags_v3_card_id_list:
-                print(cl_id,projects_demands_tag_dict[cl_id])
-                for tag_id in projects_demands_tag_dict[cl_id]:
+            if int(card_id) in tags_v3_tractate_dict:
+                # print(cl_id,projects_demands_tag_dict[cl_id])
+                for tag_id in tags_v3_tractate_dict[card_id]:
                     if tag_id in projects_demands_id_count:
                         projects_demands_id_count[tag_id][int(card_id)] = 1
                     else:

utils/func_from_es_get_article.py

@@ -342,11 +342,15 @@ def get_user_post_from_mysql(sql):
     tags_v3_card_id_list = []
     second_demands_count_dict = {}
     tags_v3_count_dict = {}
+    second_demands_tractate_dict = {}
+    tags_v3_tractate_dict = {}
     sql_res = con_sql_doris_prod(sql)
     for card_id in sql_res:
         # print(type(card_id[0]),card_id)
-
         es_res = get_tractate_tags_from_es(card_id[0])
+        second_demands_tractate_dict[card_id[0]] = es_res["_source"]["second_demands"]
+        tags_v3_tractate_dict[card_id[0]] = es_res["_source"]["tags_v3"]
+
         for position in es_res["_source"]["second_demands"]:
             if position in second_demands_count_dict:
                 second_demands_count_dict[position] += 1
@@ -360,4 +364,4 @@ def get_user_post_from_mysql(sql):
                 tags_v3_count_dict[tag] = 1
                 tags_v3_card_id_list.append(int(card_id[0]))
     # print(card_id_list)
-    return second_demands_count_dict, tags_v3_count_dict, second_demands_card_id_list,tags_v3_card_id_list
+    return second_demands_count_dict, tags_v3_count_dict, second_demands_card_id_list,tags_v3_card_id_list,second_demands_tractate_dict,tags_v3_tractate_dict

utils/func_from_redis_get_portrait.py

@@ -44,7 +44,6 @@ def get_user_portrait_tag3_from_redis(device_id, limit_score=0):
             "projects": projects,
             "anecdote_tags": anecdote_tags
         }
-    pr
     return {}