try get data

eda/smart_rank/tag3_update_user_portrait_offline.py

@@ -219,29 +219,30 @@ def update_tag3_user_portrait(cl_id):
         second_solutions_score = merge_values(second_solutions_list)
         first_positions_score = merge_values(first_positions_list)
         second_positions_score = merge_values(second_positions_list)
-        projects_score_tmp = merge_values(projects_list)
-
-        tmp = [
-            "不感兴趣", "没有想法", "八卦来了", "颜值速报", "医美审美干货", "其他项目", "网红颜值", "少年之名", "郑人予", "热点课代表", "私密", "陈瑞泽", "符仁杰", "祖力亚尔", "刘泽旭",
-            "陈家瑨", "娱乐八卦", "美白1", "情感", "颜安", "苏勋伦", "丰唇1", "牙齿矫正", "无创溶脂"
-        ]
-        forbidden_lst = ["无创溶脂kyc"]
-        for i in tmp:
-            forbidden_lst.append(i + "kyc")
-        forbidden_lst.extend(tmp)
-        # forbidden_set = set(forbidden_lst)
-
-        forbidden_set = set()
-
-        projects_score = {}
-        for (k, v) in projects_score_tmp.items():
-            if k not in forbidden_set:
-                projects_score[k] = v
-
-        tmp_res = {}
-        if (len(first_demands_score) == 0) and (len(first_solutions_score) == 0) and (len(first_positions_score) == 0) and (
-                len(second_demands_score) == 0) and (len(second_solutions_score) == 0) and (len(second_positions_score) == 0):
-            tmp_res = _get_all_by_projects(list(projects_score.keys()))
+        projects_score = merge_values(projects_list)
+        # projects_score_tmp = merge_values(projects_list)
+
+        # tmp = [
+        #     "不感兴趣", "没有想法", "八卦来了", "颜值速报", "医美审美干货", "其他项目", "网红颜值", "少年之名", "郑人予", "热点课代表", "私密", "陈瑞泽", "符仁杰", "祖力亚尔", "刘泽旭",
+        #     "陈家瑨", "娱乐八卦", "美白1", "情感", "颜安", "苏勋伦", "丰唇1", "牙齿矫正", "无创溶脂"
+        # ]
+        # forbidden_lst = ["无创溶脂kyc"]
+        # for i in tmp:
+        #     forbidden_lst.append(i + "kyc")
+        # forbidden_lst.extend(tmp)
+        # # forbidden_set = set(forbidden_lst)
+
+        # forbidden_set = set()
+
+        # projects_score = {}
+        # for (k, v) in projects_score_tmp.items():
+        #     if k not in forbidden_set:
+        #         projects_score[k] = v
+
+        # tmp_res = {}
+        # if (len(first_demands_score) == 0) and (len(first_solutions_score) == 0) and (len(first_positions_score) == 0) and (
+        #         len(second_demands_score) == 0) and (len(second_solutions_score) == 0) and (len(second_positions_score) == 0):
+        #     tmp_res = _get_all_by_projects(list(projects_score.keys()))
 
         res = {
             "first_demands": first_demands_score,
@@ -252,8 +253,8 @@ def update_tag3_user_portrait(cl_id):
             "second_positions": second_positions_score,
             "projects": projects_score
         }
-        if tmp_res:
-            res.update(tmp_res)
+        # if tmp_res:
+        #     res.update(tmp_res)
 
         key = "doris:user_portrait:tag3:device_id:" + str(cl_id)
         redis_client = redis.StrictRedis.from_url("redis://:ReDis!GmTx*0aN9@172.16.40.173:6379")
@@ -307,9 +308,9 @@ def consume_kafka():
         # result.foreach(print)
         result.collect()
 
-        result2 = device_ids_lst_rdd.repartition(100).map(lambda x: update_tag3_user_portrait_by_event(x))
+        # result2 = device_ids_lst_rdd.repartition(100).map(lambda x: update_tag3_user_portrait_by_event(x))
         # result2.foreach(print)
-        result2.collect()
+        # result2.collect()
 
         spark.stop()
     except Exception as e: