Merge branch 'offic' into 'master'

update field

See merge request !61

eda/smart_rank/tag3_update_user_portrait_offline.py

@@ -91,17 +91,18 @@ def update_tag3_user_portrait_by_event(cl_id):
             list(second_demands_dict.keys()) + list(first_positions_dict.keys()) + list(second_positions_dict.keys()) +
             list(projects_dict.keys()))
 
-        for e in events:
-            first_solutions = ",".join(first_solutions_dict.get(e, []))
-            second_solutions = ",".join(second_solutions_dict.get(e, []))
-            first_demands = ",".join(first_demands_dict.get(e, []))
-            second_demands = ",".join(second_demands_dict.get(e, []))
-            first_positions = ",".join(first_positions_dict.get(e, []))
-            second_positions = ",".join(second_positions_dict.get(e, []))
-            projects = ",".join(projects_dict.get(e, []))
-
-            write_user_portrait_by_event(cl_id, first_solutions, second_solutions, first_demands, second_demands, first_positions,
-                                         second_positions, projects, e)
+        # TODO
+        # for e in events:
+        #     first_solutions = ",".join(first_solutions_dict.get(e, []))
+        #     second_solutions = ",".join(second_solutions_dict.get(e, []))
+        #     first_demands = ",".join(first_demands_dict.get(e, []))
+        #     second_demands = ",".join(second_demands_dict.get(e, []))
+        #     first_positions = ",".join(first_positions_dict.get(e, []))
+        #     second_positions = ",".join(second_positions_dict.get(e, []))
+        #     projects = ",".join(projects_dict.get(e, []))
+
+        #     write_user_portrait_by_event(cl_id, first_solutions, second_solutions, first_demands, second_demands, first_positions,
+        #                                  second_positions, projects, e)
 
         return cl_id
 
@@ -140,6 +141,18 @@ def update_tag3_user_portrait(cl_id):
         first_positions_score = merge_values(first_positions_list)
         second_positions_score = merge_values(second_positions_list)
         projects_score = merge_values(projects_list)
+        print(projects_score)
+
+        tmp = [
+            "不感兴趣", "没有想法", "八卦来了", "颜值速报", "医美审美干货", "其他项目", "网红颜值", "少年之名", "郑人予", "热点课代表", "私密", "陈瑞泽", "符仁杰", "祖力亚尔", "刘泽旭", "陈家瑨"
+        ]
+        forbidden_lst = []
+        for i in tmp:
+            forbidden_lst.append(i + "kyc")
+        forbidden_lst.extend(tmp)
+        forbidden_set = set(forbidden_lst)
+
+        # TODO projects_score remove forbidden_set
 
         res = {
             "first_demands": first_demands_score,
@@ -154,17 +167,18 @@ def update_tag3_user_portrait(cl_id):
         key = "doris:user_portrait:tag3:device_id:" + str(cl_id)
         redis_client = redis.StrictRedis.from_url("redis://:ReDis!GmTx*0aN9@172.16.40.173:6379")
 
-        if (len(first_demands_score.keys()) > 0) or (len(second_demands_score.keys()) > 0) or \
-           (len(first_solutions_score.keys()) > 0) or (len(second_solutions_score.keys()) > 0) or \
-           (len(first_positions_score.keys()) > 0) or (len(second_positions_score.keys()) > 0) or \
-           (len(projects_score.keys()) > 0):
-            redis_client.set(key, json.dumps(res))
-            redis_client.expire(key, 60 * 60 * 24 * 30)
+        # TODO
+        # if (len(first_demands_score.keys()) > 0) or (len(second_demands_score.keys()) > 0) or \
+        #    (len(first_solutions_score.keys()) > 0) or (len(second_solutions_score.keys()) > 0) or \
+        #    (len(first_positions_score.keys()) > 0) or (len(second_positions_score.keys()) > 0) or \
+        #    (len(projects_score.keys()) > 0):
+        #     redis_client.set(key, json.dumps(res))
+        #     redis_client.expire(key, 60 * 60 * 24 * 30)
 
-            write_user_portrait(cl_id, ",".join(first_solutions_score.keys()), ",".join(second_solutions_score.keys()),
-                                ",".join(first_demands_score.keys()), ",".join(second_demands_score.keys()),
-                                ",".join(first_positions_score.keys()), ",".join(second_positions_score.keys()),
-                                ",".join(projects_score.keys()))
+        #     write_user_portrait(cl_id, ",".join(first_solutions_score.keys()), ",".join(second_solutions_score.keys()),
+        #                         ",".join(first_demands_score.keys()), ",".join(second_demands_score.keys()),
+        #                         ",".join(first_positions_score.keys()), ",".join(second_positions_score.keys()),
+        #                         ",".join(projects_score.keys()))
 
         return cl_id
 
@@ -177,7 +191,8 @@ def consume_kafka():
     db.close()
     cursor.close()
 
-    # device_ids_lst = ["androidid_a25a1129c0b38f7b"]
+    # TODO
+    device_ids_lst = ["androidid_a25a1129c0b38f7b"]
 
     sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true") \
                            .set("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive", "true") \