Merge branch 'master' into 'test'

Master

See merge request !203

libs/es.py

@@ -147,7 +147,7 @@ class ESPerform(object):
         try:
             assert (es_cli is not None)
 
-            official_index_name = cls.get_official_index_name(sub_index_name)
+            official_index_name = cls.get_official_index_name(sub_index_name, "write")
             index_exists = es_cli.indices.exists(official_index_name)
             if not index_exists:
                 if not auto_create_index:
@@ -212,3 +212,34 @@ class ESPerform(object):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return {"total_count":0,"hits":[]}
+
+
+    @classmethod
+    def if_es_node_load_high(cls, es_cli):
+        try:
+            assert (es_cli is not None)
+
+            high_num = 0
+            es_nodes_list = list()
+            es_nodes_ori_info = es_cli.cat.nodes()
+            es_nodes_info_list = es_nodes_ori_info.split("\n")
+            for item in es_nodes_info_list:
+                try:
+                    item_list = item.split(" ")
+                    if len(item_list)>4:
+                        cpu_load = item_list[3]
+                        if int(cpu_load) > 60:
+                            high_num += 1
+                        es_nodes_list.append(int(cpu_load))
+                except:
+                    logging.error("catch exception,item:%s,err_msg:%s" % (str(item),traceback.format_exc()))
+                    return True
+
+            if high_num > 3:
+                logging.info("check es_nodes_load high,cpu load:%s" % str(es_nodes_info_list))
+                return True
+            else:
+                return False
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return True

search/utils/topic.py

@@ -135,6 +135,7 @@ class TopicUtils(object):
         try:
             attention_user_id_list = list()
             pick_user_id_list = list()
+            # same_group_id_list = list()
             same_pictorial_id_list = list()
             user_tag_list = list()
 
@@ -148,6 +149,9 @@ class TopicUtils(object):
                 pick_user_info_list = result_dict["hits"][0]["_source"]["pick_user_id_list"]
                 pick_user_id_list = [item["user_id"] for item in pick_user_info_list]
 
+                # same_group_user_info_list = result_dict["hits"][0]["_source"]["same_group_user_id_list"]
+                # same_group_id_list = [item["user_id"] for item in same_group_user_info_list]
+                # same_group_id_list = same_group_id_list[:100]
                 same_pictorial_user_info_list = result_dict["hits"][0]["_source"]["same_pictorial_user_id_list"]
                 same_pictorial_id_list = [item["user_id"] for item in same_pictorial_user_info_list]
                 same_pictorial_id_list = same_pictorial_id_list[:100]
@@ -205,6 +209,14 @@ class TopicUtils(object):
                         "weight": 2
                     }
                 )
+            # if len(same_group_id_list)>0:
+            #     functions_list.append(
+            #         {
+            #             "filter": {"bool": {
+            #                 "should": {"terms":{"user_id":same_group_id_list}}}},
+            #             "weight": 1
+            #         }
+            #     )
             if len(same_pictorial_id_list)>0:
                 functions_list.append(
                     {
@@ -296,7 +308,7 @@ class TopicUtils(object):
 
                 query_function_score["query"]["bool"]["should"] = [
                     {'multi_match': multi_match},
-                    {"terms":{"tag_list":tag_id}}
+                    {"term":{"tag_list":tag_id}}
                 ]
                 query_function_score["query"]["bool"]["minimum_should_match"] = 1
 
@@ -531,7 +543,7 @@ class TopicUtils(object):
                             #{"range": {"content_level": {"gte": 3, "lte": 5}}},
                             {"term": {"is_online": True}},
                             {"term": {"is_deleted": False}},
-                            {"terms": {"tag_list":tag_id}}
+                            {"term": {"tag_list":tag_id}}
                         ],
                         "must_not":[
                             {"terms": {"content_level": [1,2]}}

search/views/topic.py

@@ -167,7 +167,29 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
         if not isinstance(device_id, str):
             device_id = ""
 
-        recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
+        recommend_topic_ids = list()
+
+        es_node_load_high_flag = False
+        try:
+            es_node_load_high_flag = ESPerform.if_es_node_load_high(ESPerform.get_cli())
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            es_node_load_high_flag = True
+
+        if es_node_load_high_flag:
+            temp_downgrading_key = "physical:home_recommend:user_id:241407656:query_type:1"
+            redis_field_list = [b'have_read_topic_list']
+            redis_field_val_list = redis_client.hmget(temp_downgrading_key, redis_field_list)
+            if redis_field_val_list[0]:
+                have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
+
+                if len(have_read_topic_id_list) > offset:
+                    recommend_topic_ids = have_read_topic_id_list[offset:offset+size]
+                else:
+                    recommend_topic_ids = have_read_topic_id_list[0:size]
+
+        else:
+            recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
                                                            query_type=query_type)
 
         return {"recommend_topic_ids": recommend_topic_ids}