modify

libs/table_scan.py

@@ -150,24 +150,17 @@ class TableSlicer(object):
 
     def __init__(self, queryset, chunk_size=None, chunk_count=None, sep_list=None):
         try:
-            logging.info("duan add,before assert queryset")
             assert isinstance(queryset, models.QuerySet)
 
-            logging.info("duan add,before assert chunk_size")
             assert chunk_size is None or isinstance(chunk_size, six.integer_types)
 
-            logging.info("duan add,before assert chunk_count")
             assert chunk_count is None or isinstance(chunk_count, six.integer_types)
 
-            logging.info("duan add,before assert sep_list")
             assert sep_list is None or isinstance(sep_list, list)
 
-            logging.info("duan add,before assert chunk_size")
             assert (chunk_size is not None) + (chunk_count is not None) + (sep_list is not None) == 1
-            logging.info("duan add,after assert chunk_size")
 
 
-            logging.info("duan add,sep_list:%s" % str(sep_list))
             if sep_list is not None:
                 sep_list = list(sep_list)
             else:
@@ -180,7 +173,6 @@ class TableSlicer(object):
                     for index in index_list
                 ]
 
-                logging.info("duan add,queryset count:%d" % count)
             self._model = queryset.model
             self._query = queryset.query
             self._sep_list = [None] + sep_list + [None]

search/utils/topic.py

@@ -339,7 +339,7 @@ class TopicUtils(object):
             return dict()
 
     @classmethod
-    def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size,es_cli_obj=None):
+    def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,have_read_topic_list,offset,size,es_cli_obj=None):
         """
         :remark 帖子详情页推荐列表,缺少按时间衰减
         :param user_id:
@@ -381,7 +381,7 @@ class TopicUtils(object):
                     }
                 )
 
-            # query_tag_term_list = cls.___get_should_term_list(topic_tag_list)
+            have_read_topic_list.append(topic_id)
             query_function_score = {
                 "query":{
                     "bool":{
@@ -391,8 +391,8 @@ class TopicUtils(object):
                             {"term": {"is_deleted": False}}
                         ],
                         "must_not":{
-                            "term":{
-                                "id":topic_id
+                            "terms":{
+                                "id":have_read_topic_list
                             }
                         }
                     }

search/views/topic.py

@@ -211,7 +211,7 @@ def home_query(device_id="",user_id=-1,query="",offset=0,size=10):
 
 
 @bind("physical/search/topic_detail_page_recommend")
-def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,filter_topic_user_id=False,offset=0,size=10):
+def topic_detail_page_recommend(device_id="",user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,filter_topic_user_id=False,offset=0,size=10):
     """
     :remark:帖子详情页推荐策略，缺少第一个卡片策略
     :param user_id:
@@ -224,17 +224,28 @@ def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_u
         if not isinstance(user_id,int):
             user_id = -1
 
+        redis_key = "physical:topic_detail_page_recommend" + ":user_id:" + str(user_id) + ":device_id:" + str(device_id)
+        have_read_topic_redis_data = redis_client.get(redis_key)
+        have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
+
         #获取es链接对象
         es_cli_obj = ESPerform.get_cli()
 
         # 获取帖子标签列表
         topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id,es_cli_obj)
 
-        result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size,es_cli_obj)
+        result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,have_read_topic_list,offset,size,es_cli_obj)
         recommend_topic_ids_list = list()
         if len(result_list)>0:
             recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
 
+        have_read_topic_list.extend(recommend_topic_ids_list)
+
+        have_read_topic_len = len(have_read_topic_list)
+        if have_read_topic_len>5000:
+            have_read_topic_list = have_read_topic_list[(have_read_topic_len-5000):]
+
+        redis_client.set(json.dumps(have_read_topic_list))
         return {"recommend_topic_ids": recommend_topic_ids_list}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/management/commands/trans2es_data2es_parallel.py

@@ -116,10 +116,8 @@ class Command(BaseCommand):
             type_info = get_type_info_map()[type_name]
             query_set = type_info.queryset
 
-            logging.info("before TableSlicer")
             slicer = TableSlicer(queryset=query_set, chunk_size=type_info.bulk_insert_chunk_size)
             for chunk in slicer.chunks():
-                logging.info("in chunks....")
                 job = Job(
                     sub_index_name=type_name,
                     type_name=type_name,