Merge branch 'buc' into 'master'

Buc

See merge request !96

libs/es.py

@@ -198,7 +198,6 @@ class ESPerform(object):
             if not batch_search:
                 res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
                                     size=size)
-
                 if is_suggest_request:
                     return res
                 else:

search_tips/routers.py

@@ -22,3 +22,26 @@ class DorisRouter(object):
         if app_label == 'doris':
             return db == 'doris'
         return None
+
+class JerryRouter(object):
+
+    def db_for_read(self, model, **hints):
+        if model._meta.app_label == "jerry_tidb":
+            return 'jerry_tidb'
+        return None
+
+    def db_for_write(self, model, **hints):
+        if model._meta.app_label == "jerry_tidb":
+            return 'jerry_tidb'
+        return None
+
+    def allow_relation(self, obj1, obj2, **hints):
+        if obj1._meta.app_label == 'jerry_tidb' or \
+                obj2._meta.app_label == 'jerry_tidb':
+            return True
+        return None
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        if app_label == 'jerry_tidb':
+            return db == 'jerry_tidb'
+        return None
\ No newline at end of file

trans2es/commons/words_utils.py

@@ -205,7 +205,6 @@ class QueryWordAttr(object):
             redis_data = redis_client.hget(cls.tips_num_redis_key_prefix, name)
             redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {}
             results_num = redis_val_dict['t'] if 't' in redis_val_dict else 0
-
             return results_num
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
@@ -217,7 +216,6 @@ class QueryWordAttr(object):
             total_num = 0
             data = StrategyHistoryQueryWords.objects.filter(keyword=name).values("answer_num", "tractate_num",
                                                                                  "diary_num").first()
-
             if data:
                 for value in data.values():
                     total_num += value

trans2es/models/strategy_search_word.py

@@ -19,3 +19,16 @@ class StrategyHistoryQueryWords(models.Model):
     answer_num = models.IntegerField(verbose_name=u'回答搜索结果次数')
     tractate_num = models.IntegerField(verbose_name=u'帖子搜索结果次数')
     diary_num = models.IntegerField(verbose_name=u'日记搜索结果次数')
+
+
+class StrategyHistoryQueryUnique(models.Model):
+    """
+        历史搜索词
+    """
+    class Meta:
+        app_label = 'doris'
+        db_table = 'strategy_history_query_unique'
+
+    keyword = models.CharField(u'项目介绍', max_length=48)
+    is_delete = models.IntegerField(u'是否上线', default=True)
+    uv = models.IntegerField(verbose_name=u'搜索次数')

trans2es/type_info.py

@@ -377,6 +377,28 @@ def get_type_info_map():
             round_insert_chunk_size=5,
             round_insert_period=2
         ),
+        TypeInfo(
+            name='suggest',
+            type='search_unique',  # tag
+            model=strategy_search_word.StrategyHistoryQueryUnique,
+            query_deferred=lambda: strategy_search_word.StrategyHistoryQueryUnique.objects.using(
+                settings.DORIS_DB_NAME).all().query,
+            get_data_func=SearchWord.get_search_unique,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2
+        ),
+        TypeInfo(
+            name='suggest-v1',
+            type='search_unique-v1',  # tag
+            model=strategy_search_word.StrategyHistoryQueryUnique,
+            query_deferred=lambda: strategy_search_word.StrategyHistoryQueryUnique.objects.using(
+                settings.DORIS_DB_NAME).all().query,
+            get_data_func=SearchWord.get_search_unique,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2
+        ),
         TypeInfo(
             name='suggest-v1',
             type='search_query-v1',  # tag
@@ -581,6 +603,18 @@ def tips_attr_sync_to_redis_type_info_map():
             bulk_insert_chunk_size=100,
             round_insert_chunk_size=5,
             round_insert_period=2,
+        ),
+        TypeInfo(
+            name='suggest',
+            type='query_unique_results_num',
+            model=strategy_search_word.StrategyHistoryQueryUnique,
+            query_deferred=lambda: strategy_search_word.StrategyHistoryQueryUnique.objects.using(
+                settings.DORIS_DB_NAME).filter(
+                is_delete=False).query,
+            get_data_func=SearchWord.set_data_to_redis,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
         )
     ]
 

trans2es/utils/itemwiki_transfer.py

@@ -20,7 +20,6 @@ class ItemWikiTransfer(object):
     def get_wiki_data_name_mapping_results_to_redis(cls, instance):
         try:
             wiki_name = instance.name.strip()
-            QueryWordAttr.save_query_to_redis(wiki_name, from_type="wiki")
             if wiki_name:
                 multi_fields = {
                     'name': 1,
@@ -60,7 +59,6 @@ class ItemWikiTransfer(object):
                 result_dict = ESPerform.get_search_results(ESPerform.get_cli(settings.GM_ORI_ES_INFO_LIST),
                                                            sub_index_name="newwiki", doc_type="newwiki", query_body=q,
                                                            offset=0, size=0)
-
                 doctor_results = result_dict["total_count"]
 
                 redis_key_prefix = "search_tips:tips_mapping_num"

trans2es/utils/search_query.py

@@ -6,6 +6,11 @@ from libs.tools import getMd5Digest
 from trans2es.commons.words_utils import QueryWordAttr, get_tips_word_type
 from trans2es.commons.commons import get_tips_suggest_list_v1
 from trans2es.models.strategy_search_word import StrategyHistoryQueryWords
+from trans2es.utils.itemwiki_transfer import ItemWikiTransfer
+from trans2es.utils.tag_transfer import TagTransfer
+from trans2es.utils.doctor_transfer import DoctorTransfer
+from trans2es.models import wordresemble
+from trans2es.utils.wordresemble import WordResemble
 
 
 class SearchWord(object):
@@ -24,13 +29,13 @@ class SearchWord(object):
                 # item_dict["id"] = getMd5Digest(keyword)
                 item_dict["id"] = "search_query_" + str(instance.id)
                 item_dict["ori_name"] = keyword
-                total_num = QueryWordAttr.get_search_query_results_num(keyword)
-                if total_num > 3:
-                    is_online = False if instance.is_delete else True
-                else:
-                    is_online = False
+                # total_num = QueryWordAttr.get_search_query_results_num(keyword)
+                # if total_num > 3:
+                #     is_online = False if instance.is_delete else True
+                # else:
+                #     is_online = False
 
-                item_dict["is_online"] = is_online
+                item_dict["is_online"] = False
                 item_dict["order_weight"] = QueryWordAttr.get_hot_search_query_word_weight(keyword)
                 item_dict["results_num"] = QueryWordAttr.get_search_query_results_num(keyword)
                 item_dict["type_flag"] = get_tips_word_type(keyword)
@@ -44,6 +49,32 @@ class SearchWord(object):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return ([], [])
 
+    @classmethod
+    def get_search_unique(self, instance):
+        try:
+            ret_list = list()
+            item_dict = dict()
+            suggest_list = list()
+            sub_query = re.sub('\W+', '', instance.keyword)
+            keyword = str(sub_query).lower()
+            ret_bol = QueryWordAttr.save_query_to_redis(query=sub_query)
+            if ret_bol:
+                item_dict["id"] = "search_unique_" + str(instance.id)
+                item_dict["ori_name"] = keyword
+                is_online = False if instance.is_delete else True
+                item_dict["is_online"] = is_online
+                item_dict["order_weight"] = int(0.01 * instance.uv)
+                item_dict["results_num"] = QueryWordAttr.get_query_results_num(keyword)
+                item_dict["type_flag"] = "unknown"
+                item_dict["offline_score"] = 0.0
+                item_dict["tips_name_type"] = 5
+                ret_list.append(item_dict)
+                suggest_list = get_tips_suggest_list_v1(keyword)
+            return (item_dict, suggest_list)
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return ([], [])
+
     @classmethod
     def get_search_query_v1(self, instance):
 
@@ -58,11 +89,11 @@ class SearchWord(object):
                 # item_dict["id"] = getMd5Digest(keyword)
                 item_dict["id"] = "search_query_" + str(instance.id)
                 item_dict["ori_name"] = keyword
-                if QueryWordAttr.get_search_query_results_num(keyword):
-                    is_online = False if instance.is_delete else True
-                else:
-                    is_online = False
-                item_dict["is_online"] = is_online
+                # if QueryWordAttr.get_search_query_results_num(keyword):
+                #     is_online = False if instance.is_delete else True
+                # else:
+                #     is_online = False
+                item_dict["is_online"] = False
                 item_dict["order_weight"] = QueryWordAttr.get_hot_search_query_word_weight(keyword)
                 item_dict["results_num"] = QueryWordAttr.get_search_query_results_num(keyword)
                 item_dict["type_flag"] = get_tips_word_type(keyword)
@@ -75,3 +106,23 @@ class SearchWord(object):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return ([], [])
+
+    @classmethod
+    def set_data_to_redis(cls, instance):
+        try:
+            total_count = 0
+            instance.name = instance.keyword
+            # 获取百科的
+            ItemWikiTransfer.get_wiki_data_name_mapping_results_to_redis(instance)
+
+            # 获取美购的
+            TagTransfer.get_tag_data_name_mapping_results_to_redis(instance)
+
+            # 获取医生医院的
+            DoctorTransfer.get_doctor_data_name_mapping_results_to_redis(instance)
+
+            # 获取日记的
+            WordResemble.get_diary_data_name_mapping_results_to_redis(instance)
+
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())