add

_celery.py

-# !/usr/bin/env python
-# encoding=utf-8
-from __future__ import absolute_import
-
-import os
-
-# set the default Django settings module for the 'celery' program.
-# os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'search_tips.settings')
-
-import raven
-from raven.contrib.celery import register_signal, register_logger_signal
-from celery import Celery
-from django.conf import settings
-
-
-class Celery(Celery):
-    """wrap for celery.Celery."""
-
-    def on_configure(self):
-        # check if sentry settings provided
-        if not settings.SENTRY_CELERY_ENDPOINT:
-            return
-
-        client = raven.Client(settings.SENTRY_CELERY_ENDPOINT)
-
-        # register a custom filter to filter out duplicate logs
-        register_logger_signal(client)
-
-        # hook into the Celery error handler
-        register_signal(client)
-
-
-app = Celery('search_tips_tasks')
-
-# Using a string here means the worker will not have to
-# pickle the object when using Windows.
-app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

injection/data_sync/tasks.py

-# -*- coding: UTF-8 -*-
-import logging
-from celery import shared_task
-from trans2es.type_info import get_type_info_map
-import logging
-import traceback
-import base64
-from libs.cache import redis_client
-import json
-from trans2es.models import wordresemble
-
-
-@shared_task
-def set_data_to_redis():
-    try:
-        QUERY_KEY = "query:{}:search_tips"
-        tips_num_redis_key_prefix = "search_tips:tips_mapping_num"
-
-        result = wordresemble.WordRel.objects.all().values("keyword", "id")
-        for wordrm in result:
-            query_sql_item = wordresemble.WordRel.objects.filter(keyword=wordrm["keyword"])
-            for sql_obj in query_sql_item:
-                words = list(sql_obj.all_resembles.all().values_list('word', flat=True))
-                for items in words:
-                    # 先存储本体词
-                    logging.info("get wordrm:%s" % wordrm["keyword"])
-                    logging.info("get words:%s" % words)
-                    query_base64 = base64.b64encode(wordrm["keyword"].encode('utf8')).decode('utf8')
-                    key = QUERY_KEY.format(query_base64)
-                    value_name = base64.b64encode(items.encode('utf8')).decode('utf8')
-
-                    # 获取本体词的结果
-                    redis_data = redis_client.hget(tips_num_redis_key_prefix, items)
-                    redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {}
-                    total_count = 0
-                    if 't' in redis_val_dict:
-                        total_count = int(redis_val_dict['t'])
-                    # 存储到Redis
-                    value = value_name + str(":" + str(total_count))
-                    redis_client.sadd(key, value)
-                    logging.info("get value:%s" % value)
-
-                    # 存储近义词
-                    query_base64 = base64.b64encode(items.encode('utf8')).decode('utf8')
-                    key = QUERY_KEY.format(query_base64)
-                    value_name = base64.b64encode(wordrm["keyword"].encode('utf8')).decode('utf8')
-                    # 获取结果
-                    redis_data = redis_client.hget(tips_num_redis_key_prefix, wordrm["keyword"])
-                    redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {}
-                    total_count = 0
-                    if 't' in redis_val_dict:
-                        total_count = int(redis_val_dict['t'])
-                    # 存储结果
-                    value = value_name + str(":" + str(total_count))
-                    redis_client.sadd(key, value)
-                    logging.info("get value:%s" % value)
-                    if words:
-                        for w in words:
-                            logging.info("get word_pops:%s" % words)
-                            value_name = base64.b64encode(w.encode('utf8')).decode('utf8')
-                            redis_data = redis_client.hget(tips_num_redis_key_prefix, w)
-                            redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {}
-                            total_count = 0
-                            if 't' in redis_val_dict:
-                                total_count = int(redis_val_dict['t'])
-                            value = value_name + str(":" + str(total_count))
-                            redis_client.sadd(key, value)
-                            logging.info("get value:%s" % value)
-
-
-    except:
-        logging.error("catch exception,err_msg:%s" % traceback.format_exc())

search_tips/celery.py

-from __future__ import absolute_import, unicode_literals
-import os
-from celery import Celery
-from django.conf import settings
-
-# set the default Django settings module for the 'celery' program.
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'search_tips.settings')
-
-app = Celery('search_tips')
-
-# Using a string here means the worker doesn't have to serialize
-# the configuration object to child processes.
-# - namespace='CELERY' means all celery-related configuration keys
-#   should have a `CELERY_` prefix.
-app.config_from_object('django.conf:settings', namespace='CELERY')
-
-# Load task modules from all registered Django app configs.
-app.autodiscover_tasks()
-app.conf.broker_url = settings.CELERY_BROKER_URL
-
-
-@app.task(bind=True)
-def debug_task(self):
-    print('Request: {0!r}'.format(self.request))