Merge branch 'master' into 'test'

Master

See merge request !202

linucb/views/collect_data.py

@@ -8,19 +8,17 @@ from linucb.views.linucb import LinUCB
 import json
 from trans2es.models.tag import TopicTag
 import traceback
+from django.conf import settings
 
 class KafkaManager(object):
-    # kafka信息
-    kafka_broker_list = "192.168.13.114:9092,192.168.13.116:9092,192.168.13.115:9092"
-    topic_name = "alpha-maidian-data"
     consumser_obj = None
 
     @classmethod
     def get_kafka_consumer_ins(cls, topic_name=None):
 
         if not cls.consumser_obj:
-            topic_name = cls.topic_name if not topic_name else topic_name
-            cls.consumser_obj = KafkaConsumer(topic_name,bootstrap_servers=cls.kafka_broker_list)
+            topic_name = settings.KAFKA_TOPIC_NAME if not topic_name else topic_name
+            cls.consumser_obj = KafkaConsumer(topic_name,bootstrap_servers=settings.KAFKA_BROKER_LIST)
             # cls.consumser_obj.subscribe([topic_name])
 
         return cls.consumser_obj

linucb/views/linucb.py

@@ -11,7 +11,7 @@ import json
 import pickle
 
 class LinUCB:
-    d = 1
+    d = 2
     alpha = 0.25
     r1 = 1
     r0 = -0.5

physical/settings.py

@@ -14,189 +14,4 @@ import os
 from .log_settings import *
 from datetime import timedelta
 from celery.schedules import crontab
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
-
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = '^j3sg)sj8rc@du74%fb$c2926tv!!4g(kp-=rx1)c5!1&1(dq='
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
-
-ALLOWED_HOSTS = []
-
-# Application definition
-SENTRY_CELERY_ENDPOINT = "http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
-BROKER_URL = "redis://127.0.0.1:6379/8"
-
-# CELERY_SEND_EVENTS = True
-# CELERY_SEND_TASK_SENT_EVENT = True
-#
-# CELERY_DEFAULT_EXCHANGE = 'celery'
-# CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
-# CELERY_DEFAULT_ROUTING_KEY = 'celery'
-#
-# CELERY_QUEUES = {
-#     'celery': {
-#         'exchange': CELERY_DEFAULT_EXCHANGE,
-#         'routing_key': CELERY_DEFAULT_ROUTING_KEY,
-#     },
-#     'order': {
-#         'exchange': 'order',
-#         'routing_key': 'order',
-#     },
-# }
-
-CELERY_BROKER_URL = "redis://127.0.0.1:6379/8"
-CELERY_TIMEZONE = 'Asia/Shanghai'
-CELERY_ROUTES = ['physical.celery_task_router.CeleryTaskRouter']
-
-INSTALLED_APPS = (
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
-    'trans2es',
-    'search',
-    'injection.data_sync',
-)
-
-CELERYBEAT_SCHEDULE = {
-    'sync_face_similar_data_to_redis': {
-        'task': 'injection.data_sync.tasks.sync_face_similar_data_to_redis',
-        'schedule': timedelta(seconds=120),
-        'args': ()
-    },
-}
-"""
-MIDDLEWARE = [
-    'django.middleware.security.SecurityMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-]
-TEMPLATES = [
-    {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [],
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
-            ],
-        },
-    },
-]
-AUTH_PASSWORD_VALIDATORS = [
-    {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
-    },
-]
-"""
-
-ROOT_URLCONF = 'physical.urls'
-
-WSGI_APPLICATION = 'physical.wsgi.application'
-
-REDIS_URL = "redis://127.0.0.1:6379"
-# Database
-# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
-
-DATABASE_ROUTERS = ['physical.DBRouter.DBRouter']
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.mysql',
-        'NAME': 'alpha',
-        'USER': 'work',
-        'PASSWORD': 'Gengmei123',
-        # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
-        "HOST": 'rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com',
-        #'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
-        'PORT': '3306',
-        'OPTIONS': {
-            "init_command": "SET foreign_key_checks = 0;",
-            "charset": "utf8mb4",
-        },
-    },
-    'face': {
-        'ENGINE': 'django.db.backends.mysql',
-        'NAME': 'face',
-        'USER': 'work',
-        'PASSWORD': 'Gengmei123',
-        # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
-        "HOST": "rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com",
-        # 'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
-        'PORT': '3306',
-        # 'CONN_MAX_AGE': None,
-        'OPTIONS': {
-            "init_command": "SET foreign_key_checks = 0;",
-            "charset": "utf8mb4",
-        },
-    }
-
-}
-ES_INFO_LIST = [
-    {
-        "host": "10.29.130.141",
-        "port": 9200
-    }
-]
-
-ES_INDEX_PREFIX = "gm-dbmw"
-
-MIDDLEWARE_CLASSES = (
-    'gm_tracer.middleware.TracerMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'helios.DjangoL5dMiddleware',
-)
-# Password validation
-# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
-
-
-# Internationalization
-# https://docs.djangoproject.com/en/1.10/topics/i18n/
-
-# LANGUAGE_CODE = 'en-us'
-#
-TIME_ZONE = 'Asia/Shanghai'
-#
-USE_I18N = True
-
-USE_L10N = True
-#
-# USE_TZ = True
-
-
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/1.10/howto/static-files/
-
-STATIC_URL = '/static/'
-
+from .settings_local import *

search/utils/topic.py

@@ -295,7 +295,7 @@ class TopicUtils(object):
 
                 query_function_score["query"]["bool"]["should"] = [
                     {'multi_match': multi_match},
-                    {"term":{"tag_list":tag_id}}
+                    {"terms":{"tag_list":tag_id}}
                 ]
                 query_function_score["query"]["bool"]["minimum_should_match"] = 1
 
@@ -530,7 +530,7 @@ class TopicUtils(object):
                             #{"range": {"content_level": {"gte": 3, "lte": 5}}},
                             {"term": {"is_online": True}},
                             {"term": {"is_deleted": False}},
-                            {"term": {"tag_list":tag_id}}
+                            {"terms": {"tag_list":tag_id}}
                         ],
                         "must_not":[
                             {"terms": {"content_level": [1,2]}}
@@ -579,6 +579,7 @@ class TopicUtils(object):
         f = [
             {"term": {"is_deleted": False}},
         ]
+
         if not filters:
             return f
 
@@ -601,6 +602,21 @@ class TopicUtils(object):
                         }
                     }
                 })
+
+            elif k in ["id_gte", "id_lte"]:
+
+                if k == "id_gte":
+                    op = "gte"
+                elif k == "id_lte":
+                    op = "lte"
+
+                f.append({
+                    "range": {
+                        "id": {
+                            op: v,
+                        }
+                    }
+                })
             else:
                 if isinstance(v, list):
                     f.append({

trans2es/management/commands/trans2es_mapping2es.py

@@ -41,11 +41,11 @@ class Command(BaseCommand):
                         official_index_name = ESPerform.get_official_index_name(type_name)
                         index_exists = es_cli.indices.exists(official_index_name)
                         if not index_exists:
-                            logging.info("begin create [%s] index and mapping!" % type_name)
+                            logging.info("begin create [%s] index!" % type_name)
                             ESPerform.create_index(es_cli, type_name)
-                            ESPerform.put_index_mapping(es_cli, type_name)
-                        else:
-                            logging.warning("index:[%s] has already existing!" % type_name)
+
+                        logging.info("begin create [%s] mapping!" % type_name)
+                        ESPerform.put_index_mapping(es_cli, type_name, force_sync=True)
 
 
             if len(options["indices_template"]):

trans2es/mapping/all_index_template.json

 {
   "index_patterns": ["*"],
   "settings":{
-    "number_of_shards": 9,
-    "number_of_replicas": 2,
+    "number_of_shards": 5,
+    "number_of_replicas": 3,
     "index":{
       "analysis":{
         "filter":{

trans2es/models/topic.py

@@ -191,6 +191,8 @@ class Topic(models.Model):
             """
             if user_is_shadow:
                 offline_score = offline_score * 0.5
+
+            offline_score -= self.drop_score
             return offline_score
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/models/user.py

@@ -118,16 +118,22 @@ class User(models.Model):
         return pick_user_detail_list
 
     def get_same_group_user_id_list(self):
+
+        #todo 有的同组数据过大，导致celery cpu过高，暂时限制同组的数据大小，后续可能会去掉同组的数据
         same_group_user_id_list = list()
-        group_items_list = GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id)
-        for group_item in group_items_list:
-            group_id = group_item.group_id
-            user_items_list = GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(group_id=group_id)
-            for user_items_list in user_items_list:
-                same_group_user_id_list.append(user_items_list.user_id)
+        group_items_list = list(GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id).values_list("group_id",flat=True))
+        for group_id in group_items_list:
+            user_items_list = list(GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(group_id=group_id).values_list("user_id",flat=True))
+            for user_id in user_items_list:
+                same_group_user_id_list.append(user_id)
+                if len(same_group_user_id_list)>=100:
+                    break
+            if len(same_group_user_id_list)>=100:
+                break
+
 
         same_group_detail_list = list()
-        for i in range(0, len(same_group_user_id_list), 1000):
+        for i in range(0, len(same_group_user_id_list), 200):
             sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id__in=same_group_user_id_list[i:i + 1000])
             for detail_data in sql_data_list:
                 item = {

trans2es/utils/tag_transfer.py

@@ -46,9 +46,11 @@ class TagTransfer(object):
             if instance.is_online==True and instance.is_deleted==False:
                 topic_id_list = list()
 
-                sql_result_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(tag_id=instance.id,is_online=True).values_list("topic_id",flat=True)
-                for item_id in sql_result_list:
-                    topic_id_list.append(item_id)
+                sql_result_results = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(
+                    tag_id=instance.id).values_list("topic_id", "is_online"))
+                for topic_id,is_online in sql_result_results:
+                    if is_online:
+                        topic_id_list.append(topic_id)
 
                 time_base_val = datetime.datetime.strftime(datetime.datetime.now()+datetime.timedelta(-7), "%Y-%m-%d")