Commit 91471eaf authored by lixiaofang's avatar lixiaofang

es6升级后doris对应的新项目

parents
FROM ccr.ccs.tencentyun.com/gm-base/gm-alpine:v1.3
MAINTAINER wph [wangpenghong@igengmei.com]
COPY ./requirements.txt /tmp
RUN apk add --no-cache --virtual .build-deps \
bzip2-dev \
coreutils \
dpkg-dev dpkg \
expat-dev \
findutils \
gcc \
gdbm-dev \
libc-dev \
libffi-dev \
libnsl-dev \
libressl-dev \
libtirpc-dev \
linux-headers \
make \
ncurses-dev \
pax-utils \
readline-dev \
sqlite-dev \
tcl-dev \
tk \
tk-dev \
xz-dev \
zlib-dev \
# 业务相关依赖和安装工具
linux-headers \
python3-dev \
librdkafka-dev \
mariadb-client \
mariadb-dev \
git \
openssh \
\
# 取消ssh第一次链接的确认
&& echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config \
&& apk add --no-cache mariadb-connector-c-dev libxml2-dev libxslt-dev librdkafka-dev \
&& pip install --no-cache-dir -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \
&& mkdir -p /data/log/doris/app
ENV GM_RPCD_DEPLOY_CONF_PATH="/srv/apps/doris/doris/deploy_prod.xml" \
DJANGO_SETTINGS_MODULE=doris.prod
COPY . /srv/apps/doris/
WORKDIR /srv/apps/doris/
RUN cat requirements.txt | grep master > /tmp/gm-requirements.txt \
&& pip install --no-deps --upgrade -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/gm-requirements.txt \
&& apk del .build-deps
CMD gunicorn gm_rpcd.wsgi:application -w 5 -k gevent -b 0.0.0.0:8000
@Library('gm-pipeline-library') _
pipeline {
agent any
options {
// Console output add timestamps
timestamps()
// Disallow concurrent executions of the Pipeline
disableConcurrentBuilds()
// On failure, retry the entire Pipeline the specified number of times.
retry(1)
}
parameters {
choice(name: 'cache', choices: ['', '--no-cache'], description: 'docker build 是否使用cache,默认使用,不使用为--no-cache')
}
environment {
// Image Tag branch.time.hash
TAG = dockerTag()
// Image Full Tag
IMAGE = "${DOCKER_REGISTRY}/gm-backend/doris:$TAG"
}
stages {
stage("Begin") {
steps {
dingNotify "before"
}
}
stage('Build') {
steps {
sh "docker build . ${params.cache} -f ./Dockerfile -t $IMAGE"
sh "docker push $IMAGE"
}
}
}
post {
always {
dingNotify "after", "${currentBuild.currentResult}"
}
}
}
\ No newline at end of file
<?xml version="1.0" encoding="utf-8"?>
<gm_rpcd_config>
<info config_name="app" version="1.0"/>
<config name="application_name" value="doris"/>
<config name="service_list">
<element value="doris"/>
</config>
<config name="initializer_list">
<element value="doris.doris_injection"/>
<element value="rank.views.strategy"/>
<element value="rank.views.interpose"/>
<element value="rank.views.stat"/>
<element value="search.views.service"/>
<element value="search.views.diary"/>
<element value="search.views.doctor"/>
<element value="search.views.question"/>
<element value="search.views.sku"/>
<element value="search.views.spu"/>
<element value="search.views.board"/>
<element value="search.views.article"/>
<element value="search.views.answer"/>
<element value="search.views.mind"/>
<element value="search.views.home_feed"/>
<element value="recommend.views.service"/>
<element value="recommend.views.feed"/>
<element value="recommend.views.content"/>
<element value="recommend.views.detail"/>
<element value="hera.queries"/>
<element value="hera.views"/>
<element value="search.views.wiki"/>
<element value="search.views.wikitab"/>
<element value="search.views.wiki_keyword"/>
<element value="search.views.tractate"/>
<element value="search.views.principal_page"/>
<element value="search.views.subscript_article"/>
<element value="search.views.search_comprehensive"/>
<element value="search.views.content_aggre"/>
<element value="search.views.relation_search"/>
</config>
<config name="statuses" value="doris.system:statuses"/>
</gm_rpcd_config>
File added
import pymysql
from .celery import app as celery_app
pymysql.install_as_MySQLdb()
__all__ = ['celery_app']
# coding=utf-8
"""
Django settings for doris project.
Generated by 'django-admin startproject' using Django 1.10.
For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""
import os
from .log_settings import *
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'w%te9@t+_hgon4dq^7udy(&l=rpzkgga1ts2(fnym8cglx)v2@'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'dal',
'dal_select2',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'raven.contrib.django.raven_compat',
'json_editor',
'word',
'rank',
'search',
'recommend',
'hera',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'doris.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [
os.path.join(BASE_DIR, 'templates')
],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'doris.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/
from django.contrib import admin
STATIC_URL = '/static/'
admin.site.site_header = '更美策略平台'
STATIC_ROOT = 'static'
WORD_FILE = '/srv/apps/doris/static/word/word.txt'
STOP_WORD_FILE = '/srv/apps/doris/static/word/stopword.txt'
ES_CONSTS = {
'TAG_ID_DIRECTOR_RECOMMEND': 1142, # "所长精选" tag id
'TAG_ID_POPULAR': 1202, # "热门讨论" tag id
'TAG_ID_POPULAR_RECOMMEND': 1203, # "热门推荐" tag id
'TAG_ID_DIARY_WITH_SERVICE': 1144, # "美购日记" tag id
}
ES_SEARCH_TIMEOUT = '10s'
COUNT_LIMIT = 200
ES_SCRIPT_LANG = 'groovy'
DIARY_SCORE_WEIGHT = {
'HEAT_SCORE_WEIGHT': 0.6, # 热度分权重
'AUDIT_SCORE_WEIGHT': 0.55, # 内容质量分权重
'CHOUCHENG_SCORE_WEIGHT': 0.1, # 抽成分权重
}
PENALTY_FACTOR = 1.5
RANK_EMAIL = ['doctor@gmei.com']
MAX_TAG = 3
VIDEO_TAG_ID = 4706 # 如果有小视频 就给日记本添加视频日记的tag
FEED_DIARY_NAME = "feed:{}:diaries"
FEED_QUESTION_NAME = "feed:{}:question"
FEED_ANSWER_NAME = "feed:{}:answer"
FEED_ARTICLE_NAME = "feed:{}:article"
DEVICE_PORTRAIT_NAME = "device:{}:category"
FEED_CACHE_VERSION = "v0"
DATABASE_ROUTERS = ['doris.routers.DatabaseRouter']
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql', # 设置为mysql数据库
'NAME': 'doris_test',
'USER': 'work',
'PASSWORD': 'Gengmei1',
'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
'PORT': '62120',
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4", # 为了支持emoji表情
},
},
'gaia': {
'ENGINE': 'django.db.backends.mysql', # 设置为mysql数据库
'NAME': 'zhengxing_test',
'USER': 'work',
'PASSWORD': 'Gengmei1',
'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
'PORT': '62120',
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4", # 为了支持emoji表情
},
},
'mimas': {
'ENGINE': 'django.db.backends.mysql', # 设置为mysql数据库
'NAME': 'mimas_test',
'USER': 'work',
'PASSWORD': 'Gengmei1',
'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
'PORT': '62120',
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4", # 为了支持emoji表情
},
},
"gold": {
'ENGINE': 'django.db.backends.mysql', # 设置为mysql数据库
'NAME': 'gold',
'USER': 'work',
'PASSWORD': 'Gengmei1',
'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
'PORT': '62120',
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4", # 为了支持emoji表情
},
}
}
# shanxi: 陕西 shannxi: 山西
NEARBY_REGION = {
"beijing": ("hebei", "tianjin"),
"tianjin": ("beijing", "hebei"),
"hebei": ("beijing", "tianjin", "shandong", "henan", "shannxi", "neimenggu"),
"shannxi": ("hebei", "shandong", "henan", "shanxi", "neimenggu"),
"neimenggu": ("hebei", "liaoning", "jilin", "heilongjiang", "shannxi", "shanxi", "ningxia", "gansu"),
"liaoning": ("hebei", "jilin", "neimenggu"),
"jilin": ("liaoning", "heilongjiang", "neimenggu"),
"heilongjiang": ("jilin", "neimenggu"),
"shanghai": ("jiangsu", "anhui", "zhejiang"),
"jiangsu": ("shandong", "anhui", "zhejiang", "shanghai"),
"zhejiang": ("shanghai", "jiangsu", "anhui", "fujian"),
"anhui": ("jiangsu", "zhejiang", "jiangxi", "hubei", "henan"),
"fujian": ("zhejiang", "jiangxi", "guangdong"),
"jiangxi": ("anhui", "zhejiang", "fujian", "guangdong", "hunan", "hubei"),
"shandong": ("hebei", "henan", "jiangsu"),
"henan": ("hebei", "shandong", "anhui", "hubei", "shanxi", "shannxi"),
"hubei": ("henan", "anhui", "jiangxi", "hunan", "chongqing", "shanxi"),
"hunan": ("hubei", "jiangxi", "guangdong", "guangxi", "guizhou", "chongqing"),
"guangdong": ("guangxi", "hunan", "jiangxi", "fujian", "hainan"),
"guangxi": ("guangdong", "hunan", "guizhou", "yunnan"),
"hainan": ("guangdong",),
"chongqing": ("shanxi", "hubei", "hunan", "guizhou"),
"sichuan": ("gansu", "shanxi", "chongqing", "guizhou", "yunnan", "xizang", "qinghai"),
"guizhou": ("chongqing", "hunan", "guangxi", "yunnan", "sichuan"),
"yunnan": ("sichuan", "guizhou", "guangxi", "xizang"),
"xizang": ("qinghai", "sichuan", "yunnan", "xinjiang"),
"shanxi": ("neimenggu", "shannxi", "henan", "hubei", "chongqing", "sichuan", "gansu", "ningxia"),
"gansu": ("ningxia", "shanxi", "sichuan", "qinghai", "xinjiang", "neimenggu"),
"qinghai": ("gansu", "sichuan", "xizang", "xinjiang"),
"ningxia": ("neimenggu", "shanxi", "gansu"),
"xinjiang": ("gansu", "qinghai", "xizang"),
"taiwan": ("fujian",),
"xianggang": ("guangdong",),
"aomen": ("guangdong",)
}
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from django.conf import settings
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'doris.devel')
app = Celery('doris')
# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
app.conf.broker_url = settings.CELERY_BROKER_URL
app.conf.beat_schedule = settings.CELERYBEAT_SCHEDULE
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
<?xml version="1.0" encoding="utf-8"?>
<gm_rpcd_config>
<info config_name="deploy" version="1.0"/>
<config name="log_dir" value="/data/log/doris/app"/>
</gm_rpcd_config>
# coding=utf-8
from doris.base import *
from .log_settings import *
from celery import *
from celery.schedules import crontab
import pyssdb
CELERY_BROKER_URL = 'redis://:123456@127.0.0.1:6379/0'
REDIS_URL = 'redis://redis.paas-week.env:6379/2'
REDIS_URL2 = 'redis://redis.paas-week.env:6379/2'
ES_V2_HOSTS = [
{'host': '127.0.0.1', 'port': 9200}
]
ES_SERVICE_HOSTS = [
{'host': '127.0.0.1', 'port': 9200}
]
# ES_V2_HOSTS = [
# {'host': 'es.paas-test.env', 'port': 80}
# ]
# ES_SERVICE_HOSTS =[
# {'host': 'es.paas-test.env', 'port': 9200}
# ]
ES_INDEX_PREFIX = 'gm_test'
COUNT_LIMIT = 100
REDIS = {
'doctor': {'host': 'localhost', 'port': 6379, 'db': 1},
}
RAVEN_CONFIG = {
'dsn': 'http://29e77782db3c4429857cb1b6d69d9463:d55eafe28ae64f6e8c4a82fa3bc50fdd@sentry.gengmei.cc/80',
}
CELERYBEAT_SCHEDULE = {
'doris_ trans2es': {
'task': 'rank.tasks.doctor_statdata_2_cache',
# 'schedule': timedelta(seconds=1),
'schedule': crontab(hour=2, minute=0),
},
'doris_cal_device_interest': {
'task': 'recommend.tasks.cal_userinterest_rank',
# 'schedule': timedelta(seconds=1),
'schedule': crontab(hour=2, minute=0),
},
'doris_cal_servicehot_rank': {
'task': 'recommend.tasks.cal_servicehot_rank',
# 'schedule': timedelta(seconds=1),
'schedule': crontab(hour=2, minute=0),
}
}
GM_KV_HOSTS = [
{
"host": "redis.paas-week.env.168.15.11",
"port": 6379,
"db": 2,
"password": ""
},
{
"host": "redis.paas-week.env.168.15.12",
"port": 6379,
"db": 2,
"password": ""
}
]
# http://elasticsearch.paas-week.env/_plugin/head/
import os
import django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'doris.devel')
django.setup()
# coding=utf-8
"""
__author__ = 'xumingming'
"""
import os
import logging
LOG_DIR = '/data/log/doris/app/'
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(levelname)s %(module)s.%(funcName)s Line:%(lineno)d %(message)s',
filename=os.path.join(LOG_DIR, 'filelog.log'),
)
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_true': {
'()': 'django.utils.log.RequireDebugTrue',
},
},
'formatters': {
'verbose': {
'format': '%(asctime)s %(levelname)s %(module)s.%(funcName)s Line:%(lineno)d %(message)s'
},
'simple': {
'format': '%(levelname)s %(message)s'
},
'profile': {
'format': '%(asctime)s %(message)s'
},
'raw': {
'format': '%(message)s'
}
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
# 默认的服务器Log(保存到log/filelog.log中, 通过linux的logrotate来处理日志的分割
'default': {
'level': 'INFO',
'class': 'logging.handlers.RotatingFileHandler',
'filename': os.path.join(LOG_DIR, 'filelog.log'),
'formatter': 'verbose',
},
# 默认的服务器ERROR log
'default_err': {
'level': 'ERROR',
'class': 'logging.handlers.RotatingFileHandler',
'filename': os.path.join(LOG_DIR, 'error_logger.log'),
'formatter': 'verbose',
},
'exception_logger': {
'level': 'INFO',
'class': 'logging.handlers.RotatingFileHandler',
'filename': os.path.join(LOG_DIR, 'exception_logger.log'),
'formatter': 'verbose',
},
'tracer_handler': {
'level': 'INFO',
'class': 'logging.handlers.RotatingFileHandler',
'filename': os.path.join(LOG_DIR, 'tracer.log'),
'formatter': 'raw'
},
},
'loggers': {
'django': {
'handlers': ['default'],
'propagate': True,
'level': 'INFO',
},
'django.request': {
'handlers': ['default_err'],
'level': 'ERROR',
'propagate': False,
},
'exception_logger': {
'handlers': ['exception_logger'],
'level': 'INFO',
'propagate': False,
},
'gm_tracer.subscribe': {
'handlers': ['tracer_handler'],
'propagate': False,
'level': 'INFO'
},
},
}
class DatabaseRouter(object):
""" Mimas router. Change all question-answer related operation to mimas.
"""
PREFIX = 'extend.models.{}'
def checkObjOrigin(self, obj):
if obj.__module__.startswith(self.PREFIX.format("gaia")):
return "gaia"
elif obj.__module__.startswith(self.PREFIX.format("mimas")):
return "mimas"
elif obj._meta.app_label == 'gaia':
return 'gaia'
return None
def db_for_read(self, model, **hints):
return self.checkObjOrigin(model)
def db_for_write(self, model, **hints):
return self.checkObjOrigin(model)
def allow_relation(self, obj1, obj2, **hints):
return None
def allow_migrate(self, db, app_label, model_name=None, **hints):
return None
from django.http import HttpResponse
from libs.es import health
from rank.models import Strategy
from libs.cache import redis_client
from doris.celery import debug_task
import json, traceback
def _check_es():
error = {}
es_ok = True
try:
res = health()
if res['status'] == 'green':
es_ok = True
else:
error = {
"elasticsearch": res,
}
es_ok = False
except:
error = {
"elasticsearch": traceback.format_exc()
}
es_ok = False
return es_ok, error
def _check_mysql():
error = {}
mysql_ok = True
try:
item = Strategy.objects.last()
except:
mysql_ok = False
error = {
"mysql": traceback.format_exc()
}
return mysql_ok, error
def _check_redis():
error = {}
redis_ok = True
try:
redis_client.set("test", "test")
except:
redis_ok = False
error = {
"redis": traceback.format_exc()
}
return redis_ok, error
def _check_celery():
error = {}
celery_ok = True
try:
debug_task.delay()
except:
celery_ok = False
error = {
"celery": traceback.format_exc()
}
return celery_ok, error
def statuses(request):
ok = True
errors = {}
resp = {
'ok': ok,
'errors': errors
}
# check es
es_ok, error = _check_es()
if not es_ok:
errors.update(error)
# check redis
mysql_ok, error = _check_mysql()
if not mysql_ok:
errors.update(error)
redis_ok, error = _check_redis()
if not redis_ok:
errors.update(error)
celery_ok, error = _check_celery()
if not celery_ok:
errors.update(error)
resp['ok'] = es_ok and mysql_ok and redis_ok and celery_ok
return resp
\ No newline at end of file
"""doris URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.conf.urls import url, include
2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls'))
"""
import json
from django.conf.urls import url, include
from django.contrib import admin
from libs.http import render_json
from doris.system import statuses
http_statuses = render_json(statuses)
urlpatterns = [
url(r'^admin/', admin.site.urls),
url(r'^rank/', include('rank.urls')),
url(r'^system/statuses', http_statuses)
]
"""
WSGI config for doris project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "doris.devel")
application = get_wsgi_application()
from django.contrib import admin
# Register your models here.
from django.apps import AppConfig
class ExtendConfig(AppConfig):
name = 'extend'
from django.db import models
class Province(models.Model):
class Meta:
verbose_name = u'省份'
verbose_name_plural = u'省份'
db_table = 'api_province'
app_label = 'api'
id = models.CharField(max_length=40, primary_key=True, verbose_name=u'省份拼音名')
name = models.CharField(max_length=20, null=False, verbose_name=u'中文名')
display_in_filter = models.BooleanField(default=True, verbose_name=u'显示在筛选中')
is_municipality = models.BooleanField(default=False,
verbose_name=u"是否为直辖市(True:是,False:否)")
class City(models.Model):
class Meta:
verbose_name = u'62. 城市'
verbose_name_plural = u'62. 城市'
db_table = 'api_city'
app_label = 'api'
id = models.CharField(max_length=40, primary_key=True, verbose_name=u'拼音名')
name = models.CharField(max_length=20, null=False, verbose_name=u'中文名')
display_in_filter = models.BooleanField(default=True, verbose_name=u'显示在筛选中')
is_online = models.BooleanField(default=True, verbose_name=u'是否在线')
province = models.ForeignKey(Province, null=True, related_name='cities', verbose_name=u'省份')
tag_id = models.IntegerField(default=None, null=True)
level = models.PositiveIntegerField('城市级别')
class UserExtra(models.Model):
"""
以下字段定义已复制到passport项目, 如需修改请联系passport负责人:
user
portrait
"""
class Meta:
verbose_name = '用户附加信息'
verbose_name_plural = '用户附加信息'
db_table = 'api_userextra'
app_label = 'api'
city = models.ForeignKey(City, verbose_name='城市', null=True)
class DiarySpreadFit(models.Model):
class Meta:
verbose_name = u'首页日记本按地域设置分布'
app_label = 'api'
city = models.ForeignKey(City, verbose_name=u'城市')
commoncity = models.IntegerField(u'同城市日记本个数', default=0)
bescity = models.IntegerField(u'临近地域日记本个数', default=0)
is_online = models.BooleanField(u"是否下线", default=True)
create_time = models.DateTimeField(auto_now_add=True)
class CityScale(models.Model):
class Meta:
verbose_name = u'城市“桶”内容按比混合'
db_table = 'api_cityscale'
app_label = 'api'
city_id = models.CharField(null=False)
native = models.PositiveIntegerField(u'本地', default=0)
nearby = models.PositiveIntegerField(u'临近城市', default=0)
nation = models.PositiveIntegerField(u'全国', default=0)
megacity = models.PositiveIntegerField(u'特大城市', default=0)
class WordRel(models.Model):
class Meta:
verbose_name = u"词根"
db_table = "api_wordrel"
app_label = 'api'
id = models.IntegerField(u"词根主键id", primary_key=True)
keyword = models.CharField(max_length=50, null=False, verbose_name=u'中文名')
class WordRelSynonym(models.Model):
class Meta:
app_label = 'api'
db_table = 'api_wordrelsynonym'
wordrel = models.ForeignKey(WordRel, related_name='all_synonyms')
word = models.CharField(u'同义词', max_length=50, db_index=True)
class WordRelResemble(models.Model):
"""近义词数据表
"""
class Meta:
app_label = 'api'
db_table = 'api_wordrelresemble'
wordrel = models.ForeignKey(WordRel, related_name='all_resembles')
word = models.CharField(u'近义词', max_length=50, db_index=True)
from django.db import models
from gm_types.gaia import DIARY_CONTENT_LEVEL
class DiaryQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'日记本城市队列'
app_label = 'gold'
db_table = 'diary_queue'
city_id = models.CharField(max_length=64)
native_queue = models.TextField()
nearby_queue = models.TextField()
nation_queue = models.TextField()
megacity_queue = models.TextField()
class WikiQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'日记本城市队列'
app_label = 'gold'
db_table = 'wiki_queue'
queue = models.TextField()
class IconQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'图标队列'
app_label = 'gold'
db_table = 'icon_queue'
queue = models.TextField()
class QAQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'问答队列'
app_label = 'gold'
db_table = 'qa_queue'
queue = models.TextField()
class AnswerQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'回答队列'
app_label = 'gold'
db_table = 'answer_queue'
type = models.CharField(max_length=64)
queue = models.TextField()
class UserTopicQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'帖子队列'
app_label = 'gold'
db_table = 'user_topic_queue'
queue = models.TextField()
class DoctorTopicQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'帖子队列'
app_label = 'gold'
db_table = 'doctor_topic_queue'
queue = models.TextField()
class ArticleQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'专栏队列'
app_label = 'gold'
db_table = 'article_queue'
type = models.CharField(max_length=64)
queue = models.TextField()
class QuestionQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'问题队列'
app_label = 'gold'
db_table = 'question_queue'
type = models.CharField(max_length=64)
queue = models.TextField()
class DeviceIconQueue(models.Model):
class Meta:
verbose_name = u'用户队列'
app_label = 'gold'
db_table = 'device_icon_queue'
device_id = models.CharField(max_length=64)
queue = models.TextField()
class DeviceQAQueue(models.Model):
class Meta:
verbose_name = u'问答队列'
app_label = 'gold'
db_table = 'device_qa_queue'
device_id = models.CharField(max_length=64)
queue = models.TextField()
class DeviceAnswerQueue(models.Model):
class Meta:
verbose_name = u'用户队列'
app_label = 'gold'
db_table = 'device_answer_queue'
device_id = models.CharField(max_length=64)
queue = models.TextField()
class DeviceUserTopicQueue(models.Model):
class Meta:
verbose_name = u'用户队列'
app_label = 'gold'
db_table = 'device_user_topic_queue'
device_id = models.CharField(max_length=64)
queue = models.TextField()
class DeviceDoctorTopicQueue(models.Model):
class Meta:
verbose_name = u'用户队列'
app_label = 'gold'
db_table = 'device_doctor_topic_queue'
device_id = models.CharField(max_length=64)
queue = models.TextField()
class DeviceArticleQueue(models.Model):
class Meta:
verbose_name = u'用户队列'
app_label = 'gold'
db_table = 'device_article_queue'
device_id = models.CharField(max_length=64)
queue = models.TextField()
class DeviceDiaryQueue(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'用户队列'
app_label = 'gold'
db_table = 'device_diary_queue'
device_id = models.CharField(max_length=64)
city_id = models.CharField(max_length=64)
native_queue = models.TextField()
nearby_queue = models.TextField()
nation_queue = models.TextField()
megacity_queue = models.TextField()
class DeviceQuestionQueue(models.Model):
class Meta:
verbose_name = u'用户队列'
app_label = 'gold'
db_table = 'device_question_queue'
device_id = models.CharField(max_length=64)
queue = models.TextField()
\ No newline at end of file
from django.db import models
from gm_types.gaia import DIARY_CONTENT_LEVEL
class Diary(models.Model):
"""
日记本
"""
class Meta:
verbose_name = u'日记本'
verbose_name_plural = u'日记本'
app_label = 'talos'
db_table = 'api_diary'
user_id = models.IntegerField()
content_level = models.CharField(verbose_name=u'内容等级', max_length=1,
choices=DIARY_CONTENT_LEVEL,
default=DIARY_CONTENT_LEVEL.UNAUDITED)
from django.test import TestCase
# Create your tests here.
from django.shortcuts import render
# Create your views here.
from .city_rank import *
\ No newline at end of file
# coding=utf-8
from django.db import models
class CityWhiteList(models.Model):
class Meta:
verbose_name = u'排序白名单城市'
app_label = 'hera'
city_id = models.CharField(u'城市ID', unique=True, max_length=40)
from .city_rank import *
\ No newline at end of file
# coding=utf-8
from django.db.models import Q
from gm_dataquery.dataquery import DataBuilder, DataSQLQuery
from gm_dataquery.db import DB
from gm_dataquery.dict_mixin import to_dict
from gm_dataquery.client import Model
from gm_rpcd.all import RPCDFaultException
from gm_types.error import ERROR
from ..models import CityWhiteList
class CityWhiteListDB(DataBuilder):
def getval_name(self, obj):
data_model = Model('gaia', 'city', ['id', 'name'])
name = data_model.get(id=obj.city_id)['name']
return name
@DB
class CityWhiteListDQ(DataSQLQuery):
model = CityWhiteList
data_model = CityWhiteListDB
@classmethod
def create(self, **kwargs):
try:
obj = self.model.objects.create(**kwargs)
except:
raise RPCDFaultException(ERROR.UNIVERSAL, '该城市已在白名单中')
return to_dict(obj)
@classmethod
def update(self, updates, **kwargs):
obj = self.model.objects.get(id=kwargs['id']).delete()
return kwargs
# -*- coding: UTF-8 -*-
from hera.models import CityWhiteList
from libs.cache import redis_client
def check_city_in_whitelist(city_id):
cache_key = 'check_city_whitelist_{city_id}'.format(city_id=city_id)
try:
ret = redis_client.get(cache_key)
if ret is not None:
ret = bool(ret)
except:
ret = None
if ret is not None:
return ret
city_in_whitelist = CityWhiteList.objects.filter(city_id=city_id).exists()
redis_client.set(cache_key, city_in_whitelist, 300) # 5分钟缓存
return city_in_whitelist
from gm_rpcd.all import bind
from libs.rpc import get_rpc_invoker
from hera.models.city_rank import CityWhiteList
@bind('doris/hera/in_city_whitelist')
def city_is_in_whitelist(city_ids):
rs = []
if not city_ids:
return rs
qs = CityWhiteList.objects.filter(city_id__in=city_ids).values_list('city_id', flat=True)
return list(qs)
def in_city_white_list(city_tag_id):
# rpc_invoker = get_rpc_invoker()
# city = rpc_invoker['api/city/city_by_tag_id'](tag_id=city_tag_id).unwrap()
# if city.get('city_id'):
# return CityWhiteList.objects.filter(city_id=city.get('city_id')).exists()
return False
\ No newline at end of file
from collections import deque,OrderedDict
import unittest,random
import itertools
from hera.views import in_city_white_list
from .cache import redis_client
def drop_dup(items):
duplicates = set()
def check(item):
if item[1] in duplicates:
return False
else:
duplicates.add(item[1])
return True
return list(filter(check, items))
def drop_dup2(items):
duplicates = set()
def check(item):
if item["dup"] in duplicates:
return False
else:
duplicates.add(item["dup"])
return True
return list(filter(check, items))
def variousness_for_problem(items, drop_duplicated, variety_size):
import traceback
src = deque(items)
dst = []
while len(src) > 0:
pos, temp, recover, dup, diary_id_set = 0, [], [],[], set()
while pos < variety_size:
try:
item = src.popleft()
if item["type"] == 0:
if drop_duplicated and item.get("group") in dup:
recover.append(item)
else:
temp.append(item)
dup.append(item.get("group"))
pos = pos+1
else:
if item["diary_id"] in diary_id_set:
recover.append(item)
else:
temp.append(item)
dup.append(item["id"])
diary_id_set.add(item["diary_id"])
pos = pos + 1
except IndexError as e:
diff = variety_size - pos
diff, remain = recover[:diff], recover[diff:]
temp.extend(diff)
recover = remain
break
multi = OrderedDict()
for key in dup:
multi[key] = []
for item in temp:
if item["type"] == 0:
multi[item.get("group")].append(item)
else:
multi[item["id"]] = [item]
temp = filter(lambda a: a!=None,itertools.chain.from_iterable(itertools.zip_longest(*multi.values())))
dst.extend(temp)
src.extendleft(reversed(recover))
return dst
def variousness(items, variety_size):
src = deque(items)
dst = []
while len(src) > 0:
pos, temp, recover, dup = 0, [], [],[]
while pos < variety_size:
try:
item = src.popleft()
if item.get("group") in dup:
recover.append(item)
else:
temp.append(item)
dup.append(item.get("group"))
pos = pos+1
except IndexError:
diff = variety_size - pos
diff, remain = recover[:diff], recover[diff:]
temp.extend(diff)
recover = remain
break
multi = OrderedDict()
for key in dup:
multi[key] = []
for item in temp:
multi[item.get("group")].append(item)
temp = filter(lambda a: a!=None,itertools.chain.from_iterable(itertools.zip_longest(*multi.values())))
dst.extend(temp)
src.extendleft(reversed(recover))
return dst
def region_division(items, current_city):
city, nearby, other = [], [], []
for item in items:
if current_city == item.get("city"):
city.append(item)
elif current_city in item.get("nearby"):
nearby.append(item)
else:
other.append(item)
if in_city_white_list(current_city):
return [city, nearby, other]
return [city + nearby, other]
class VariousnessTestCase(unittest.TestCase):
def test_hello(self):
dup_collction = ["a","b","c"]
tests = []
result = []
for i in range(1000):
result.append(i)
tests.append(({"id":i,"group":random.choice(dup_collction)}))
print(tests)
dst = variousness(tests,10)
dst2 = variousness(tests,10)
print(dst)
assert dst == dst2
answer = []
for item in dst:
answer.append(item['id'])
answer.sort()
assert answer == result
if __name__ == "__main__":
unittest.main()
\ No newline at end of file
# coding=utf-8
from dal.autocomplete import ViewMixin
from dal_select2.widgets import WidgetMixin, Select2WidgetMixin
from django.views.generic.list import View
from django import forms
from django.http import JsonResponse
from django.utils.html import format_html
from django.utils.encoding import force_text
class BaseAutocompleteListView(ViewMixin, View):
def get_list_by_query(self, q):
""""Return the list of tuples(id, text) from which to autocomplete."""
raise NotImplementedError
def get(self, request, *args, **kwargs):
""""Return option list json response."""
results = self.get_list_by_query(self.q)
data = {
'results': [dict(id=k, text=v) for k, v in results]
}
return JsonResponse(data)
class LongListSelect(forms.Select):
def get_selected_options(self, selected_choices):
raise NotImplementedError
def render_options(self, selected_choices):
options = self.get_selected_options(selected_choices)
# Normalize to strings.
selected_choices = set(force_text(v) for v in selected_choices)
output = []
for option_value, option_label in options:
if isinstance(option_label, (list, tuple)):
output.append(format_html('<optgroup label="{}">', force_text(option_value)))
for option in option_label:
output.append(self.render_option(selected_choices, *option))
output.append('</optgroup>')
else:
output.append(self.render_option(selected_choices, option_value, option_label))
return '\n'.join(output)
class LongListSelectWidget(WidgetMixin, Select2WidgetMixin, LongListSelect):
pass
import redis
from django.conf import settings
redis_client = redis.StrictRedis.from_url(settings.REDIS_URL)
redis_client2 = redis.StrictRedis.from_url(settings.REDIS_URL2)
import time
def datetime_to_timestamp(obj):
timestamp = time.mktime(obj.timetuple())
return timestamp
def datetime_to_timestamp_millis(obj):
timestamp = datetime_to_timestamp(obj)
return timestamp*1000
\ No newline at end of file
from pygments import highlight, lexers, formatters
import json
def pretty_json(json_object):
formatted_json = json.dumps(json_object, sort_keys=True, indent=4)
colorful_json = highlight(formatted_json, lexers.JsonLexer(), formatters.TerminalFormatter())
print(colorful_json)
from gm_rpcd.all import RPCDFaultException
from gm_types.doris.error import ERROR
from raven.contrib.django.raven_compat.models import client as _sentry_client
def raise_error(error_code, message=None):
assert error_code != 0
if message is None:
message = ERROR.getDesc(error_code)
raise RPCDFaultException(code=error_code, message=message)
def logging_exception(send_to_sentry=True):
try:
# send exception info to sentry, fail silently
_sentry_client.captureException()
except:
pass
# coding=utf-8
import elasticsearch
from elasticsearch import Elasticsearch as Es
from django.conf import settings
from pytz import timezone
from libs.debug import pretty_json
import logging
def tzlc(dt, truncate_to_sec=True):
if dt is None:
return None
if truncate_to_sec:
dt = dt.replace(microsecond=0)
return timezone(settings.TIME_ZONE).localize(dt)
def es_index_adapt(index_prefix, doc_type, rw=None):
"""get the adapted index name
"""
assert rw in [None, 'read', 'write']
index = '-'.join((index_prefix, doc_type))
if rw:
index = '-'.join((index, rw))
return index
def get_es(es_hosts_config=None):
init_args = {
# 'hosts':settings.ES_HOSTS,
# no sniffing
'sniff_on_start': False,
'sniff_on_connection_fail': False,
}
new_hosts = settings.ES_V2_HOSTS if not es_hosts_config else es_hosts_config
new_es = Es(hosts=new_hosts, **init_args)
return new_es
HIGHLIGHT_TAG = 'ems'
def get_highlight(fields=[]):
field_highlight = {
'fields': {k: {} for k in fields},
'pre_tags': ['<%s>' % HIGHLIGHT_TAG],
'post_tags': ['</%s>' % HIGHLIGHT_TAG],
'fragment_size': 10000,
"require_field_match": False
}
return field_highlight
def get_highlight_query(fields=[], query=''):
field_highlight = {
'fields': {k: {"highlight_query": {
"bool": {
"must": {
"match": {
k: {
"query": query
}
}
},
"minimum_should_match": 0
}
}} for k in fields},
'fragment_size': 10000,
'pre_tags': ['<%s>' % HIGHLIGHT_TAG],
'post_tags': ['</%s>' % HIGHLIGHT_TAG],
"require_field_match": False,
}
return field_highlight
def get_highlight_query_analyzer(fields=[], query=''):
field_highlight = {
'fields': {k: {"highlight_query": {
"bool": {
"must": {
"match": {
k: {
"query": query
}
}
},
"minimum_should_match": 0
}
}} for k in fields},
'fragment_size': 10000,
'pre_tags': ['<%s>' % HIGHLIGHT_TAG],
'post_tags': ['</%s>' % HIGHLIGHT_TAG],
"require_field_match": False,
}
return field_highlight
def health():
res = {
'status': 'red',
'available_nodes': [],
}
es = get_es()
cluster_health = es.cluster.health()
res['status'] = cluster_health['status']
nodes_info = es.nodes.info()
available_nodes = nodes_info['nodes']
for key, node in available_nodes.items():
res['available_nodes'].append(node.get('ip'))
return res
def es_msearch(query_body, es_cli=None, es_hosts_config=None):
if es_cli is None:
if es_hosts_config:
es_cli = get_es(es_hosts_config)
else:
es_cli = get_es()
res = es_cli.msearch(query_body)
return res
def es_query(sub_index_name, body, offset, size, es=None, doc_type="_doc"):
if es is None:
es = get_es()
index = es_index_adapt(
index_prefix=settings.ES_INDEX_PREFIX,
doc_type=sub_index_name,
rw='read'
)
res = es.search(
index=index,
# doc_type=doc_type,
timeout=settings.ES_SEARCH_TIMEOUT,
body=body,
from_=offset,
size=size)
return res
def es_indices_analyze(doc_type, body, es):
if es is None:
es = get_es()
index = es_index_adapt(
index_prefix=settings.ES_INDEX_PREFIX,
doc_type=doc_type,
rw='read'
)
res = es.indices.analyze(index=index, body=body)
return res
def get_must_query(f):
q = {
"query": {
'bool': {
'must': f
}
}
}
return q
from extend.models.gaia import DiarySpreadFit
from libs.region import Graph
# from libs.ssdb import SsdbHelper
from gm_types.doris import CARD_TYPE
from gm_types.gaia import DIARY_CONTENT_LEVEL
g = Graph.buildGraph()
CANDIDATE_LEVEL = (DIARY_CONTENT_LEVEL.FINE, DIARY_CONTENT_LEVEL.EXCELLENT)
# class FeedBuilder(object):
# def __init__(self, ssdb_client, device_id, feed_type="diary"):
# """
#
# :param ssdb_client:
# :param device_id:
# :param feed_type:
# """
# self.ssdb_client = ssdb_client
# self.device_id = device_id
# self.type = feed_type
#
# def get(self, count, **kwargs):
# """
# get feed.
# :param count: feed size
# :param kwargs level 评级, city_id 当前城市
# :return:
# """
# level = kwargs.get("content_level")
# cityid = kwargs.get("city_id")
#
# local, nearby = kwargs.get("local", 0), kwargs.get("nearby", 0)
#
# if local == nearby == 0:
# try:
# spread = DiarySpreadFit.objects.get(city_id=cityid, is_online=True)
# local = spread.commoncity
# nearby = spread.bescity - local
# except DiarySpreadFit.DoesNotExist:
# local = nearby = count / 2
#
# unread_key = SsdbHelper.getQueueKey(self.type, self.device_id)
# readed_key = SsdbHelper.getQueueKey(self.type, self.device_id, True)
# unread = self.ssdb_client.zrange(unread_key, 0, -1, True)
# readed = self.ssdb_client.zrange(readed_key, 0, -1, True)
#
# unread_zset = ZsetBuilder(unread)
# readed_zset = ZsetBuilder(readed)
#
# if self.type == CARD_TYPE.DIARY:
# data = unread_zset.popLocal(local, city_id=cityid, content_level=level)
# data.extend(unread_zset.popNearby(nearby, city_id=cityid))
# else:
# data = unread_zset.popAny(count)
#
# readed_zset.items.extend(data)
#
# self.ssdb_client.zclear(unread_key)
# self.ssdb_client.zclear(readed_key)
#
# if len(data) < count or len(unread_zset.items) < count:
# """
# 循环队列
# """
# unread_zset.items.extend(readed_zset.items)
# if len(unread_zset.rebuild()):
# self.ssdb_client.multi_zset(unread_key, *unread_zset.rebuild())
# else:
# """
# """
# if len(unread_zset.rebuild()):
# self.ssdb_client.multi_zset(unread_key, *unread_zset.rebuild())
# if len(readed_zset.rebuild()):
# self.ssdb_client.multi_zset(readed_key, *readed_zset.rebuild())
#
# return [obj['pk'] for obj in data]
# class ZsetBuilder(object):
# def __init__(self, raw, readed=False):
# """
#
# :param raw:
# :param readed:
# """
# self.raw = raw
# self.readed = readed
# self.items = []
# self.initialize()
#
# def initialize(self):
# for i in range(0, len(self.raw), 2):
# key = self.raw[i].decode()
# pk, author_id, content_level, city_id = key.split(":")
# self.items.append({
# "pk": pk,
# "author_id": author_id,
# "content_level": content_level,
# "city_id": city_id,
# "score": self.raw[i + 1].decode(),
# })
#
# def remove(self, item):
# self.items.remove(item)
#
# def append(self, item):
# self.items.append(item)
#
# def rebuild(self):
# data = []
# for obj in self.items:
# data.append("{}:{}:{}:{}".format(obj['pk'], obj['author_id'], obj['content_level'], obj['city_id']))
# data.append(int(obj['score']))
# return data
#
# def popLocal(self, num, city_id, content_level=None):
# """
#
# :param num:
# :param city_id:
# :param content_level:
# :return:
# """
# data = []
# author_ids = []
#
# # 优先取同城,指定content—level的日记
# for idx, obj in enumerate(self.items):
# if len(data) >= num:
# break
#
# if not obj or obj['author_id'] in author_ids:
# continue
#
# if city_id == obj['city_id']:
# if content_level and content_level != obj['content_level']:
# continue
# author_ids.append(obj['author_id'])
# data.append(obj)
# self.items[idx] = None
#
# # 取临近地域的优秀日记本补足
# if len(data) < num:
# for idx, obj in enumerate(self.items):
# if len(data) >= num:
# break
#
# if not obj or obj['author_id'] in author_ids:
# continue
#
# if g.isNearBy(city_id, obj['city_id']) and obj['content_level'] in CANDIDATE_LEVEL:
# author_ids.append(obj['author_id'])
# data.append(obj)
# self.items[idx] = None
#
# # 取无地域限制的优秀日记本补足
# if len(data) < num:
# for idx, obj in enumerate(self.items):
# if len(data) >= num:
# break
#
# if not obj or obj['author_id'] in author_ids:
# continue
#
# if obj['content_level'] in CANDIDATE_LEVEL:
# author_ids.append(obj['author_id'])
# data.append(obj)
# self.items[idx] = None
#
# self.items = list(filter(lambda x: x is not None, self.items))
#
# return data
#
# def popNearby(self, num, city_id):
# """
#
# :param num:
# :param city_id:
# :return:
# """
# data = []
# author_ids = []
#
# for idx, obj in enumerate(self.items):
#
# if len(data) >= num:
# break
#
# if not obj or obj['author_id'] in author_ids:
# continue
#
# if g.isNearBy(city_id, obj['city_id']):
# author_ids.append(obj['author_id'])
# data.append(obj)
# self.items[idx] = None
#
# if len(data) < num:
# for idx, obj in enumerate(self.items):
# if len(data) >= num:
# break
#
# if not obj or obj['author_id'] in author_ids:
# continue
# if obj['content_level'] in CANDIDATE_LEVEL:
# author_ids.append(obj['author_id'])
# data.append(obj)
# self.items[idx] = None
#
# self.items = list(filter(lambda x: x is not None, self.items))
# return data
#
# def popAny(self, num):
# """
#
# :param num:
# :return:
# """
# data = []
# author_ids = []
# for idx, obj in enumerate(self.items):
# if len(data) >= num:
# break
#
# if not obj or obj['author_id'] in author_ids:
# continue
#
# author_ids.append(obj['author_id'])
# data.append(obj)
# self.items[idx] = None
#
# self.items = list(filter(lambda x: x is not None, self.items))
# return data
# coding=utf-8
from __future__ import unicode_literals, print_function, absolute_import
from libs.date import datetime_to_timestamp_millis
from libs.es import es_query, get_must_query
from django.conf import settings
from datetime import datetime
import functools
limited_size = functools.partial(min, settings.COUNT_LIMIT)
"""
def limited_size(size):
return min(settings.COUNT_LIMIT, size)
"""
def area_tag_id_filter(prefix_list, value):
return {
'bool': {
'should': [
{'term': {prefix + field: value}}
for prefix in prefix_list
for field in [
'city_tag_id',
'city_province_tag_id',
'city_province_country_tag_id',
]
]
}
}
def query_by_term(es, term, offset, size):
now = datetime.now()
f = [
{"term": {"is_online": True}},
{"term": {"is_can_be_sold": True}},
{"range": {"can_sold_time_range.start_time_long": {"lte": now}}},
{"range": {"can_sold_time_range.end_time_long": {"gt": now}}}
]
f.append(term)
sorting = {'smart_rank': {
'order': 'desc'
}}
q = get_must_query(f)
q['sort'] = sorting
res = es_query('service', q, offset, size, es)
hits = res['hits']['hits']
hit_ids = [hit['_source']['id'] for hit in hits]
return hit_ids
def service_recall(filters={}, sors=[], offset=0, size=10):
now = datetime.now()
f = [
{"term": {"is_online": True}},
{"range": {"can_sold_time_range.start_time_long": {"lte": now}}},
{"range": {"can_sold_time_range.end_time_long": {"gt": now}}}
]
import json
from django.http import HttpResponse
def render_json(wrapper):
def wrappered(*args, **argv):
json_data = wrapper(*args, **argv)
data = HttpResponse(json.dumps(json_data))
return data
return wrappered
# coding=utf-8
import elasticsearch
from elasticsearch import Elasticsearch as Es
from django.conf import settings
from pytz import timezone
from libs.debug import pretty_json
import logging
import traceback
import py2neo
from py2neo import Graph
import json
from libs.error import logging_exception
class Neo4jManager(object):
graph_obj = None
@classmethod
def neo4j_get_graph_obj(cls):
try:
if not cls.graph_obj:
cls.graph_obj = Graph(host=settings.NEO4J_HOST, auth=(settings.NEO4J_USER, settings.NEO4J_PWD))
return cls.graph_obj
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
return None
@classmethod
def neo4j_get_related_word_by_graph(cls, graph_obj=None, query_word="", graph_related_word_set=set(),
have_read_name_set=set(), need_size=6, redis_client=None,need_many_relation=False):
try:
assert (query_word and len(query_word) > 0)
assert (graph_obj is not None)
assert (redis_client is not None)
query_forbidden_redis_key = "doris_search:neo4j_forbidden_word_list:word:" + str(query_word)
query_forbidden_redis_data = redis_client.get(query_forbidden_redis_key)
query_forbidden_word_list = list()
if query_forbidden_redis_data:
query_forbidden_word_list = json.loads(str(query_forbidden_redis_data, encoding="utf-8"))
query_neo4j_redis_key = "doris_search:query_neo4j_word:" + str(query_word)
query_neo4j_redis_data = redis_client.get(query_neo4j_redis_key)
query_neo4j_word_set = set()
if query_neo4j_redis_data:
query_neo4j_word_set = set(json.loads(str(query_neo4j_redis_data,encoding="utf-8")))
else:
# if need_many_relation:
# graph_query = "match (na)-[re]->(nb) where na.name = \"" + query_word + "\" WITH na,re,nb match (nb)-[re2]->(nc) return na,re,nb,re2,nc"
# else:
graph_query = "match (na)-[re]->(nb) where na.name = \"" + query_word + "\" return na,re,nb"
graph_results = graph_obj.run(graph_query)
graph_total_num = 0
graph_valid_num = 0
for item in graph_results:
nb = item.get("nb").get("name")
graph_total_num += 1
if nb != query_word:
query_neo4j_word_set.add(nb)
graph_valid_num += 1
if need_many_relation:
nc = item.get("nc").get("name")
graph_total_num += 1
if nc != query_word:
query_neo4j_word_set.add(nc)
graph_valid_num += 1
redis_client.set(query_neo4j_redis_key,json.dumps(list(query_neo4j_word_set)))
redis_client.expire(query_neo4j_redis_key,2*24*60*60)
valid_query_neo4j_word_set = query_neo4j_word_set - have_read_name_set - set(query_forbidden_word_list) - graph_related_word_set
get_neo4j_word_num = 0
for word in valid_query_neo4j_word_set:
graph_related_word_set.add(word)
get_neo4j_word_num += 1
if get_neo4j_word_num >= need_size:
break
logging.info("query_word:%s graph related word num:%d" % (query_word,len(query_neo4j_word_set)))
return graph_related_word_set
except:
logging_exception()
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return graph_related_word_set
# coding=utf-8
from django.conf import settings
from extend.models.gaia import Province, City
class VertexCity(object):
def __init__(self, id, name):
"""
二级地域(市)
:param id:
:param name:
"""
self.id = id
self.name = name
class VertexProvince(object):
def __init__(self, id, name):
"""
一级地域(省,直辖市)
:param id:
:param name:
"""
self.id = id
self.name = name
self.connectedTo = {}
self.children = {}
def addNeighbor(self, nbr, weight=0):
self.connectedTo[nbr] = weight
def addChildren(self, child):
self.children[child.id] = child
def getConnections(self):
return self.connectedTo.keys()
def getId(self):
return self.id
def getWeight(self, nbr):
return self.connectedTo[nbr]
def __str__(self):
return str(self.id) + ' connectedTo: ' + str([x.id for x in self.connectedTo])
class Graph(object):
def __init__(self):
self.vertexList = {}
self.cityProvinceMapping = {}
def addVertex(self, v):
self.vertexList[v.id] = v
def getVertex(self, id):
return self.vertexList[id]
def addEdge(self, f, t, cost=0):
"""
临近地域有向图
:param f: vertex id
:param t: vertex id
:param cost:
:return:
"""
nbr = self.getVertex(t)
self.vertexList[f].addNeighbor(nbr, cost)
def getVertices(self):
"""
获取所有地域的id
:return:
"""
return self.vertexList.keys()
def isNearBy(self, c1, c2):
"""
是否是临近地域
:param c1: city 1
:param c2: city 2
:return:
"""
v1, v2 = self.cityProvinceMapping.get(c1), self.cityProvinceMapping.get(c2)
if v1 and v2:
return self.getVertex(v2) in self.getVertex(v1).getConnections()
return False
@classmethod
def buildGraph(cls):
"""
临近地域有向图
:return:
"""
g = Graph()
for p in Province.objects.all():
v = VertexProvince(p.id, p.name)
g.addVertex(v)
for c in City.objects.filter(province__id=p.id):
g.cityProvinceMapping[c.id] = p.id
for p1, pids in settings.NEARBY_REGION.items():
for p2 in pids:
g.addEdge(p1, p2)
return g
from django.conf import settings
from helios.rpc import create_default_invoker
_rpc_invoker = create_default_invoker(debug=settings.DEBUG)
def get_rpc_invoker():
return _rpc_invoker
from django.conf import settings
# class SsdbHelper(object):
#
# TYPE_DIARY = "diary"
# TYPE_ANSWER = "answer"
# TYPE_ARTICLE = "article"
#
# def __init__(self, ssdb_client):
# self.ssdb_client = ssdb_client
#
# @classmethod
# def getQueueKey(cls, type, device_id, read=False):
# """
#
# :param type:
# :param device_id:
# :param read:
# :return:
# """
# return "{}:{}:{}:{}".format(settings.FEED_CACHE_VERSION, type, device_id, int(read))
#
# @staticmethod
# def getZsetKey(element_id, content_level, city_id):
# """
#
# :param element_id:
# :param content_level:
# :param city_id:
# :return:
# """
# return "{}:{}:{}".format(element_id, content_level, city_id)
#
# def updateCache(self, key, kv_tuple):
# self.ssdb_client.zclear(key)
# v = []
# for i in kv_tuple:
# v.append(i[0])
# v.append(i[1])
# if len(v):
# print(key, v)
# self.ssdb_client.multi_zset(key, *v)
import unittest, os
from gm_rpcd.internals.protocol.request import make_request_from_v1
from gm_rpcd.internals.initializations import initialize
from gm_rpcd.internals.protocol.response import SuccessResponse
from django.test import TestCase
class DorisTestCase(TestCase):
def setUp(self):
os.environ["GM_RPCD_MODE"] = "develop"
def call(self, method, params = {}, session_key = None):
dispatcher = initialize().dispatcher
request = make_request_from_v1(
method=method,
params=params,
session_key=session_key,
environment=None,
)
response = dispatcher.process_single_request(request)
if type(response) == SuccessResponse:
response.code = 0
return response
from django.test.runner import DiscoverRunner
class NoDbTestRunner(DiscoverRunner):
""" A test runner to test without database creation """
def setup_databases(self, **kwargs):
""" Override the database creation defined in parent class """
pass
def teardown_databases(self, old_config, **kwargs):
""" Override the database teardown defined in parent class """
pass
import json
def ensure_str(obj):
if isinstance(obj,bytes):
return obj.decode('utf8')
else:
return obj
def ensure_dict(obj):
if isinstance(obj,str):
return json.loads(obj)
else:
return obj
\ No newline at end of file
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "doris.devel")
try:
from django.core.management import execute_from_command_line
except ImportError:
# The above import may fail for some other reason. Ensure that the
# issue is really that Django is missing to avoid masking other
# exceptions on Python 2.
try:
import django
except ImportError:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
)
raise
execute_from_command_line(sys.argv)
from django import forms
from django.conf import settings
from django.contrib import admin
from gm_types.doris import STRATEGY_TYPE
from json_editor import JSONEditorWidget
from rank.forms.interpose import InterposeForm, InterposeObjectForm, InterposeObjectFormSet
from rank.models import Strategy, Interpose, InterposeObject
from rank.tasks import export_email_task
class JsonForm(forms.ModelForm):
class Meta:
model = Strategy
fields = '__all__'
widgets = {
'json': JSONEditorWidget(schema={}),
}
def _check_strategy(self, cleaned_data):
if cleaned_data['name'] == STRATEGY_TYPE.DOCTOR_ORDER:
self._check_doctor(cleaned_data)
def _check_doctor(self, cleaned_data):
sum = 0
for v in cleaned_data['json'].values():
sum += v
if sum != 1:
raise forms.ValidationError(
"权重之和必须等于1"
)
def clean(self):
cleaned_data = super(JsonForm, self).clean()
self._check_strategy(cleaned_data)
class StrategyAdmin(admin.ModelAdmin):
form = JsonForm
actions = ['push_search_cache', 'export_email']
def export_email(self, request, queryset):
export_email_task(to_user_email=settings.RANK_EMAIL)
def push_search_cache(self, request, queryset):
print('run generate_word_action')
for item in queryset:
item.push_search_cache()
def has_delete_permission(self,request, obj=None):
return False
push_search_cache.short_description='推送策略'
class InterposeObjectInline(admin.TabularInline):
model = InterposeObject
form = InterposeObjectForm
formset = InterposeObjectFormSet
class InterposeAdmin(admin.ModelAdmin):
form = InterposeForm
inlines = [InterposeObjectInline]
def get_formsets_with_inlines(self, request, obj=None):
for inline in self.get_inline_instances(request, obj):
if isinstance(inline, InterposeObjectInline) and obj is None:
continue
yield inline.get_formset(request, obj), inline
def save_formset(self, request, form, formset, change):
super(InterposeAdmin, self).save_formset(request, form, formset, change)
if form.instance and form.instance.pk:
form.instance.update_obj_ids()
admin.site.register(Strategy, StrategyAdmin)
admin.site.register(Interpose, InterposeAdmin)
from django.apps import AppConfig
class RankConfig(AppConfig):
name = 'rank'
[
{
"model": "rank.strategy",
"pk": 1,
"fields": {
"name": "service_search",
"desc": "service search platform",
"json": "{\"a\":12,\"b\":3,\"c\":1,\"d\":5,\"e\":3}"
}
},
{
"model": "rank.interpose",
"pk": 1,
"fields": {
"interpose_type": "service_search",
"keyword": "\u859b\u7ea2\u5b87",
"city_tag_id": 592,
"tag_id": 15,
"sort_order": "0",
"last_modified": "2017-04-25T07:18:19.642Z",
"object_ids": "[321, 123]",
"is_online": true
}
}
]
This diff is collapsed.
# coding=utf-8
from __future__ import unicode_literals, absolute_import
# coding=utf-8
from __future__ import unicode_literals, absolute_import
# coding=utf-8
import random
from django.core.management.base import BaseCommand
from recommend.models import ServiceHot, DeviceInterest
import random
query_list = [
'双眼皮',
'埋线',
'瘦脸针',
'丰胸',
'玻尿酸',
]
service_list = [
4667613, 4627748, 2008, 4636677, 2094, 4667978,1665,4667840,4667968,4667904
]
tags = [
2,3,4,5,6,22,23,24,25,26,991,992,993,994,995
]
device_ids = [
"aaaa","aaab","aaac","aaad","aabb","aabc","aabd","abbb","abbc","abbd","abca","abcb","abcc","abcd"
]
class Command(BaseCommand):
def handle(self, *args, **options):
for service in service_list:
for tag in tags:
servicehot, created = ServiceHot.objects.get_or_create(service_id=service, tag_id=tag)
servicehot.rank =random.uniform(0, 10)
servicehot.save()
for device_id in device_ids:
for tag in tags:
device, created = DeviceInterest.objects.get_or_create(device_id=device_id, tag_id=tag)
device.rank = random.uniform(0, 10)
device.save()
# coding=utf-8
from __future__ import unicode_literals, absolute_import
from django.core.management.base import BaseCommand
from rank.tasks import doctor_statdata_2_cache
class Command(BaseCommand):
def handle(self, *args, **options):
doctor_statdata_2_cache()
# -*- coding: utf-8 -*-
# Generated by Django 1.10.6 on 2017-03-27 09:28
from __future__ import unicode_literals
from django.db import migrations, models
import jsonfield.fields
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Strategy',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128, unique=True)),
('desc', models.CharField(max_length=128, unique=True)),
('json', jsonfield.fields.JSONField()),
],
),
]
from .strategy import *
from .interpose import *
from .doctor import *
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment