Skip to content

P
physical
Commit d3dde44a authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'test' into 'master' 

# Conflicts:
#   search/utils/topic.py
parents d56866c2 8675f182
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 636 additions and 51 deletions
.idea/codeStyles/Project.xml 0 → 100644
View file @ d3dde44a
<component name="ProjectCodeStyleConfiguration">
<code_scheme name="Project" version="173">
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</code_scheme>
</component>
\ No newline at end of file
.idea/dbnavigator.xml 0 → 100644
View file @ d3dde44a
This diff is collapsed.
.idea/dictionaries/lxrent.xml 0 → 100644
View file @ d3dde44a
<component name="ProjectDictionaryState">
<dictionary name="lxrent" />
</component>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="PROJECT" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
.idea/misc.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>
</project>
\ No newline at end of file
.idea/modules.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/physical.iml" filepath="$PROJECT_DIR$/.idea/physical.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/physical.iml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
.idea/workspace.xml 0 → 100644
View file @ d3dde44a
This diff is collapsed.
app_conf.xml
View file @ d3dde44a
......@@ -9,10 +9,12 @@
<config name="initializer_list">
<element value="physical.django_init"/>
<element value="search.views.topic"/>
<element value="search.views.business_topic"/>
<element value="search.views.pick"/>
<element value="search.views.group"/>
<element value="search.views.user"/>
<element value="search.views.tag"/>
<element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/>
</config>
......
libs/es.py
View file @ d3dde44a
......@@ -207,4 +207,4 @@ class ESPerform(object):
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0,"hits":[]}
\ No newline at end of file
return {"total_count":0,"hits":[]}
physical/DBRouter.py 0 → 100644
View file @ d3dde44a
# !/usr/bin/env python
# -*- coding: utf-8 -*-
class DBRouter:
"""
A router to control all database operations on models in the
user application.
"""
def db_for_read(self, model, **hints):
"""
Attempts to read user models go to users_db.
"""
if model._meta.app_label == 'face':
return 'face'
return None
def db_for_write(self, model, **hints):
"""
Attempts to write user models go to users_db.
"""
if model._meta.app_label == 'face':
return 'face'
return None
def allow_relation(self, obj1, obj2, **hints):
"""
Allow relations if a model in the user app is involved.
"""
if obj1._meta.app_label == 'face' or \
obj2._meta.app_label == 'face':
return True
return None
def allow_migrate(self, db, app_label, model_name=None, **hints):
"""
Make sure the auth app only appears in the 'users_db'
database.
"""
if app_label == 'face':
return db == 'face'
return None
physical/settings.py
View file @ d3dde44a
......@@ -14,4 +14,189 @@ import os
from .log_settings import *
from datetime import timedelta
from celery.schedules import crontab
from .settings_local import *
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '^j3sg)sj8rc@du74%fb$c2926tv!!4g(kp-=rx1)c5!1&1(dq='
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = []
# Application definition
SENTRY_CELERY_ENDPOINT = "http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
BROKER_URL = "redis://127.0.0.1:6379/8"
# CELERY_SEND_EVENTS = True
# CELERY_SEND_TASK_SENT_EVENT = True
#
# CELERY_DEFAULT_EXCHANGE = 'celery'
# CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
# CELERY_DEFAULT_ROUTING_KEY = 'celery'
#
# CELERY_QUEUES = {
# 'celery': {
# 'exchange': CELERY_DEFAULT_EXCHANGE,
# 'routing_key': CELERY_DEFAULT_ROUTING_KEY,
# },
# 'order': {
# 'exchange': 'order',
# 'routing_key': 'order',
# },
# }
CELERY_BROKER_URL = "redis://127.0.0.1:6379/8"
CELERY_TIMEZONE = 'Asia/Shanghai'
CELERY_ROUTES = ['physical.celery_task_router.CeleryTaskRouter']
INSTALLED_APPS = (
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'trans2es',
'search',
'injection.data_sync',
)
CELERYBEAT_SCHEDULE = {
'sync_face_similar_data_to_redis': {
'task': 'injection.data_sync.tasks.sync_face_similar_data_to_redis',
'schedule': timedelta(seconds=120),
'args': ()
},
}
"""
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
"""
ROOT_URLCONF = 'physical.urls'
WSGI_APPLICATION = 'physical.wsgi.application'
REDIS_URL = "redis://127.0.0.1:6379"
# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
DATABASE_ROUTERS = ['physical.DBRouter.DBRouter']
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'alpha',
'USER': 'work',
'PASSWORD': 'Gengmei123',
# 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
"HOST": 'rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com',
#'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
'PORT': '3306',
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4",
},
},
'face': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'face',
'USER': 'work',
'PASSWORD': 'Gengmei123',
# 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
"HOST": "rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com",
# 'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
'PORT': '3306',
# 'CONN_MAX_AGE': None,
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4",
},
}
}
ES_INFO_LIST = [
{
"host": "10.29.130.141",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
MIDDLEWARE_CLASSES = (
'gm_tracer.middleware.TracerMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'helios.DjangoL5dMiddleware',
)
# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/
# LANGUAGE_CODE = 'en-us'
#
TIME_ZONE = 'Asia/Shanghai'
#
USE_I18N = True
USE_L10N = True
#
# USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/
STATIC_URL = '/static/'
search/utils/topic.py
View file @ d3dde44a
......@@ -41,7 +41,6 @@ class TopicUtils(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0,"hits":[]}
@classmethod
def analyze_related_user_id_list(cls,related_user_id_list):
"""
......@@ -574,21 +573,43 @@ class TopicUtils(object):
return list()
@classmethod
def process_filters(cls, filters):
def process_filters(cls, filters, filter_online=True):
"""处理过滤器部分。"""
f = [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
]
if not filters:
return f
for k, v in filters.items():
if k == "group_id":
if k in ["create_time_gte", "create_time_lte"]:
if k == "create_time_gte":
op = "gte"
elif k == "create_time_lte":
op = "lte"
f.append({
"term": {"group_id": v},
"range": {
"create_time_val": {
op: v,
}
}
})
else:
if isinstance(k, list):
f.append({
"terms": {k: v},
})
else:
f.append({
"term": {k: v},
})
if filter_online:
f.append({"term": {"is_online": True}})
return f
......@@ -620,22 +641,29 @@ class TopicUtils(object):
},
})
elif sorts_by == TOPIC_SEARCH_SORT.VOTE_NUM:
pass
elif sorts_by == '2':
sort_rule.append({
"id":{
"order":"desc"
},
})
return sort_rule
@classmethod
def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10):
def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
must = cls.process_filters(filters, filter_online=True)
q = {
"query": {
"bool": {
"must": cls.process_filters(filters),
"must": must,
"must_not": cls.process_nfilters(nfilters),
}
},
"_source": {
"includes":["id"]
},
"sort": [],
}
}
if sorts_by:
......@@ -643,10 +671,18 @@ class TopicUtils(object):
q["sort"] = sorts
try:
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic",
query_body=q, offset=offset, size=size)
return result_dict["hits"]
result_dict = ESPerform.get_search_results(
ESPerform.get_cli(), sub_index_name=index_name,
query_body=q, offset=offset, size=size
)
return {
"hits": result_dict["hits"],
"total_count": result_dict["total_count"]
}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
return {
"hits": [],
"total_count": 0
}
search/views/business_topic.py 0 → 100644
View file @ d3dde44a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function
from gm_rpcd.all import bind
import logging
import traceback
import json
from search.utils.topic import TopicUtils
from libs.es import ESPerform
from libs.cache import redis_client
from search.utils.common import *
from libs.es import ESPerform
@bind("physical/search/business/topic")
def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。"""
try:
result_list = TopicUtils.list_topic_ids(
filters=filters, nfilters=nfilters, sorts_by=sorts_by,
offset=offset, size=size, filter_online=False,
index_name="topic"
)
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc())
return {"topic_ids": [], "total_count": 0}
search/views/topic.py
View file @ d3dde44a
......@@ -302,11 +302,14 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
try:
result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
topic_ids = [item["_source"]["id"] for item in result_list]
return {"topic_ids": topic_ids}
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {
"topic_ids": topic_ids,
"total_count": result_list["total_count"]
}
except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc())
return {"topic_ids": []}
return {"topic_ids": [], "total_count": 0}
@bind("physical/search/query_topic_by_user_similarity")
......@@ -319,6 +322,7 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
:return:
"""
try:
must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size,
must_topic_id_list=must_topic_id_list)
......
trans2es/management/commands/trans2es_mapping2es.py
View file @ d3dde44a
......@@ -6,29 +6,29 @@ from django.core.management.base import BaseCommand, CommandError
import traceback
import logging
from libs.es import ESPerform
from trans2es.type_info import get_type_info_map,TypeInfo
from trans2es.type_info import get_type_info_map, TypeInfo
class Command(BaseCommand):
args = ''
help = 'dump mapping to elasticsearch'
from optparse import make_option
option_list = BaseCommand.option_list + (
make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',default=''),
make_option('-T', '--indices_template', dest='indices_template', help='index template name to dump data to elasticsearch', metavar='TYPE',
make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',
default=''),
make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch', metavar='INDEX_PREFIX'),
make_option('-T', '--indices_template', dest='indices_template',
help='index template name to dump data to elasticsearch', metavar='TYPE',
default=''),
make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch',
metavar='INDEX_PREFIX'),
make_option('-p', '--parallel', dest='parallel', help='parallel process count', metavar='PARALLEL'),
make_option('-s', '--pks', dest='pks', help='specify sync pks, comma separated', metavar='PKS', default=''),
make_option('--streaming-slicing', dest='streaming_slicing', action='store_true', default=True),
make_option('--no-streaming-slicing', dest='streaming_slicing', action='store_false', default=True),
)
def handle(self, *args, **options):
try:
es_cli = ESPerform.get_cli()
......@@ -37,23 +37,21 @@ class Command(BaseCommand):
for type_name in type_name_list:
if len(options["type"]):
if type_name==options["type"]:
# ESPerform.put_index_mapping(es_cli, type_name, force_sync=True)
if options["type"] == "all" or type_name == options["type"]:
official_index_name = ESPerform.get_official_index_name(type_name)
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
logging.info("begin create [%s] index!" % type_name)
ESPerform.create_index(es_cli,type_name)
logging.info("begin create [%s] mapping!" % type_name)
ESPerform.put_index_mapping(es_cli,type_name,force_sync=True)
logging.info("begin create [%s] index and mapping!" % type_name)
ESPerform.create_index(es_cli, type_name)
ESPerform.put_index_mapping(es_cli, type_name)
else:
logging.warning("index:[%s] has already existing!" % type_name)
if len(options["indices_template"]):
template_file_name = options["indices_template"]
if ESPerform.put_indices_template(es_cli=es_cli,template_file_name=template_file_name,template_name=template_file_name):
if ESPerform.put_indices_template(es_cli=es_cli, template_file_name=template_file_name,
template_name=template_file_name):
logging.info("put indices template suc!")
else:
logging.error("put indices template err!")
......
trans2es/mapping/contrast_similar.json 0 → 100644
View file @ d3dde44a
{
"dynamic":"strict",
"properties": {
"id":{"type":"long"},
"is_online":{"type":"boolean"},//上线
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"is_deleted":{"type":"boolean"},//是否被删除
"participant_user_id":{"type":"long"},//参与者ID
"contrast_user_id_list":{"type":"long"},//对照者ID
"similarity":{"type":"double"},//相似度
}
}
\ No newline at end of file
trans2es/mapping/topic.json
View file @ d3dde44a
......@@ -25,6 +25,29 @@
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
"update_time_val":{"type":"long"},
"language_type":{"type":"long"}
"language_type":{"type":"long"},
"is_shadow": {"type": "boolean"},
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"virtual_content_level":{"type": "text"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
}
}
}
\ No newline at end of file
trans2es/models/__init__.py
View file @ d3dde44a
......@@ -13,3 +13,4 @@ from .tag import AccountUserTag
from .user import User
from .group import Group
from .topic import Topic
# from .pictorial import PictorialTopic
trans2es/models/contrast_similar.py 0 → 100644
View file @ d3dde44a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
import traceback
import logging
from django.db import models
import datetime
from .topic import Topic
manager = lambda: models.Manager().db_manager(using='face')
class ContrastSimilar(models.Model):
class Meta:
verbose_name = u"用户脸相似度表"
db_table = "face_user_contrast_similar"
objects = manager()
id = models.IntegerField(verbose_name="主键ID", primary_key=True)
is_online = models.BooleanField(verbose_name="是否上线")
is_deleted = models.BooleanField(verbose_name="是否删除")
participant_user_id = models.IntegerField(verbose_name=u'参与者用户ID')
contrast_user_id = models.IntegerField(verbose_name="对照者用户ID")
similarity = models.FloatField(verbose_name=u'相似度', default=0)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
def get_similar_user_id(self):
try:
similar_user_id_list = list()
if self.similarity > 0.3:
query_list = Topic.objects.filter(user_id=self.participant_user_id, is_deleted=False, is_online=True,
content_level__in=("3", "4", "5"))
for item in query_list:
if item.user_id != self.participant_user_id:
similar_user_id_list.append(item.id)
return similar_user_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
trans2es/models/face_user_contrast_similar.py
View file @ d3dde44a
......@@ -10,19 +10,21 @@ import datetime
manager = lambda: models.Manager().db_manager(using='face')
class FaceUserContrastSimilar(models.Model):
class Meta:
verbose_name=u"用户脸相似表"
db_table="face_user_contrast_similar"
app_label = "face"
verbose_name = u"用户脸相似表"
db_table = "face_user_contrast_similar"
objects = manager()
id = models.IntegerField(verbose_name="主键ID",primary_key=True)
id = models.IntegerField(verbose_name="主键ID", primary_key=True)
is_online = models.BooleanField(verbose_name="是否上线")
is_deleted = models.BooleanField(verbose_name="是否删除")
participant_user_id = models.IntegerField(verbose_name=u'参与者用户ID')
similarity = models.FloatField(verbose_name=u'相似度',default=0)
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
similarity = models.FloatField(verbose_name=u'相似度', default=0)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
contrast_user_id = models.IntegerField(verbose_name=u'对照者用户ID')
......
trans2es/models/pictorial.py 0 → 100644
View file @ d3dde44a
# from django.db import models
# class PictorialTopic(models.Model):
# """画报帖子关系"""
# class Meta:
# verbose_name = u'画报帖子关系'
# app_label = 'community'
# db_table = 'community_pictorial_topic'
# id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
# pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
# topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
# user_id = models.BigIntegerField(verbose_name=u'用户ID')
# is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
# is_online = models.BooleanField(verbose_name=u'是否上线')
# is_deleted = models.BooleanField(verbose_name=u'是否删除')
trans2es/models/topic.py
View file @ d3dde44a
......@@ -10,6 +10,8 @@ from libs.es import ESPerform
from django.db import models
import datetime
from alpha_types.venus import GRAP_PLATFORM
from .pick_topic import PickTopic
from .tag import TopicTag, Tag
from .user_extra import UserExtra
......@@ -70,6 +72,26 @@ class Topic(models.Model):
create_time = models.DateTimeField(verbose_name=u'日记创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
is_crawl = models.BooleanField(verbose_name="是否是爬回的数据", default=False)
has_image = models.BooleanField(verbose_name=u"是否有图片", default=False)
virtual_content_level = models.CharField(verbose_name=u'模拟内容等级', max_length=100, default='')
like_num_crawl = models.IntegerField(verbose_name=u'爬取点赞数', default=0)
comment_num_crawl = models.IntegerField(verbose_name=u'爬取评论数', default=0)
is_recommend = models.IntegerField(verbose_name=u'是否推荐 0 非推荐 1 推荐', default=0)
sort_score = models.FloatField(verbose_name='排序分', default=0)
is_shadow = models.BooleanField(verbose_name="是否是马甲账户", default=False)
platform = models.IntegerField(verbose_name=u'平台来源', choices=GRAP_PLATFORM, default=GRAP_PLATFORM.ALPHA)
platform_id = models.BigIntegerField(verbose_name='用平台ID', null=True)
@property
def is_complaint(self):
"""是否被举报"""
if TopicComplaint.objects.filter(topic_id=self.id, is_online=True).exists():
return True
return False
def topic_has_image(self):
try:
has_image = False
......@@ -173,3 +195,19 @@ class Topic(models.Model):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0.0
class TopicComplaint(models.Model):
"""帖子举报"""
class Meta:
verbose_name = u'帖子举报'
app_label = 'community'
db_table = 'topic_complaint'
id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
user_id = models.BigIntegerField(verbose_name=u'用户ID', db_index=True)
topic = models.ForeignKey(
Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE, related_name='complaints')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
\ No newline at end of file
trans2es/type_info.py
View file @ d3dde44a
......@@ -12,13 +12,14 @@ import elasticsearch
import elasticsearch.helpers
import sys
from trans2es.models import topic,user,pick_celebrity,group,celebrity,tag
from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar
from trans2es.utils.user_transfer import UserTransfer
from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
from trans2es.utils.group_transfer import GroupTransfer
from trans2es.utils.topic_transfer import TopicTransfer
from trans2es.utils.celebrity_transfer import CelebrityTransfer
from trans2es.utils.tag_transfer import TagTransfer
from trans2es.utils.contrast_similar_transfer import Contrast_Similar_Transfer
from libs.es import ESPerform
__es = None
......@@ -160,7 +161,7 @@ class TypeInfo(object):
# traceback.print_exc()
# es_result = 'error'
return ESPerform.es_helpers_bulk(es,data_list,sub_index_name,True)
return ESPerform.es_helpers_bulk(es, data_list, sub_index_name, True)
def elasticsearch_bulk_insert(self, sub_index_name, instance_iterable, es=None):
data_list = self.bulk_get_data(instance_iterable)
......@@ -266,11 +267,11 @@ def get_type_info_map():
round_insert_period=2,
),
TypeInfo(
name = "user", #用户
name="user", # 用户
type="user",
model=user.User,
#query_deferred=lambda:user.User.objects.all().query,
#query_deferred=lambda: user.User.objects.prefetch_related('myuserfollow').query,
# query_deferred=lambda:user.User.objects.all().query,
# query_deferred=lambda: user.User.objects.prefetch_related('myuserfollow').query,
query_deferred=lambda: user.User.objects.all().query,
get_data_func=UserTransfer.get_user_data,
bulk_insert_chunk_size=100,
......@@ -319,6 +320,16 @@ def get_type_info_map():
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='contrast_similar', # facesimilar
type='contrast_similar',
model=contrast_similar.ContrastSimilar,
query_deferred=lambda: contrast_similar.ContrastSimilar.objects.all().query,
get_data_func=Contrast_Similar_Transfer.get_contrast_similar_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2
)
]
......
trans2es/utils/contrast_similar_transfer.py 0 → 100644
View file @ d3dde44a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
from libs.tools import tzlc
import time
from trans2es.models.contrast_similar import ContrastSimilar
class Contrast_Similar_Transfer(object):
@classmethod
def get_contrast_similar_data(cls, instance):
try:
res = dict()
res["id"] = instance.id
res["is_online"] = instance.is_online
res["is_deleted"] = instance.is_deleted
res["participant_user_id"] = instance.participant_user_id
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
res["create_time"] = tzlc_create_time
update_time = instance.update_time
tzlc_update_time = tzlc(update_time)
res["update_time"] = tzlc_update_time
res["similarity"] = instance.similarity
res["contrast_user_id_list"] = ContrastSimilar.get_similar_user_id()
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment