Skip to content

P
physical
Commit e81a0cbc authored by lixiaofang's avatar lixiaofang
Browse files
Options

新增数据

parent 3dccfcd0
Hide whitespace changes
Inline Side-by-side
Showing with 272 additions and 92 deletions
physical/DBRouter.py
View file @ e81a0cbc
......@@ -12,24 +12,33 @@ class DBRouter:
"""
Attempts to read user models go to users_db.
"""
if model._meta.app_label == 'face':
return 'face'
# if model._meta.app_label == 'face':
# return 'face'
if model._meta.app_label == 'commodity':
return 'commodity'
return None
def db_for_write(self, model, **hints):
"""
Attempts to write user models go to users_db.
"""
if model._meta.app_label == 'face':
return 'face'
# if model._meta.app_label == 'face':
# return 'face'
if model._meta.app_label == 'commodity':
return 'commodity'
return None
def allow_relation(self, obj1, obj2, **hints):
"""
Allow relations if a model in the user app is involved.
"""
if obj1._meta.app_label == 'face' or \
obj2._meta.app_label == 'face':
# """
# if obj1._meta.app_label == 'face' or \
# obj2._meta.app_label == 'face':
# return True
if obj1._meta.app_label == 'commodity' or \
obj2._meta.app_label == 'commodity':
return True
return None
......@@ -38,6 +47,8 @@ class DBRouter:
Make sure the auth app only appears in the 'users_db'
database.
"""
if app_label == 'face':
return db == 'face'
# if app_label == 'face':
# return db == 'face'
if app_label == 'commodity':
return db == 'commodity'
return None
trans2es/mapping/product.json
View file @ e81a0cbc
......@@ -10,73 +10,17 @@
"update_time_val":{"type":"long"},//更新时间
"is_deleted":{"type":"boolean"}, //是否被删除
"price":{"type":"double"}, //价格
"cn_name_pre":{"type":"string","index":"not_analyzed"}, //
"en_name_pre":{"type": "string", "index": "not_analyzed"}, //
"cn_name_pre":{"type":"string","index":"not_analyzed"}, //商品名称
"en_name_pre":{"type": "string", "index": "not_analyzed"}, //商品原名
"alias":{"type": "string", "index": "not_analyzed"},
"description":{"type":"string","analyzer":"keyword","search_analyzer":"keyword"},
"have_image":{"type":"boolean"},
"vote_num":{"type":"long"},
"total_vote_num":{"type":"long"},
"reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"share_num":{"type":"long"},
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
"manual_score":{"type":"double"},//人工赋分
"has_image":{"type":"boolean"},//是否有图
"has_video":{"type":"boolean"},//是否是视频
"language_type":{"type":"long"},
"is_shadow": {"type": "boolean"},
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"virtual_content_level":{"type": "text"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "long"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "keyword",
"search_analyzer": "keyword"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"}, //是否历史数据
"related_billboard":{
"type":"nested",
"properties":{
"pictorial_id":{"type":"long"},
"topic_add_time":{"type":"long"},
"real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"}
}
}
"comment_nums":{"type":"long"},
"brand_cn_name":{"type":"string","index":"not_analyzed"}, //所属品牌名称
"brand_en_name":{"type":"string","index":"not_analyzed"}, //所属品牌原名
"brand_alias":{"type":"string","index":"not_analyzed"}, //所属品牌别名
"category_cn_name":{"type":"string","index":"not_analyzed"}, //所属类目的名称
"effect_cn_name":{"type":"string","index":"not_analyzed"}//所属功效的名称
}
}
......
trans2es/models/product.py 0 → 100644
View file @ e81a0cbc
import datetime
from django.db import models
import logging
import traceback
from .tag import Tag
from .topic import Topic
class CommodityProduct(models.Model):
"""画报关注"""
class Meta:
verbose_name = u"商品"
app_label = "commodity"
db_table = "commodity_product"
id = models.IntegerField(verbose_name=u'商品ID', primary_key=True)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
cn_name = models.CharField(verbose_name=u'商品名称', max_length=64)
en_name = models.CharField(verbose_name=u'商品原名', max_length=64, default="")
alias = models.CharField(verbose_name=u'别名', max_length=64)
image = models.CharField(verbose_name=u'图片', max_length=120)
description = models.CharField(verbose_name=u'商品描述', max_length=200)
comment_nums = models.IntegerField(verbose_name=u'评论数', max_length=11)
price = models.IntegerField(verbose_name="价格", max_length=11)
def get_brand_name(self):
try:
brand_id = CommodityProductBrand.objects.filter(product_id=self.id, is_deleted=False).values_list(
"brand_id", flat=True)
result_name = CommodityBrand.objects.filter(id=brand_id, is_online=True, is_deleted=False).values("cn_name",
"en_name",
"alias").first()
return result_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_category_en_name(self):
try:
category_id = CommodityProductCategory.objects.filter(product_id=self.id, is_deleted=False).values_list(
"category_id", flat=True)
result_name = CommodityCategory.objects.filter(id=category_id, is_online=True,
is_deleted=False).values_list("cn_name", flat=True).first()
return result_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ""
def get_effect_cn_name(self):
try:
effect_id = CommodityProductEffect.objects.filter(product_id=self.id, is_deleted=False).values_list(
"effect_id", flat=True)
result_name = CommodityEffect.objects.filter(id=effect_id, is_deleted=False).values_list("cn_name",
flat=True).first()
return result_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ""
class CommodityBrand(models.Model):
"""品牌"""
class Meta:
verbose_name = u'品牌'
app_label = 'commodity'
db_table = 'commodity_brand'
id = models.IntegerField(verbose_name=u'品牌ID', primary_key=True)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
cn_name = models.CharField(verbose_name=u'品牌名称', max_length=64)
en_name = models.CharField(verbose_name=u'品牌原名', max_length=64)
alias = models.CharField(verbose_name=u'别名', max_length=64)
description = models.CharField(verbose_name=u'品牌描述', max_length=200)
class CommodityCategory(models.Model):
"""类目"""
class Meta:
verbose_name = u'类目'
app_label = 'commodity'
db_table = 'commodity_category'
id = models.IntegerField(verbose_name=u'品牌ID', primary_key=True)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
cn_name = models.CharField(verbose_name=u'品牌名称', max_length=64)
class CommodityEffect(models.Model):
"""功效"""
class Meta:
verbose_name = u'功效'
app_label = 'commodity'
db_table = 'commodity_effect'
id = models.IntegerField(verbose_name=u'品牌ID', primary_key=True)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
is_deleted = models.BooleanField(verbose_name=u'是否删除')
cn_name = models.CharField(verbose_name=u'品牌名称', max_length=64)
class CommodityProductBrand(models.Model):
"""商品品牌关系"""
class Meta:
verbose_name = u'商品品牌关系'
app_label = 'commodity'
db_table = 'commodity_productbrand'
id = models.IntegerField(verbose_name=u'ID', primary_key=True)
product_id = models.BigIntegerField(verbose_name=u'商品ID')
brand_id = models.BigIntegerField(verbose_name=u'品牌ID')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
class CommodityProductCategory(models.Model):
"""画报关注"""
class Meta:
verbose_name = u"画报"
app_label = "commodity"
db_table = "commodity_productcategory"
id = models.IntegerField(verbose_name=u'关注ID', primary_key=True)
is_deleted = models.BooleanField(verbose_name=u'是否删除')
product_id = models.BigIntegerField(verbose_name=u'商品ID')
category_id = models.BigIntegerField(verbose_name=u'分类ID')
class CommodityProductEffect(models.Model):
"""画报关注标签"""
class Meta:
verbose_name = u"画报标签"
app_label = "commodity"
db_table = "commodity_producteffect"
id = models.IntegerField(verbose_name=u'关注ID', primary_key=True)
is_deleted = models.BooleanField(verbose_name=u'是否删除')
product_id = models.BigIntegerField(verbose_name=u'商品ID')
effect_id = models.BigIntegerField(verbose_name=u'功效ID')
trans2es/type_info.py
View file @ e81a0cbc
......@@ -14,7 +14,7 @@ import sys
from libs.cache import redis_client
import copy
from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar,pictorial
from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar, pictorial, product
from trans2es.utils.user_transfer import UserTransfer
from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
from trans2es.utils.group_transfer import GroupTransfer
......@@ -24,7 +24,7 @@ from trans2es.utils.pictorial_transfer import PictorialTransfer
from trans2es.utils.celebrity_transfer import CelebrityTransfer
from trans2es.utils.tag_transfer import TagTransfer
from trans2es.utils.contrast_similar_transfer import Contrast_Similar_Transfer
from trans2es.utils.product_transfer import ProductTransfer
__es = None
......@@ -196,7 +196,7 @@ class TypeInfo(object):
old_data["is_history"] = True
data_list.append(old_data)
if int_ori_topic_star>=4:
if int_ori_topic_star >= 4:
topic_data_high_star_list.append(old_data)
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
# data_list = [
......@@ -207,12 +207,11 @@ class TypeInfo(object):
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
if data["content_level"] and int(data["content_level"])>=4:
if data["content_level"] and int(data["content_level"]) >= 4:
topic_data_high_star_list.append(data)
data_list.append(data)
return (data_list,topic_data_high_star_list)
return (data_list, topic_data_high_star_list)
def elasticsearch_bulk_insert_data(self, sub_index_name, data_list, es=None):
......@@ -255,17 +254,17 @@ class TypeInfo(object):
else:
qs = self.model.objects.all()
end = time.time()
time0=end-begin
time0 = end - begin
begin = time.time()
instance_list = qs.filter(pk__in=pk_list)
end = time.time()
time1=end-begin
time1 = end - begin
begin = time.time()
data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
end = time.time()
time2=end-begin
time2 = end - begin
begin = time.time()
# logging.info("get sub_index_name:%s"%sub_index_name)
......@@ -277,7 +276,7 @@ class TypeInfo(object):
es=es,
)
if sub_index_name=="topic":
if sub_index_name == "topic":
self.elasticsearch_bulk_insert_data(
sub_index_name="topic-star-routing",
data_list=data_list,
......@@ -285,7 +284,7 @@ class TypeInfo(object):
)
# 同时写4星及以上的帖子
if len(topic_data_high_star_list)>0:
if len(topic_data_high_star_list) > 0:
self.elasticsearch_bulk_insert_data(
sub_index_name="topic-high-star",
data_list=topic_data_high_star_list,
......@@ -293,9 +292,8 @@ class TypeInfo(object):
)
end = time.time()
time3=end-begin
logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0,time1,time2,time3))
time3 = end - begin
logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0, time1, time2, time3))
def insert_table_chunk(self, sub_index_name, table_chunk, es=None):
try:
......@@ -317,7 +315,7 @@ class TypeInfo(object):
auto_create_index=True
)
logging.info("es_helpers_bulk,sub_index_name:%s,data_list len:%d" % (sub_index_name,len(data_list)))
logging.info("es_helpers_bulk,sub_index_name:%s,data_list len:%d" % (sub_index_name, len(data_list)))
stage_3_time = time.time()
end_clock = time.clock()
......@@ -355,8 +353,8 @@ def get_type_info_map():
name='topic-star',
type='topic-star',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,#假的
get_data_func=TopicTransfer.get_topic_data,#假的
query_deferred=lambda: topic.Topic.objects.all().query, # 假的
get_data_func=TopicTransfer.get_topic_data, # 假的
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
......@@ -505,7 +503,7 @@ def get_type_info_map():
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
)
),
# TypeInfo(
# name="account_user_tag", # 用户标签
# type="account_user_tag",
......@@ -516,6 +514,16 @@ def get_type_info_map():
# round_insert_chunk_size=5,
# round_insert_period=2,
# )
TypeInfo(
name="product", # 商品
type="product",
model=product.CommodityProduct,
query_deferred=lambda: product.CommodityProduct.objects.all().query,
get_data_func=ProductTransfer.get_product_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
)
]
type_info_map = {
......@@ -525,4 +533,3 @@ def get_type_info_map():
_get_type_info_map_result = type_info_map
return type_info_map
trans2es/utils/product_transfer.py 0 → 100644
View file @ e81a0cbc
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
import time
from libs.tools import tzlc
class ProductTransfer(object):
@classmethod
def get_product_data(cls, instance):
try:
res = dict()
res["id"] = instance.id
res["is_online"] = instance.is_online
res["is_deleted"] = instance.is_deleted
res["create_time"] = tzlc(instance.create_time)
res["update_time"] = tzlc(instance.update_time)
res["create_time_val"] = int(time.mktime(pictorial_id.get("create_time", None).timetuple()))
res["update_time_val"] = int(time.mktime(pictorial_id.get("update_time", None).timetuple()))
res["price"] = instance.price
res["cn_name_pre"] = instance.cn_name
res["en_name_pre"] = instance.en_name
res["alias"] = instance.alias
res['description'] = instance.description
res["have_image"] = True if instance.image else False
res["comment_nums"] = instance.comment_nums
result_name = instance.get_brand_name()
if result_name:
res["brand_cn_name"] = result_name.get("cn_name", "")
res["brand_en_name"] = result_name.get("en_name", "")
res["brand_alias"] = result_name.get("alias", "")
else:
res["brand_cn_name"] = ""
res["brand_en_name"] = ""
res["brand_alias"] = ""
category_cn_name = instance.get_category_en_name()
if category_cn_name:
res["category_cn_name"] = category_cn_name
else:
res["category_cn_name"] = ""
effect_cn_name = instance.get_effect_cn_name()
if effect_cn_name:
res['effect_cn_name'] = effect_cn_name
else:
res["effect_cn_name"] = ""
logging.info("get product:%s" % res)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment