#!/usr/bin/env python # -*- coding: utf-8 -*- from django.conf import settings from django.db import models from django.utils import timezone import logging import traceback import datetime, time, json, redis from django.utils.functional import cached_property from talos.services.tag import TagService from gm_serializer import fields from gm_types.gaia import VIDEO_CODE_STATUS from gm_types.mimas import ( TRACTATE_STATUS, TRACTATE_PLATFORM, TRACTATE_CONTENT_LEVEL, PGC_TYPE, MEDIA_IMAGE_URL_SOURCE, IMAGE_TYPE, TRACATE_VIDEO_URL_SOURCE, TRACTATE_COVER_TYPE, ) from data_sync.utils import to_epoch, tzlc from gm_upload import ImgUrlField, IMG_TYPE from talos.models.tractate.reply import TractateReply from talos.models.tractate.favor import TractateFavor from talos.models.tractate.vote import TractateVote from talos.services.convert_service.user_convert_service import UserConvertService from gm_types.gaia import ( DOCTOR_TYPE, ) from talos.cache.base import tractate_pv_cache, tractate_favor_count_cache, tractate_vote_count_cache from utils.rpc import RPCMixin from utils.pic import PictureTools doris_redis_client = redis.StrictRedis.from_url(settings.REDIS_URL) class UserManager(RPCMixin): def __call__(self, pk_list): """ :param pk_list: :return: """ return self.call_rpc('api/user/get_fundamental_info_by_user_ids', user_ids=pk_list) class TagManager(RPCMixin): def __call__(self, pk_list): return self.call_rpc('api/tag/info_by_ids', tag_ids=pk_list) class Tractate(models.Model): """ 短文(新帖子) """ class Meta: verbose_name = u'新帖子' db_table = 'api_tractate' app_label = 'talos' # user = fields.MagicField(type=int, manager=UserManager, ttl=60 * 5, db_column="user_id") user_id = models.IntegerField(verbose_name="用户id", db_index=True) title = models.CharField(max_length=512, verbose_name=u'标题', default="") content = models.TextField(verbose_name=u'正文描述', default="") is_online = models.BooleanField(verbose_name=u"是否在线", default=True) status = models.CharField(verbose_name=u"审核状态", max_length=12, choices=TRACTATE_STATUS, default=TRACTATE_STATUS.UNAUDITED) platform = models.CharField(verbose_name=u"数据来源", max_length=12, choices=TRACTATE_PLATFORM) content_level = models.CharField(verbose_name=u"内容等级", max_length=12, choices=TRACTATE_CONTENT_LEVEL, default=TRACTATE_CONTENT_LEVEL.NULL) is_excellent = models.BooleanField(verbose_name=u"加精", default=False) create_time = models.DateTimeField(verbose_name="创建时间", default=timezone.now) last_modified = models.DateTimeField(verbose_name=u'最后修改时间', auto_now=True) audit_time = models.DateTimeField(verbose_name=u'最近一次的审核时间', null=True, default=None) user_del = models.BooleanField(verbose_name=u"用户是否删除", default=False) low_quality = models.IntegerField(default=0, verbose_name=u'低质量反馈数量', blank=True, null=False) low_quality_deal = models.BooleanField(default=False, verbose_name=u'低质量反馈是否处理', blank=True) platform_id = models.CharField(verbose_name=u"数据来源id", max_length=125, db_index=True) pgc_type = models.SmallIntegerField(verbose_name='pgc类别', choices=PGC_TYPE, default=PGC_TYPE.DDEFAULT) cover_url = models.CharField('封面地址', max_length=256, default='') cover_type = models.IntegerField('封面类型', choices=TRACTATE_COVER_TYPE, default=TRACTATE_COVER_TYPE.NO_COVER) can_optimize = models.BooleanField('是否可进行优化', default=False) @property def view_amount(self): """ 浏览量 :return: """ return int(tractate_pv_cache.get(str(self.id)) or 0) @property def vote_amount(self): """ 点赞数 :return: """ return self.base_vote_num + self.real_vote_amount @property def real_vote_amount(self): """ 真实点赞数 """ return int(tractate_vote_count_cache.get(str(self.id)) or 0) @property def base_vote_num(self): """ 后台配置的点赞基数 """ return int(tractate_vote_count_cache.hget('base_vote_num', str(self.id)) or 0) @base_vote_num.setter def base_vote_num(self, value): """ 设置点赞基数 """ if value: tractate_vote_count_cache.hset('base_vote_num', str(self.id), int(value or 0)) else: tractate_vote_count_cache.hdel('base_vote_num', str(self.id)) @cached_property def reply_amount(self): """ 评论数 :return: """ try: extra = TractateExtra.objects.get(tractate_id=self.id) except TractateExtra.DoesNotExist: return 0 return extra.reply_count @property def favor_amount(self): """ 收藏数 :return: """ return int(tractate_favor_count_cache.get(str(self.id)) or 0) @property def get_tractate_score(self): """ 帖子分数 :return: """ try: results = list( TractateScore.objects.filter(tractate_id=self.id, type="user").values_list("tractate_score", flat=True)) _score = results and results[0] or 0 return _score except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 @property def get_good_click_tractate_score(self): """ 帖子加上godd_click分数 :return: """ try: results = list( TractateScoreV2.objects.filter(tractate_id=self.id).values_list("new_score", flat=True)) good_click_score = results and results[0] or 0 return good_click_score except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 @property def cover_is_dynamic(self): if self.cover_url: if self.cover_type == TRACTATE_COVER_TYPE.VIDEO: return True else: if PictureTools.is_dynamic(self.cover_url): return True return False if TractateVideo.objects.filter(tractate_id=self.id).exists(): return True img = TractateImages.objects.filter(tractate_id=self.id).order_by("-id").first() if img and PictureTools.is_dynamic(img.image_url): return True return False def get_update_time_stratific(self, create_time): try: now = datetime.datetime.now() d2 = datetime.datetime.strptime(now.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S') delta_30 = datetime.timedelta(days=30) delta_90 = datetime.timedelta(days=90) delta_365 = datetime.timedelta(days=365) if d2 - delta_30 <= create_time: return 30 elif d2 - delta_90 <= create_time: return 90 elif d2 - delta_365 <= create_time: return 365 else: return 1000 except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 1000 @property def get_tractate_image(self): """ 是否有图片 :return: """ try: image_url = list( TractateImages.objects.filter(tractate_id=self.id).values_list("image_url", flat=True)) if image_url: return 1 else: return 0 except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 @property def get_goodclick(self): try: from talos.models.soft_article.soft_article import PageGoodClick results = list( PageGoodClick.objects.filter(business_id=self.id, page_name="user_post_detail") .order_by('-create_date').values_list("avg_new_gc", flat=True)) good_click_score = results and results[0] or 0 return good_click_score except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 @property def con_goodclick(self): try: from talos.models.soft_article.soft_article import GoodClickCom today = datetime.datetime.now() - datetime.timedelta(days=1) lastday = str(datetime.datetime(today.year, today.month, today.day)) lastday = lastday.split(" ") date_change = lastday[0].replace("-", "") results = list( GoodClickCom.objects.using("doris").filter(business_id=str(self.id), page_name="user_post_detail", create_date=str(date_change)).values_list( "goodclick_rate_30", flat=True)) good_click_score = results and results[0] or 0 return good_click_score except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 @property def get_tag_list(self): """ 帖子关联标签 :return: """ try: tag_id_list = list(TractateTag.objects.filter(tractate_id=self.id).values_list("tag_id", flat=True)) return tag_id_list except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return [] @property def get_fresh_tag_list(self): """ 帖子关联标签 :return: """ try: tag_id_list = list(TractateNewTag.objects.filter(tractate_id=self.id).values_list("tag_id", flat=True)) return tag_id_list except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return [] @property def get_is_video(self): """ 判断该帖子是否有视频 :return: """ try: bol = TractateVideo.objects.filter(tractate_id=self.id, is_online=True) if len(bol): return True else: return False except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) # 获取贴子标签和名称 def get_tag(self, tag_list): try: association_tags_id_list = list() tag_list = TagService._get_by_ids_from_cache_type(tag_list) for item in tag_list: association_tags_id_list.append( {"id": item.id, "tag_name": item.name, "recommend_type": item.recommend_type, "tag_type": item.tag_type}) return association_tags_id_list except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return [] def get_tag_names(self, tag_list): try: association_tags_id_list = list() tag_list = TagService._get_by_ids_from_cache_type(tag_list) for item in tag_list: if int(item.recommend_type) != 5 or int(item.tag_type) != 11: association_tags_id_list.append(item.name) return association_tags_id_list except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return [] def get_fresh_tag_names(self, new_tag_ids): try: from talos.services import AgileTagService tag_name_list = list() agile_tag_info_dic = AgileTagService.get_agile_tags_by_agile_tag_ids(new_tag_ids) for item in new_tag_ids: data = agile_tag_info_dic.get(item, None) tag_name_list.append(data.name) return tag_name_list except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return [] def get_tag_names_content(self, tag_list): try: association_tags_id_list = list() tag_list = TagService._get_by_ids_from_cache_type(tag_list) for item in tag_list: if int(item.recommend_type) == 5 and int(item.tag_type) == 11: association_tags_id_list.append(item.name) return association_tags_id_list except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return [] def get_hot_score(self): try: now = datetime.datetime.now() three_day = datetime.datetime.now() - datetime.timedelta(days=60) yesterday_begin_time = "%s-%s-%s 00:00:00" % (three_day.year, three_day.month, three_day.day) # 点赞数+3评论数+5收藏数 # 获得点赞数 vote_num = TractateVote.objects.filter(tractate_id=self.id, create_time__gte=yesterday_begin_time, create_time__lte=now, is_online=True).count() # 获得评论数 reply_num = TractateReply.objects.filter(tractate_id=self.id, create_time__gte=yesterday_begin_time, create_time__lte=now, is_online=True).count() # 获得收藏数 click_num = TractateFavor.objects.filter(tractate_id=self.id, create_time__gte=yesterday_begin_time, create_time__lte=now, is_deleted=False).count() return int(vote_num) + 3 * int(reply_num) + 5 * int(click_num) except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 def get_show(self): try: redis_d = doris_redis_client.get("tractate_simi_tids") if redis_d: tractate_ids = json.loads(redis_d.decode('utf-8')) must_not_show = tractate_ids.get("must_not_show", []) if self.id in must_not_show: return 0 else: return 1 except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 1 def get_tractate_last_any_reply_time(self): try: result = TractateReply.objects.filter(tractate_id=self.id).values_list("create_time", flat=True).order_by( "-create_time").first() return result except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return '' def get_user_info_office(self): try: is_office = 0 result = UserConvertService.get_user_info_by_user_id(self.user_id) if result.get("doctor_id") and result.get("doctor_type") == DOCTOR_TYPE.OFFICER: is_office = 1 return is_office except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 # 获取帖子最新互动时间 def get_tractate_latest_interaction_time(self, is_online, content_level, last_modified): try: update_time_value = -1 if is_online: update_time_value = int(time.mktime(last_modified.timetuple())) if last_modified else -1 # 最新点赞时间 vote_result = TractateVote.objects.filter(tractate_id=self.id).values_list("update_time", flat=True).order_by( "-update_time").first() vote_time_value = int(time.mktime(tzlc(vote_result).timetuple())) if vote_result else -1 if vote_time_value > update_time_value: update_time_value = vote_time_value # 最新回复时间 reply_result = TractateReply.objects.filter(tractate_id=self.id).values_list("update_time", flat=True).order_by( "-update_time").first() reply_time_value = int(time.mktime(tzlc(reply_result).timetuple())) if reply_result else -1 if reply_time_value > update_time_value: update_time_value = reply_time_value # 小组创建时间 tractate_group_redis_name = "doris:tractate_group_data" redis_tractate_group_data = doris_redis_client.hget(tractate_group_redis_name, str(self.id)) tractate_group_id_list = json.loads( str(redis_tractate_group_data, encoding="utf-8")) if redis_tractate_group_data else [] if len(tractate_group_id_list) > 0: from talos.models.soft_article.soft_article import Group group_update_time_result = Group.objects.using(settings.ZHENGXING_DB).filter( id__in=tractate_group_id_list).values_list( "create_time", flat=True).order_by("-create_time").first() group_update_time_value = int( time.mktime(tzlc(group_update_time_result).timetuple())) if group_update_time_result else -1 if group_update_time_value > update_time_value: update_time_value = group_update_time_value + 5 return update_time_value except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return -1 def get_tractate_new_smart_rank_score(self, tractate_id): try: today = datetime.datetime.now().date() delta = datetime.timedelta(days=1) n_days = today - delta date_change = str(n_days).replace("-", "") score = StrategyTractateSmrScoreV2.objects.using(settings.DORIS_DB_NAME).filter( tractate_id=tractate_id).order_by('-create_date').values_list("smart_rank_score", flat=True).first() if score: return score else: return 0 except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 0 def get_tractate_newuser_smr(self, tractate_id): score = SrategyTractateSmrScore.objects.using(settings.DORIS_DB_NAME).filter( tractate_id=tractate_id).order_by('-create_date').values_list("smart_rank_score", flat=True).first() if score: return score else: return 0 def get_search_tractate_new_smart_rank_score(self, tractate_id): try: today = datetime.datetime.now().date() delta = datetime.timedelta(days=1) n_days = today - delta date_change = str(n_days).replace("-", "") score = SearchStrategyTractateSmrScore.objects.using(settings.DORIS_DB_NAME).filter( tractate_id=tractate_id).order_by('-create_date').values("smart_rank_score", "new_goodclick").first() if score: return score else: return {} except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return {} def get_show_by_index(self, tractate_id): try: today = datetime.datetime.now().date() delta = datetime.timedelta(days=2) n_days = today - delta data = StrategyContentExposureIndex.objects.using(settings.DORIS_DB_NAME).filter(create_day=n_days, card_id=tractate_id, card_content_type="user_post").first() if data and data.ctr >= 0.04 and data.preciseexposure_num >= 30 and data.avg_page_stay >= 15: return 0 # 0是正常展示 1不展示 else: return 1 except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return 1 def get_tractate_tagv4_names(self, tractate_id): try: data = TractateTagV4.objects.using(settings.HERA_READ_DB).filter(tractate_id=tractate_id).values_list( "tag_v4_id", flat=True) return data except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) return [] class TractateExtra(models.Model): """ 新帖子 数据相关 """ class Meta: verbose_name = u'新帖子其他数据表' db_table = 'api_tractate_extra' app_label = 'talos' tractate_id = models.IntegerField(verbose_name="新帖子id", unique=True) reply_count = models.IntegerField(verbose_name="所有评论数", default=0) vote_count = models.IntegerField(verbose_name="点赞数", default=0) favor_count = models.IntegerField(verbose_name="收藏数", default=0) class TractateTag(models.Model): """ 新帖子 标签相关 """ class Meta: verbose_name = u'新帖子标签相关' db_table = 'api_tractate_tag' app_label = 'talos' tractate_id = models.IntegerField(verbose_name="新帖子id", db_index=True) # tag = fields.MagicField(type=int, manager=TagManager, ttl=60 * 60 * 24, db_column="tag_id") tag_id = models.IntegerField(verbose_name="标签id", db_index=True) create_time = models.DateTimeField(verbose_name=u'创建时间', default=timezone.now) update_time = models.DateTimeField(verbose_name=u'更新时间', auto_now=True) class TractateNewTag(models.Model): """ 新帖子 标签相关 """ class Meta: verbose_name = u'新帖子标签相关' db_table = 'api_tractate_new_tag' app_label = 'talos' tractate_id = models.IntegerField(verbose_name="新帖子id", db_index=True) tag_id = models.IntegerField(verbose_name="新标签id", db_index=True) create_time = models.DateTimeField(verbose_name=u'创建时间', default=timezone.now) update_time = models.DateTimeField(verbose_name=u'更新时间', auto_now=True) class TractateTagV3(models.Model): class Meta: db_table = 'api_tractate_tag_v3' app_label = "talos" tractate_id = models.IntegerField(verbose_name="新帖子id", db_index=True) tag_v3_id = models.IntegerField(verbose_name="标签V3", db_index=True) class TractateTagV4(models.Model): class Meta: db_table = 'api_tractate_tag_v4' app_label = "talos" tractate_id = models.IntegerField(verbose_name="新帖子id", db_index=True) tag_v4_id = models.IntegerField(verbose_name="标签V4", db_index=True) class TractateImages(models.Model): """ 新帖子 图片相关 默认第一张图片为封面图,故未设置 is_cover 字段 """ class Meta: verbose_name = u'新帖子图片相关' db_table = 'api_tractate_images' app_label = 'talos' tractate_id = models.IntegerField(verbose_name="新帖子id", db_index=True) image_url = ImgUrlField(img_type=IMG_TYPE.TOPICIMAGE, max_length=300, verbose_name=u'图片地址') width = models.IntegerField(verbose_name="图片宽度", default=0) height = models.IntegerField(verbose_name="图片高度", default=0) image_url_source = models.CharField(verbose_name="图片地址来源(创建、富文本)", max_length=3, choices=MEDIA_IMAGE_URL_SOURCE, default=MEDIA_IMAGE_URL_SOURCE.HEAD) image_type = models.IntegerField(verbose_name="图片类型", default=IMAGE_TYPE.OTHER) image_webp = models.CharField(max_length=128, default="", verbose_name=u'webp格式的图片') create_time = models.DateTimeField(verbose_name=u'创建时间', default=timezone.now) update_time = models.DateTimeField(verbose_name=u'更新时间', auto_now=True) is_cover = models.BooleanField(verbose_name=u'是否封面', default=False) class TractateVideo(models.Model): """ 新帖子 视频相关 """ class Meta: verbose_name = u'新帖子视频相关' db_table = 'api_tractate_video' app_label = 'talos' tractate_id = models.IntegerField(verbose_name="新帖子id", db_index=True) video_cover_url = models.CharField(max_length=300, verbose_name="视频封面图地址", default="") width = models.IntegerField(verbose_name="视频封面图片宽度", default=0) height = models.IntegerField(verbose_name="视频封面图片高度", default=0) raw_video_url = models.CharField(max_length=128, verbose_name=u"视频地址") water_video_url = models.CharField(max_length=128, verbose_name=u"带图片水印的视频地址", default="") persistent_id = models.CharField(max_length=128, verbose_name=u'七牛视频处理id', default="") persistent_status = models.IntegerField(verbose_name=u"七牛状态", choices=VIDEO_CODE_STATUS, default=VIDEO_CODE_STATUS.NOSTART) is_online = models.BooleanField(verbose_name=u"是否在线", default=True) persistent_clip_id = models.CharField(max_length=128, verbose_name=u'七牛视频截取处理id', default="") persistent_clip_status = models.IntegerField(default=VIDEO_CODE_STATUS.NOSTART, verbose_name=u"七牛截取状态", db_index=True) intercept_video_url = models.CharField(max_length=128, verbose_name=u"截取的视频地址", default="") webp_url = models.CharField(max_length=128, verbose_name=u'webp图片地址', default="") video_url_source = models.CharField(verbose_name="视频地址来源(头部、富文本)", max_length=3, choices=TRACATE_VIDEO_URL_SOURCE, default=TRACATE_VIDEO_URL_SOURCE.HEAD) create_time = models.DateTimeField(verbose_name=u'创建时间', default=timezone.now) update_time = models.DateTimeField(verbose_name=u'更新时间', auto_now=True) @classmethod def cleaned_video_url(cls, video_url): return video_url.replace(settings.VIDEO_HOST, '') class TractateScore(models.Model): """ 新帖子 排序分 """ class Meta: verbose_name = u'新帖子视频相关' db_table = 'api_tractate_score' app_label = 'doris' tractate_id = models.IntegerField(verbose_name="新帖子id") mean = models.FloatField(verbose_name="平均分") tractate_score = models.FloatField(verbose_name="新帖子分数", default=mean) new_score = models.FloatField(verbose_name="新帖子分数", default=mean) type = models.TextField(verbose_name="新帖子类型") class TractateScoreV2(models.Model): class Meta: verbose_name = u'新帖子smart rank 分' db_table = 'api_tractate_score_v2' app_label = 'doris' tractate_id = models.IntegerField(verbose_name="新帖子id") new_score = models.FloatField(verbose_name="新帖子分数") class TractateRelationService(models.Model): """ 帖子关联美购 """ class Meta: verbose_name = u'帖子关联美购信息' db_table = 'api_tractate_service' app_label = 'talos' tractate_id = models.IntegerField(verbose_name="新帖子id") service_id = models.IntegerField(verbose_name="美购id") from_order = models.BooleanField(verbose_name="作者是否通过订单关联这个美购", default=False) # class PGCTractate(models.Model): # """ # pgc 不为属性不为0的帖子 # """ # tractate_id = models.IntegerField(verbose_name="新帖子id") # pgc_type = models.SmallIntegerField(verbose_name='pgc类别', choices=PGC_TYPE) # tractate_score = models.FloatField(verbose_name="smart_rank值", default=mean) # tractate_create_time = models.DateTimeField(verbose_name=u'创建时间') # update_time = models.DateTimeField(verbose_name=u'更新时间', auto_now=True) # is_online = models.SmallIntegerField(verbose_name="是否上线", default=True) class StrategyTractateSmrScore(models.Model): class Meta: db_table = 'strategy_tractate_new_smart_rank_score' tractate_id = models.IntegerField(unique=True) smart_rank_score = models.FloatField(verbose_name=u'新的smr_score', blank=True, default=0) create_date = models.BigIntegerField(max_length=50, verbose_name="时间") class StrategyTractateSmrScoreV2(models.Model): class Meta: db_table = 'search_strategy_tractate_new_smart_rank_score_v2' tractate_id = models.IntegerField(unique=True) smart_rank_score = models.FloatField(verbose_name=u'新的smr_score', blank=True, default=0) create_date = models.BigIntegerField(max_length=50, verbose_name="时间") class SrategyTractateSmrScore(models.Model): class Meta: db_table = 'strategy_tractate_smr_score' tractate_id = models.IntegerField(unique=True) smart_rank_score = models.FloatField(verbose_name=u'新的smr_score', blank=True, default=0) create_date = models.BigIntegerField(max_length=50, verbose_name="时间") class SearchStrategyTractateSmrScore(models.Model): class Meta: db_table = 'search_strategy_tractate_new_smart_rank_score' tractate_id = models.IntegerField(unique=True) smart_rank_score = models.FloatField(verbose_name=u'新的smr_score', blank=True, default=0) create_date = models.BigIntegerField(max_length=50, verbose_name="时间") new_goodclick = models.FloatField(verbose_name=u'新的点击分', blank=True, default=0) class StrategyContentExposureIndex(models.Model): class Meta: db_table = 'strategy_content_exposure_index' card_id = models.IntegerField(verbose_name=u'帖子ID') create_day = models.CharField(verbose_name=u'创建时间', max_length=128) card_content_type = models.CharField(verbose_name=u'卡片ID', max_length=128) click_num = models.IntegerField(verbose_name=u'点击数') preciseexposure_num = models.IntegerField(verbose_name=u'精准曝光数') ctr = models.FloatField(verbose_name=u'点击/精准曝光') avg_page_stay = models.FloatField(verbose_name=u'人均停留时长')