Merge branch 'dlis' into 'master'

Dlis

See merge request !118

trans2es/models/topic.py

@@ -10,34 +10,35 @@ from django.db import models
 import datetime
 
 from .pick_topic import PickTopic
-from .tag import TopicTag,Tag
+from .tag import TopicTag, Tag
 from .user_extra import UserExtra
 from .group import Group
 
 
 class ActionSumAboutTopic(models.Model):
     class Meta:
-        verbose_name=u"帖子埋点数据汇总"
-        db_table="action_sum_about_topic"
+        verbose_name = u"帖子埋点数据汇总"
+        db_table = "action_sum_about_topic"
 
-    partiton_date = models.CharField(verbose_name=u"日期",max_length=20)
-    device_id = models.CharField(verbose_name=u"用户设备号",max_length=50)
-    topic_id = models.CharField(verbose_name=u"帖子ID",max_length=50)
-    user_id = models.CharField(verbose_name=u"用户ID",max_length=50)
+    partiton_date = models.CharField(verbose_name=u"日期", max_length=20)
+    device_id = models.CharField(verbose_name=u"用户设备号", max_length=50)
+    topic_id = models.CharField(verbose_name=u"帖子ID", max_length=50)
+    user_id = models.CharField(verbose_name=u"用户ID", max_length=50)
     data_type = models.IntegerField(verbose_name=u"动作类型")
     data_value = models.BigIntegerField(verbose_name=u"值")
 
+
 class TopicImage(models.Model):
     class Meta:
         verbose_name = u'日记图片'
         db_table = 'topic_image'
 
-    id = models.IntegerField(verbose_name='日记图片ID',primary_key=True)
+    id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
     topic_id = models.IntegerField(verbose_name=u'日记ID')
-    url = models.CharField(verbose_name=u'图片URL',max_length=300)
+    url = models.CharField(verbose_name=u'图片URL', max_length=300)
     is_online = models.BooleanField(verbose_name='是否上线')
     is_deleted = models.BooleanField(verbose_name='是否删除')
-    create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
+    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
 
 
@@ -46,34 +47,34 @@ class Topic(models.Model):
         verbose_name = u'日记'
         db_table = 'topic'
 
-    id = models.IntegerField(verbose_name=u'日记ID',primary_key=True)
-    name = models.CharField(verbose_name=u'日记名称',max_length=100)
-    #group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
+    id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
+    name = models.CharField(verbose_name=u'日记名称', max_length=100)
+    # group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
     group = models.ForeignKey(
-        Group, verbose_name=u"关联的小组", related_name=u"group_topics",null=True, blank=True, default=None, on_delete=models.CASCADE)
+        Group, verbose_name=u"关联的小组", related_name=u"group_topics", null=True, blank=True, default=None,
+        on_delete=models.CASCADE)
     user_id = models.IntegerField(verbose_name=u'用户ID')
-    drop_score = models.IntegerField(verbose_name=u'人工赋分',default=0)
-    description = models.CharField(verbose_name=u'日记本描述',max_length=200)
-    content = models.CharField(verbose_name=u'日记本内容',max_length=1000)
+    drop_score = models.IntegerField(verbose_name=u'人工赋分', default=0)
+    description = models.CharField(verbose_name=u'日记本描述', max_length=200)
+    content = models.CharField(verbose_name=u'日记本内容', max_length=1000)
     share_num = models.IntegerField(verbose_name='')
     vote_num = models.IntegerField(verbose_name=u'点赞数')
     reply_num = models.IntegerField(verbose_name=u'回复数')
-    cover = models.CharField(verbose_name='',max_length=200)
+    cover = models.CharField(verbose_name='', max_length=200)
     is_online = models.BooleanField(verbose_name=u'是否上线')
     is_deleted = models.BooleanField(verbose_name=u'是否删除')
-    content_level = models.CharField(verbose_name=u'内容等级',max_length=3)
+    content_level = models.CharField(verbose_name=u'内容等级', max_length=3)
     language_type = models.IntegerField(verbose_name=u'语种类型')
-    create_time = models.DateTimeField(verbose_name=u'日记创建时间',default=datetime.datetime.fromtimestamp(0))
+    create_time = models.DateTimeField(verbose_name=u'日记创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
 
-
     def topic_has_image(self):
         try:
             has_image = False
 
-            query_list = TopicImage.objects.filter(topic_id=self.id,is_deleted=False,is_online=True)
-            if len(query_list)>0:
-                has_image=True
+            query_list = TopicImage.objects.filter(topic_id=self.id, is_deleted=False, is_online=True)
+            if len(query_list) > 0:
+                has_image = True
 
             return has_image
         except:
@@ -83,7 +84,7 @@ class Topic(models.Model):
     def get_pick_id_info(self):
         try:
             pick_id_list = list()
-            query_list = PickTopic.objects.filter(topic_id=self.id,is_deleted=False)
+            query_list = PickTopic.objects.filter(topic_id=self.id, is_deleted=False)
             for item in query_list:
                 pick_id_list.append(item.pick_id)
 
@@ -108,39 +109,22 @@ class Topic(models.Model):
     def get_tag_name_list(self, tag_id_list):
         try:
             tag_name_list = list()
-            logging.info("get tag_id_list :%s"%tag_id_list)
-            if len(tag_id_list) > 100:
-                lens = int(len(tag_id_list) / 100)
-                for i in range(lens+1):
-                    if i != 0:
-                        tag_id_100 = tag_id_list[i * 100:i * 100 + 100]
-                    if i == 0:
-                        tag_id_100 = tag_id_list[i:i + 100]
-                    if i * 100 == lens * 100 and len(tag_id_list[lens * 100:]) > 0:
-                        tag_id_100 = tag_id_list[lens * 100:]
-                    if tag_id_100:
-                        logging.info("get tag_id_100 :%s" % tag_id_100)
-
-                        query_results = Tag.objects.filter(id__in=tag_id_100).values_list("name", flat=True)
-                        for item in query_results:
-                            tag_name_list.append(item)
-            else:
-                query_results = Tag.objects.filter(id__in=tag_id_list).values_list("name", flat=True)
+            logging.info("get tag_id_list :%s" % tag_id_list)
+            for i in range(0, len(tag_name_list), 1000):
+                query_results = Tag.objects.filter(id__in=tag_id_list[i:i + 1000])
                 for item in query_results:
                     tag_name_list.append(item)
-
             return tag_name_list
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return []
 
-
     def get_topic_offline_score(self):
         try:
             offline_score = 0.0
             user_is_shadow = False
 
-            #是否官方推荐用户
+            # 是否官方推荐用户
             user_query_results = UserExtra.objects.filter(user_id=self.user_id)
             if user_query_results.count() > 0:
                 if user_query_results[0].is_recommend:
@@ -152,7 +136,7 @@ class Topic(models.Model):
             # if self.group and self.group.is_recommend:
             #     offline_score += 4.0
 
-            #帖子等级
+            # 帖子等级
             if self.content_level == '5':
                 offline_score += 6.0
             elif self.content_level == '4':
@@ -160,20 +144,20 @@ class Topic(models.Model):
             elif self.content_level == '3':
                 offline_score += 2.0
 
-            exposure_count = ActionSumAboutTopic.objects.filter(topic_id=self.id,data_type=1).count()
+            exposure_count = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=1).count()
             click_count = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=2).count()
-            uv_num = ActionSumAboutTopic.objects.filter(topic_id=self.id,data_type=3).count()
+            uv_num = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=3).count()
 
-            if exposure_count>0:
-                offline_score += click_count/exposure_count
-            if uv_num>0:
-                offline_score += (self.vote_num/uv_num + self.reply_num/uv_num)
+            if exposure_count > 0:
+                offline_score += click_count / exposure_count
+            if uv_num > 0:
+                offline_score += (self.vote_num / uv_num + self.reply_num / uv_num)
 
             """
                 1:马甲账号是否对总分降权？
             """
             if user_is_shadow:
-                offline_score = offline_score*0.5
+                offline_score = offline_score * 0.5
             return offline_score
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/models/user.py

@@ -62,27 +62,10 @@ class User(models.Model):
         for user_follow in user_follows:
             follow_user_id_list.append(user_follow.follow_id)
         follow_user_detail_list = list()
-        logging.info("follow_user_id_list:%s" % follow_user_id_list)
-        if len(follow_user_id_list) > 100:
-            lens = int(len(follow_user_id_list) / 100)
-            for i in range(lens + 1):
-                if i != 0:
-                    follow_user_id_list_100 = follow_user_id_list[i * 100:i * 100 + 100]
-                if i == 0:
-                    follow_user_id_list_100 = follow_user_id_list[i:i + 100]
-                if i * 100 == lens * 100 and len(follow_user_id_list[lens * 100:]) > 0:
-                    follow_user_id_list_100 = follow_user_id_list[lens * 100:]
-                if follow_user_id_list_100:
-                    logging.info("follow_user_id_list_100:%s" % follow_user_id_list_100)
-                    sql_data_list = User.objects.filter(user_id__in=follow_user_id_list_100)
-                    for detail_data in sql_data_list:
-                        item = {
-                            "user_id": detail_data.user_id,
-                            "country_id": detail_data.country_id
-                        }
-                        follow_user_detail_list.append(item)
-        else:
-            sql_data_list = User.objects.filter(user_id__in=follow_user_id_list)
+        logging.info("get follow_user_id_list :%s" % follow_user_id_list)
+        for i in range(0, len(follow_user_id_list), 1000):
+            logging.info("get follow_user_id_list :%s" % follow_user_id_list[i:i + 1000])
+            sql_data_list = User.objects.filter(user_id__in=follow_user_id_list[i:i + 1000])
             for detail_data in sql_data_list:
                 item = {
                     "user_id": detail_data.user_id,
@@ -115,47 +98,16 @@ class User(models.Model):
             pick_topic_id_list.append(user_pick.picktopic_id)
 
         pick_user_id_list = []
-
-        if len(pick_topic_id_list) > 100:
-            lens = int(len(pick_topic_id_list) / 100)
-            for i in range(lens + 1):
-                if i != 0:
-                    pick_topic_id_list_100 = pick_topic_id_list[i * 100:i * 100 + 100]
-                if i == 0:
-                    pick_topic_id_list_100 = pick_topic_id_list[i:i + 100]
-                if i * 100 == lens * 100 and len(pick_topic_id_list[lens * 100:]) > 0:
-                    pick_topic_id_list_100 = pick_topic_id_list[lens * 100:]
-                if pick_topic_id_list_100:
-                    topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list_100)
-                    for topic_data in topic_sql_list:
-                        pick_user_id_list.append(topic_data.user_id)
-        else:
-            topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list)
+        for i in range(0, len(pick_topic_id_list), 1000):
+            topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list[i:i + 1000])
             for topic_data in topic_sql_list:
                 pick_user_id_list.append(topic_data.user_id)
 
         pick_user_id_list = tuple(pick_user_id_list)
         pick_user_detail_list = list()
 
-        if len(pick_user_id_list) > 100:
-            lens = int(len(pick_user_id_list) / 100)
-            for i in range(lens + 1):
-                if i != 0:
-                    pick_user_id_list_100 = pick_user_id_list[i * 100:i * 100 + 100]
-                if i == 0:
-                    pick_user_id_list_100 = pick_user_id_list[i:i + 100]
-                if i * 100 == lens * 100 and len(pick_user_id_list[lens * 100:]) > 0:
-                    pick_user_id_list_100 = pick_user_id_list[lens * 100:]
-                if pick_topic_id_list_100:
-                    sql_data_list = User.objects.filter(user_id__in=pick_user_id_list_100)
-                    for detail_data in sql_data_list:
-                        item = {
-                            "user_id": detail_data.user_id,
-                            "country_id": detail_data.country_id
-                        }
-                        pick_user_detail_list.append(item)
-        else:
-            sql_data_list = User.objects.filter(user_id__in=pick_user_id_list)
+        for i in range(0, len(pick_user_id_list), 1000):
+            sql_data_list = User.objects.filter(user_id__in=pick_user_id_list[i:i + 1000])
             for detail_data in sql_data_list:
                 item = {
                     "user_id": detail_data.user_id,
@@ -175,25 +127,8 @@ class User(models.Model):
                 same_group_user_id_list.append(user_items_list.user_id)
 
         same_group_detail_list = list()
-        if len(same_group_user_id_list) > 100:
-            lens = int(len(same_group_user_id_list) / 100)
-            for i in range(lens + 1):
-                if i != 0:
-                    same_group_user_id_list_100 = same_group_user_id_list[i * 100:i * 100 + 100]
-                if i == 0:
-                    same_group_user_id_list_100 = same_group_user_id_list[i:i + 100]
-                if i * 100 == (lens - 1) * 100:
-                    same_group_user_id_list_100 = same_group_user_id_list[lens * 100:]
-                if same_group_user_id_list_100:
-                    sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list_100)
-                    for detail_data in sql_data_list:
-                        item = {
-                            "user_id": detail_data.user_id,
-                            "country_id": detail_data.country_id
-                        }
-                        same_group_detail_list.append(item)
-        else:
-            sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list)
+        for i in range(0, len(same_group_user_id_list), 1000):
+            sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list[i:i + 1000])
             for detail_data in sql_data_list:
                 item = {
                     "user_id": detail_data.user_id,

trans2es/utils/user_transfer.py

@@ -8,29 +8,31 @@ import time
 from libs.tools import tzlc
 from trans2es.models.user import User
 
+
 class UserTransfer(object):
 
     @classmethod
-    def get_follow_user_id_list(cls,userInstance):
+    def get_follow_user_id_list(cls, userInstance):
         follow_user_id_list = list()
         user_follows = userInstance.userfollow.filter(is_online=True)
         for user_follow in user_follows:
             follow_user_id_list.append(user_follow.follow_id)
 
         follow_user_detail_list = list()
-        sql_data_list = User.objects.filter(user_id__in=follow_user_id_list,is_online=True,is_deleted=False)
+        for i in range(0, len(follow_user_id_list), 1000):
+            sql_data_list = User.objects.filter(user_id__in=follow_user_id_list[i:i + 1000], is_online=True,
+                                                is_deleted=False)
             for detail_data in sql_data_list:
                 item = {
-                "user_id":detail_data.user_id,
-                "country_id":detail_data.country_id
+                    "user_id": detail_data.user_id,
+                    "country_id": detail_data.country_id
                 }
                 follow_user_detail_list.append(item)
 
         return follow_user_detail_list
 
-
     @classmethod
-    def get_user_data(cls,instance):
+    def get_user_data(cls, instance):
         try:
             res = dict()
 
@@ -82,7 +84,6 @@ class UserTransfer(object):
                 res["pick_user_id_list"] = []
                 res["same_group_user_id_list"] = []
 
-
             return res
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())