Commit 238020e2 authored by zhongshangwu's avatar zhongshangwu

fix

parent 9df8c75a
""" 微博帖子入库榜单脚本 """ """ 微博帖子入库榜单脚本 """
from co
import time import time
import os import os
import re import re
from datetime import datetime from datetime import datetime
from collections import defaultdict
import json import json
import requests import requests
from random import randint from random import randint
...@@ -15,7 +17,7 @@ from engine.logger import info_logger, error_logger, logging_exception ...@@ -15,7 +17,7 @@ from engine.logger import info_logger, error_logger, logging_exception
IMAGE_SUFFIX = '-w' IMAGE_SUFFIX = '-w'
FILE_PATH = '/Users/zhongshangwu/workspace/gengmei/like/saturn/weibo/' FILE_PATH = '/srv/apps/saturn/weibo/'
class Command(BaseCommand): class Command(BaseCommand):
...@@ -132,13 +134,13 @@ class Command(BaseCommand): ...@@ -132,13 +134,13 @@ class Command(BaseCommand):
return comment, replies return comment, replies
images = [] images = []
if not comment["images"] and len(reply) > 1: if not comment["images"] and reply:
normal_images = set(reply[0]["images"]) counter = defaultdict(int)
for info in reply[1:]: for info in reply:
info_images = set(info["images"]) for image in info["images"]:
normal_images = normal_images & info_images counter[image] += 1
comment["images"] = list(normal_images) comment["images"] = [url for url, count in counter if count > 1]
for info in reply: for info in reply:
if self.filter_second_comment(info["content"], comment["images"], info["images"]): if self.filter_second_comment(info["content"], comment["images"], info["images"]):
...@@ -228,7 +230,7 @@ class Command(BaseCommand): ...@@ -228,7 +230,7 @@ class Command(BaseCommand):
} }
else: # -> to pictorial comment else: # -> to pictorial comment
if len(self.stats[weibo_id]["first_comments"]) > 50: if len(self.stats[weibo_id]["first_comments"]) >= 50:
continue continue
top_comments_obj = rpc_invoker['venus/community/crawl/replys'](data=[comment], platform=platform, pictorial_id=pictorial_id).unwrap() top_comments_obj = rpc_invoker['venus/community/crawl/replys'](data=[comment], platform=platform, pictorial_id=pictorial_id).unwrap()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment