Commit ee5855c2 authored by zhongshangwu's avatar zhongshangwu

修复小红书数据入库阻塞问题

parent dd262629
...@@ -49,7 +49,6 @@ class Command(BaseCommand): ...@@ -49,7 +49,6 @@ class Command(BaseCommand):
start_user_id=self.user_id_start, start_user_id=self.user_id_start,
end_user_id=self.user_id_start + 5000 end_user_id=self.user_id_start + 5000
).unwrap() ).unwrap()
print(data, ">>>>>")
self.shadow_user_ids = data self.shadow_user_ids = data
def get_random_user_id(self): def get_random_user_id(self):
...@@ -68,19 +67,19 @@ class Command(BaseCommand): ...@@ -68,19 +67,19 @@ class Command(BaseCommand):
def get_user_id(self, id_, platform): def get_user_id(self, id_, platform):
# 获取用户ID 缓存记录保留用户关系 # 获取用户ID 缓存记录保留用户关系
cache_key = 'grap:{}:{}'.format(platform, id_) cache_key = 'grap:{}:{}'.format(platform, id_)
exist_key = 'grap:{}:{}' # exist_key = 'grap:{}:{}'
value = ins_cache.get(cache_key) value = ins_cache.get(cache_key)
user_id = None # user_id = None
if not value: if not value:
while True: # while True:
user_id = self.get_random_user_id() user_id = self.get_random_user_id()
exist = exist_key.format(platform, user_id) # exist = exist_key.format(platform, user_id)
if not ins_cache.get(exist): # if not ins_cache.get(exist):
ins_cache.set(exist, id_) # ins_cache.set(exist, id_)
self.del_cache_keys.append(exist) # self.del_cache_keys.append(exist)
break # break
ins_cache.set(cache_key, user_id) # ins_cache.set(cache_key, user_id)
self.del_cache_keys.append(exist) # self.del_cache_keys.append(exist)
else: else:
user_id = int(value) user_id = int(value)
return user_id return user_id
...@@ -117,7 +116,7 @@ class Command(BaseCommand): ...@@ -117,7 +116,7 @@ class Command(BaseCommand):
# 获取图片宽高 # 获取图片宽高
try: try:
url = image_url + IMAGE_SUFFIX + '?imageInfo' url = image_url + IMAGE_SUFFIX + '?imageInfo'
response = requests.request("GET", url) response = requests.request("GET", url, timeout=3)
info = response.json() info = response.json()
return info.get('width'), info.get('height') return info.get('width'), info.get('height')
except Exception as e: except Exception as e:
...@@ -163,7 +162,7 @@ class Command(BaseCommand): ...@@ -163,7 +162,7 @@ class Command(BaseCommand):
def create_topic(self, topics, platform): def create_topic(self, topics, platform):
count = 0 count = 0
for topic in topics[:100]: for topic in topics:
count += 1 count += 1
topic_comments = topic.pop('comments', None) topic_comments = topic.pop('comments', None)
images = topic.pop('image') images = topic.pop('image')
...@@ -279,7 +278,7 @@ class Command(BaseCommand): ...@@ -279,7 +278,7 @@ class Command(BaseCommand):
print('----- start deal pictorial at {} -----'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S %f'))) print('----- start deal pictorial at {} -----'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S %f')))
pictorial_data = self.get_json_data_from_dir(is_pictorial=1) pictorial_data = self.get_json_data_from_dir(is_pictorial=1)
count = 0 count = 0
for pictorial in pictorial_data[:50]: for pictorial in pictorial_data:
count += 1 count += 1
print('------- current pictorial count :', count) print('------- current pictorial count :', count)
self.create_pictorial(pictorial=pictorial, platform=platform) self.create_pictorial(pictorial=pictorial, platform=platform)
......
...@@ -9,7 +9,7 @@ from PIL import Image ...@@ -9,7 +9,7 @@ from PIL import Image
def upload_image(url, img_type=IMG_TYPE.TOPIC): def upload_image(url, img_type=IMG_TYPE.TOPIC):
'''非站内图片处理''' '''非站内图片处理'''
try: try:
response = requests.get(url) response = requests.get(url, timeout=3)
return upload(response.content, img_type=img_type) return upload(response.content, img_type=img_type)
except: except:
return None return None
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment