1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
# -*- coding: UTF-8 -*-
import json
import requests
import math
import time
import queue
import os
import ffmpy
import hashlib
from concurrent.futures import ThreadPoolExecutor
from django.core.management import BaseCommand
from django.conf import settings
from django.db.models import Q, Max
from gm_upload import upload_file
from gm_types.mimas.enum import IMAGE_TYPE
from gm_upload.utils.image_utils import Picture
from qa.models import QuestionImage, AnswerImage
from concurrent.futures import ThreadPoolExecutor
from qa.utils.image import img_type, handle_image_type
answer_img_queue = queue.Queue()
max_answer_img_info = AnswerImage.objects.order_by("id").last()
question_img_queue = queue.Queue()
max_question_img_info = QuestionImage.objects.order_by("id").last()
class UpdateQaImgType(object):
step = 200
def qa_image_update(self, qa_images):
"""处理图片内容类型"""
for qa_image in qa_images:
if qa_image.image_webp:
continue
hash_key = hashlib.md5(str(time.time()).encode("utf8")).hexdigest()
input_path = os.path.join(settings.GIF_CONVERT_PATH, "{}twp.gif".format(hash_key))
output_path = os.path.join(settings.GIF_CONVERT_PATH, '{}twp.webp'.format(hash_key))
img_url = Picture.get_no_watermark_path(qa_image.image_url)
res = requests.get(img_url)
# 文件存储到本地
with open(input_path, 'wb') as f:
f.write(res.content)
# 使用底层 FFmpeg 库进行转码
ff = ffmpy.FFmpeg(
inputs={input_path: None},
outputs={output_path: settings.GIF_FF_WEBP}
)
ff.run()
# 上传webP图片
qa_image.image_webp = upload_file(output_path)
qa_image.save()
if os.path.exists(input_path):
os.remove(input_path)
if os.path.exists(output_path):
os.remove(output_path)
return
def handle_question(self):
last_question_id = question_img_queue.get()
if max_question_img_info.id <= last_question_id:
question_img_queue.put(last_question_id)
return
print("当前执行的问答ID=======目标问答ID", last_question_id, max_question_img_info.id)
per_num = 200
q_imgs = QuestionImage.objects.filter(id__gt=last_question_id, image_type=IMAGE_TYPE.GIF)[:per_num]
max_id = q_imgs.aggregate(max_id=Max('id'))
question_img_queue.put(max_id.get('max_id'))
self.qa_image_update(qa_images=q_imgs)
def handle_answer(self):
last_answer_id = answer_img_queue.get()
if max_answer_img_info.id <= last_answer_id:
answer_img_queue.put(last_answer_id)
return
print("当前执行的问答ID=======目标问答ID", last_answer_id, max_answer_img_info.id)
per_num = 200
a_imgs = AnswerImage.objects.filter(id__gt=last_answer_id, image_type=IMAGE_TYPE.GIF)[:per_num]
max_id = a_imgs.aggregate(max_id=Max('id'))
answer_img_queue.put(max_id.get('max_id'))
self.qa_image_update(qa_images=a_imgs)
class Command(BaseCommand):
step = 200
def handle(self, *args, **kwargs):
update_action = UpdateQaImgType()
# q_count = QuestionImage.objects.filter(image_type=IMAGE_TYPE.GIF).order_by("id").count()
# q_range = math.ceil(float(q_count) / 200)
# question_img_queue.put(0)
# for i in range(q_range):
# update_action.handle_question()
# with ThreadPoolExecutor(max_workers=4) as ex:
# for i in range(q_range):
# ex.submit(update_action.handle_question)
a_count = AnswerImage.objects.filter(id__gte=54207, image_type=IMAGE_TYPE.GIF).order_by("id").count()
a_range = math.ceil(float(a_count) / 200)
answer_img_queue.put(0)
# for i in range(a_range):
# update_action.handle_answer()
with ThreadPoolExecutor(max_workers=4) as ex:
for i in range(a_range):
ex.submit(update_action.handle_answer)