#!/usr/bin/env python # -*- coding: utf-8 -*- # from gevent import monkey; monkey.patch_socket() import threading import time from concurrent.futures import ThreadPoolExecutor from django.core.management import BaseCommand from gm_types.gaia import VIDEO_CODE_STATUS from talos.models.topic.video import Video from talos.models.tractate import TractateVideo from talos.models.topic.video import VideoCover from live.tasks import ( set_clipping_to_video, check_video_clipping_is_finish, set_video_webp_pic ) from talos.tasks.tractate import ( set_tractate_video_clipping_to_video, check_tractate_video_clipping_is_finish, set_tractate_video_webp_pic ) from qa.tasks.get_video_blurcover import ( video_cover_for_rich_text_set_clipping_to_video, check_video_cover_for_rich_text_clipping_is_finish, set_video_cover_for_rich_text_webp_pic ) class VideoWebP(object): running = False @staticmethod def start_clipping(): clipping_id_list = list(Video.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.NOSTART ).values_list("id", flat=True)) for video_id in clipping_id_list: try: set_clipping_to_video(video_id) except: Video.objects.filter(id=video_id).update(persistent_clip_status=VIDEO_CODE_STATUS.FAIL) @classmethod def clip_status_check(cls): cnt = 0 while True: video_ids = list(Video.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.WAITING ).values_list("id", flat=True)[:1000]) if len(video_ids) == 0 or cnt > 100: cls.running = False break for video_id in video_ids: check_video_clipping_is_finish(video_id) cnt += 1 # gevent.sleep(10) @classmethod def set_web_p(cls): while True: video_ids = list(Video.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.OPERATING_LOCAL ).values_list("id", flat=True)[:1000]) if len(video_ids) == 0 and not cls.running: break for video_id in video_ids: set_video_webp_pic(video_id) @classmethod def run(cls): print("video start-----{}".format(time.time())) cls.running = True cls.start_clipping() workers = [] workers.append(threading.Thread(target=cls.clip_status_check)) for i in range(3): workers.append(threading.Thread(target=cls.set_web_p)) for worker in workers: worker.start() for worker in workers: worker.join() print("video end-----{}".format(time.time())) @classmethod def run_new(cls): print("video start-----{}".format(time.time())) cls.start_clipping() with ThreadPoolExecutor(max_workers=4) as ex: while True: video_waiting_ids = list(Video.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.WAITING ).values_list("id", flat=True)[:100]) video_local_ids = list(Video.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.OPERATING_LOCAL ).values_list("id", flat=True)[:100]) if not video_waiting_ids and not video_local_ids: break for video_id in video_waiting_ids: ex.submit(check_video_clipping_is_finish, video_id) for video_id in video_local_ids: ex.submit(set_video_webp_pic, video_id) time.sleep((len(video_waiting_ids) + len(video_local_ids)) / 2) print("video end-----{}".format(time.time())) class TractateVideoWebP(object): running = False @staticmethod def start_clipping(): clipping_id_list = list(TractateVideo.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.NOSTART ).values_list("id", flat=True)) for video_id in clipping_id_list: try: set_tractate_video_clipping_to_video(video_id) except: TractateVideo.objects.filter(id=video_id).update(persistent_clip_status=VIDEO_CODE_STATUS.FAIL) @classmethod def clip_status_check(cls): cnt = 0 while True: video_ids = list(TractateVideo.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.WAITING ).values_list("id", flat=True)[:1000]) if len(video_ids) == 0 or cnt > 100: cls.running = False break for video_id in video_ids: check_tractate_video_clipping_is_finish(video_id) cnt += 1 # gevent.sleep(10) @classmethod def set_web_p(cls): while True: video_ids = list(TractateVideo.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.OPERATING_LOCAL ).values_list("id", flat=True)[:1000]) if len(video_ids) == 0 and not cls.running: break for video_id in video_ids: set_tractate_video_webp_pic(video_id) @classmethod def run(cls): print("tractate_video start-----{}".format(time.time())) cls.running = True cls.start_clipping() workers = [] workers.append(threading.Thread(target=cls.clip_status_check)) for i in range(3): workers.append(threading.Thread(target=cls.set_web_p)) for worker in workers: worker.start() for worker in workers: worker.join() print("tractate_video start-----{}".format(time.time())) @classmethod def run_new(cls): print("tractate_video start-----{}".format(time.time())) cls.start_clipping() with ThreadPoolExecutor(max_workers=4) as ex: while True: video_waiting_ids = list(TractateVideo.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.WAITING ).values_list("id", flat=True)[:100]) video_local_ids = list(TractateVideo.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.OPERATING_LOCAL ).values_list("id", flat=True)[:100]) if not video_waiting_ids and not video_local_ids: break for video_id in video_waiting_ids: ex.submit(check_tractate_video_clipping_is_finish, video_id) for video_id in video_local_ids: ex.submit(set_tractate_video_webp_pic, video_id) time.sleep((len(video_waiting_ids) + len(video_local_ids)) / 2) print("tractate_video end-----{}".format(time.time())) class VideoCoverWebP(object): running = False @staticmethod def start_clipping(): clipping_id_list = list(VideoCover.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.NOSTART ).values_list("id", flat=True)) for video_id in clipping_id_list: try: video_cover_for_rich_text_set_clipping_to_video(video_id) except: VideoCover.objects.filter(id=video_id).update(persistent_clip_status=VIDEO_CODE_STATUS.FAIL) @classmethod def clip_status_check(cls): cnt = 0 while True: video_ids = list(VideoCover.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.WAITING ).values_list("id", flat=True)[:1000]) if len(video_ids) == 0 or cnt > 2000: cls.running = False break for video_id in video_ids: check_video_cover_for_rich_text_clipping_is_finish(video_id) cnt += 1 # gevent.sleep(10) @classmethod def set_web_p(cls): while True: video_ids = list(VideoCover.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.OPERATING_LOCAL ).values_list("id", flat=True)[:1000]) if len(video_ids) == 0 and not cls.running: break for video_id in video_ids: set_video_cover_for_rich_text_webp_pic(video_id) @classmethod def run(cls): print("video_cover start-----{}".format(time.time())) cls.running = True cls.start_clipping() workers = [] workers.append(threading.Thread(target=cls.clip_status_check)) for i in range(3): workers.append(threading.Thread(target=cls.set_web_p)) for worker in workers: worker.start() for worker in workers: worker.join() print("video_cover start-----{}".format(time.time())) @classmethod def run_new(cls): print("video_cover start-----{}".format(time.time())) cls.start_clipping() with ThreadPoolExecutor(max_workers=4) as ex: while True: video_waiting_ids = list(VideoCover.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.WAITING ).values_list("id", flat=True)[:100]) video_local_ids = list(VideoCover.objects.filter( persistent_clip_status=VIDEO_CODE_STATUS.OPERATING_LOCAL ).values_list("id", flat=True)[:100]) if not video_waiting_ids and not video_local_ids: break for video_id in video_waiting_ids: ex.submit(check_video_cover_for_rich_text_clipping_is_finish, video_id) for video_id in video_local_ids: ex.submit(set_video_cover_for_rich_text_webp_pic, video_id) time.sleep((len(video_waiting_ids) + len(video_local_ids)) / 2) print("video_cover end-----{}".format(time.time())) class Command(BaseCommand): """ 为所有已有视频添加webP动图 """ def handle(self, *args, **options): print('------ starting -----') start_time = time.time() print("start at: ", start_time) VideoWebP.run_new() TractateVideoWebP.run_new() VideoCoverWebP.run_new() end_time = time.time() print("end at: ", end_time) print('total use {} s.'.format(end_time - start_time)) print('Done!')