#!/usr/bin/env python # -*- coding: utf-8 -*- import datetime from django.conf import settings from django.core.management import BaseCommand from gm_types.mimas import GRABBING_PLATFORM from qa.models.answer import ( Answer, AnswerTag ) from utils.common import big_data_iter # prod kyc_v1_tag_ids = [ 11009, 11010, 11011, 11012, 11013, 11014, 11015, 11016, 11017, 11018, 11019, 11020, 11071, 11072, 11073, 11074, 11079, 11416, 11417, 11985, 11986, 11987, 12300, 12301, 12303, 12304, 12306, 12307, 12308, 12309, 12310, 12311, 12712 ] # test # kyc_v1_tag_ids = [10085] start_date_time = datetime.datetime.strptime("2019-12-20 00:00:00", "%Y-%m-%d %H:%M:%S") end_date_time = datetime.datetime.strptime("2020-04-15 00:00:00", "%Y-%m-%d %H:%M:%S") class Command(BaseCommand): """ python django_manage.py kyc_create_answer_clean """ def handle(self, *args, **options): print('------ starting -----') start_time = datetime.datetime.now() print("start at: ", start_time.strftime("%Y-%m-%d %H:%M:%S.%f")) # TODO 工作区 answer_infos = Answer.objects.using(settings.SLAVE_DB_NAME).filter( create_time__gte=start_date_time, create_time__lte=end_date_time, platform=GRABBING_PLATFORM.GM ).values_list("id", flat=True) print("\nwill modify answer nums {} \n".format(answer_infos.count())) quantity_processed = 0 for items in big_data_iter(answer_infos, fetch_num=200): valid_answer_ids = set(AnswerTag.objects.using(settings.SLAVE_DB_NAME).filter( answer_id__in=items, tag__in=kyc_v1_tag_ids ).values_list("answer_id", flat=True)) if valid_answer_ids: Answer.objects.filter(pk__in=valid_answer_ids).update( platform=GRABBING_PLATFORM.KYC, update_time=datetime.datetime.now() ) quantity_processed += len(valid_answer_ids) with open("/tmp/kyc_create_answer_clean_id.log", "a+") as f: f.write("{}\n".format(','.join(map(str, valid_answer_ids)))) print("quantity processed nums {} \n".format(quantity_processed)) end_time = datetime.datetime.now() print("end at: ", end_time.strftime("%Y-%m-%d %H:%M:%S.%f")) print('\ntotal use {} s.'.format((end_time - start_time).total_seconds())) print('Done! \n')