update

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -54,7 +54,7 @@ parser.add_argument('-f', '--output_file_path', default='', type=str,
                     help=('Specify output file path, default None.'))
 parser.add_argument('-r', '--push_to_redis', default="True", type=str,
                     help=('Write urls to redis or not, default to True'))
-parser.add_argument('-w', '--output_to_es_raw', default='True', type=str,
+parser.add_argument('-w', '--output_to_es_raw', default='False', type=str,
                     help=('Write data into es or not, default to True'))
 parser.add_argument('-index', '--es_index', default='crawler-data-raw', type=str,
                     help=('assign a es_index to write into, default to crawler-data-raw'))
@@ -204,6 +204,7 @@ def single_thead(processe,name):
                                           platform=platform,
                                           output_to_file=output_to_file,
                                           filepath=None,
+                                          push_to_redis=push_to_redis,
                                           output_to_es_raw=output_to_es_raw,
                                           es_index=es_index,
                                           output_to_es_register=output_to_es_register)
@@ -219,6 +220,7 @@ def single_thead(processe,name):
                         output_result(result_Lst=data_list,
                                       platform=platform,
                                       output_to_file=output_to_file,
+                                      push_to_redis=push_to_redis,
                                       filepath=None,
                                       output_to_es_raw=output_to_es_raw,
                                       es_index=es_index,

crawler_sys/site_crawler_by_redis/crawler_weibo.py

@@ -174,6 +174,7 @@ class Crawler_weibo():
                                     "releaserUrl": "https://www.weibo.com/u/%s" % releaser_id,
                                     "releaser_id_str": "weibo_%s" % releaser_id,
                                     "img_list":self.get_img(mblog),
+                                    "platform":"weibo",
                                     # "doc_id":doc_id
                                 }
                                 res_dic["doc_id"] = cal_doc_id(platform="weibo", url=one["scheme"], data_dict=res_dic,