test

crawler_sys/utils/output_results.py

@@ -41,7 +41,7 @@ def form_data_Lst_for_url_register(data_Lst_ori):
     return data_Lst_reg
 
 
-def hot_words_output_result(result_Lst,output_index="short-video-hotwords",output_doc="doc"):
+def hot_words_output_result(result_Lst,output_index="short-video-hotwords"):
     bulk_all_body = ""
     for count,result in enumerate(result_Lst):
         doc_id = result["platform"] + "_"+ result["title"]
@@ -51,7 +51,7 @@ def hot_words_output_result(result_Lst,output_index="short-video-hotwords",outpu
         bulk_all_body += bulk_one_body
         if count % 500 == 0 and count >0:
 
-            eror_dic = es_site_crawler.bulk(index=output_index, doc_type=output_doc,
+            eror_dic = es_site_crawler.bulk(index=output_index,
                                body=bulk_all_body, request_timeout=200)
             bulk_all_body = ''
             if eror_dic['errors'] is True:
@@ -63,7 +63,7 @@ def hot_words_output_result(result_Lst,output_index="short-video-hotwords",outpu
     if bulk_all_body != '':
         eror_dic = es_site_crawler.bulk(body=bulk_all_body,
                            index=output_index,
-                           doc_type=output_doc,
+
                            request_timeout=200)
         if eror_dic['errors'] is True:
             print(eror_dic)
@@ -151,11 +151,12 @@ def bulk_write_into_es(dict_Lst,
     bulk_write_body = ''
     write_counter = 0
 
-    def bulk_write_with_retry_UnicodeEncodeError(bulk_write_body,
+    def bulk_write_with_retry_UnicodeEncodeError(index,bulk_write_body,
                                                  retry_counter_for_UnicodeEncodeError):
         if bulk_write_body != '':
             try:
-                bulk_write_resp = es_site_crawler.bulk(body=bulk_write_body,
+                bulk_write_resp = es_site_crawler.bulk(index=index,
+                                                       body=bulk_write_body,
                                                        request_timeout=100)
                 bulk_write_body = ''
 #                print(bulk_write_resp)
@@ -168,7 +169,7 @@ def bulk_write_into_es(dict_Lst,
                 if len(ill_str_idxs) == 2:
                     ill_str = bulk_write_body[ill_str_idxs[0]: ill_str_idxs[1]+1]
                     bulk_write_body = bulk_write_body.replace(ill_str, '')
-                    bulk_write_with_retry_UnicodeEncodeError(bulk_write_body,
+                    bulk_write_with_retry_UnicodeEncodeError(index,bulk_write_body,
                                                              retry_counter_for_UnicodeEncodeError
                                                             )
             except TransportError:
@@ -196,7 +197,7 @@ def bulk_write_into_es(dict_Lst,
                                                    len(dict_Lst)))
             if bulk_write_body != '':
                 retry_counter_for_UnicodeEncodeError = 0
-                retry_counter_for_UnicodeEncodeError = bulk_write_with_retry_UnicodeEncodeError(
+                retry_counter_for_UnicodeEncodeError = bulk_write_with_retry_UnicodeEncodeError(index,
                     bulk_write_body,
                     retry_counter_for_UnicodeEncodeError)
                 bulk_write_body = ''