update

crawler_sys/site_crawler/crawler_toutiao.py

@@ -325,6 +325,10 @@ class Crawler_toutiao():
         url = "https://m.toutiao.com/i{0}/info/?i={1}".format(article_id, article_id)
         requests_res = retry_get_url(url, headers=headers, proxies=proxies_num)
         res_json = requests_res.json()
+        try:
+            content = res_json["data"].get("content").replace("\r", "").replace("\n", "")
+        except:
+            content = ""
         res_dic = {
             "title": res_json["data"].get("title").replace("\r", "").replace("\n", ""),
             'high_quality_flag': int(res_json["data"].get('high_quality_flag')),
@@ -334,7 +338,7 @@ class Crawler_toutiao():
             "favorite_count": res_json["data"].get("digg_count"),
             'releaser_followers_count': res_json["data"].get("follower_count"),
             'release_time': int(res_json["data"].get('publish_time') * 1e3),
-            "content": res_json["data"].get("content").replace("\r", "").replace("\n", ""),
+            "content": content,
             "img_list": re.findall('img src=".*?"', res_json["data"].get("content"))
         }
         return res_dic
@@ -438,8 +442,6 @@ class Crawler_toutiao():
                         # The search api just return something seems related to search
                         print(e)
                         continue
-            else:
-                break
                     if len(toutiao_Lst) >= 100:
                         output_result(result_Lst=toutiao_Lst,
                                       platform=self.platform,