update

crawler_sys/site_crawler_by_redis/crawler_douban.py

@@ -144,8 +144,7 @@ class CrawlerDouban():
                 else:
                     get_page = retry_get_url(url, headers=self.headers, timeout=self.timeout)
             except:
-                get_page = None
-                has_more = False
+                continue
             if get_page and get_page.status_code == 200:
                 try:
                     page_json = get_page.json()
@@ -157,8 +156,10 @@ class CrawlerDouban():
                 except Exception as e:
                     print("load data error %s" % e)
                     continue
+
                 if page_dic:
                     for one in page_dic:
+                        try:
                             releaser_id = one["author"]["id"]
                             mid = one["id"]
                             if True:
@@ -186,6 +187,8 @@ class CrawlerDouban():
                                 # print(res_dic)
     
                                 yield res_dic
+                        except Exception as e:
+                            print("single data parse error %s " %e)
                         # except Exception as e:
                         #     print(one)
                         #     print("row formate error %s" % e)