修改预测集sql，并加上if判断为空

eda/esmm/Model_pipline/feature_engineering.py

@@ -302,7 +302,8 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
           "left join jerry_test.sixin_tag sixin on e.device_id = sixin.device_id " \
           "left join jerry_test.cart_tag cart on e.device_id = cart.device_id " \
           "left join jerry_test.knowledge k on feat.level2 = k.level2_id " \
-          "left join jerry_test.search_doris doris on e.device_id = doris.device_id and e.stat_date = doris.get_date"
+          "left join jerry_test.search_doris doris on e.device_id = doris.device_id and e.stat_date = doris.get_date " \
+          "limit 100000"
 
     features = ["ucity_id", "ccity_name", "device_type", "manufacturer",
                 "channel", "top", "time", "hospital_id",
@@ -336,29 +337,38 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
                                               value_map.get(x[29], 15)],
                                              app_list_func(x[30], leve2_map),app_list_func(x[31], leve3_map)))
 
-
-
     rdd.persist(storageLevel= StorageLevel.MEMORY_ONLY_SER)
     print("预测集样本大小：")
     print(rdd.count())
 
-    spark.createDataFrame(rdd.filter(lambda x: x[0] == 0)
+    native = spark.createDataFrame(rdd.filter(lambda x: x[0] == 0)
                           .map(lambda x: (x[1], x[2], x[6], x[7], x[8], x[9], x[10], x[11],
                                           x[12], x[13], x[14], x[15], x[16], x[17], x[18],x[3],x[4],x[5]))) \
         .toDF("y", "z", "app_list", "level2_list", "level3_list", "tag1_list", "tag2_list", "tag3_list", "tag4_list",
-              "tag5_list", "tag6_list", "tag7_list", "ids", "search_tag2_list","search_tag3_list","city","uid","cid_id") \
-        .repartition(1).write.format("tfrecords").save(path=path+"native/", mode="overwrite")
-    print("native tfrecord done")
-    h = time.time()
-    print((h-f)/60)
-
-    spark.createDataFrame(rdd.filter(lambda x: x[0] == 1)
+              "tag5_list", "tag6_list", "tag7_list", "ids", "search_tag2_list",
+              "search_tag3_list","city","uid","cid_id")
+    if native.take(1).nonEmpty:
+        print("预测集native有数据")
+        native.repartition(1).write.format("tfrecords").save(path=path + "native/", mode="overwrite")
+        print("native tfrecord done")
+        h = time.time()
+        print((h - f) / 60)
+    else:
+        print("预测集native为空")
+
+    nearby = spark.createDataFrame(rdd.filter(lambda x: x[0] == 1)
                           .map(lambda x: (x[1], x[2], x[6], x[7], x[8], x[9], x[10], x[11],
                                           x[12], x[13], x[14], x[15], x[16], x[17], x[18],x[3],x[4],x[5]))) \
         .toDF("y", "z", "app_list", "level2_list", "level3_list", "tag1_list", "tag2_list", "tag3_list", "tag4_list",
-              "tag5_list", "tag6_list", "tag7_list", "ids", "search_tag2_list","search_tag3_list","city","uid","cid_id") \
-        .repartition(1).write.format("tfrecords").save(path=path + "nearby/", mode="overwrite")
-    print("nearby tfrecord done")
+              "tag5_list", "tag6_list", "tag7_list", "ids", "search_tag2_list",
+              "search_tag3_list","city","uid","cid_id")
+
+    if nearby.take(1).nonEmpty:
+        print("预测集nearby有数据")
+        nearby.repartition(1).write.format("tfrecords").save(path=path + "nearby/", mode="overwrite")
+        print("nearby tfrecord done")
+    else:
+        print("预测集nearby为空")
 
 
 if __name__ == '__main__':