update

eda/smart_rank/gyz_test.py

@@ -96,7 +96,7 @@ spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCre
 spark.sparkContext.setLogLevel("WARN")
 spark.sparkContext.addPyFile("/srv/apps/ffm-baseline_git/eda/smart_rank/tool.py")
 device_ids_lst_rdd = spark.sparkContext.parallelize(device_info)
-result = device_ids_lst_rdd.repartition(100).map(lambda x: get_user_service_portrait(x, all_word_tags, all_tag_tag_type, all_3tag_2tag, all_tags_name, size=None, pay_time=pay_time))
+result = device_ids_lst_rdd.repartition(100).map(lambda x: get_user_service_portrait(x, all_word_tags, all_tag_tag_type, all_3tag_2tag, all_tags_name, size=None, pay_time=pay_time)).dropna()
 print(result.take(10))
 df = spark.createDataFrame(result).na.drop().toDF("device", "search_words", "user_portrait").toPandas()
 df.to_csv("~/test_df.csv", index=False)