模型调试

spark/featureEng.py

@@ -830,11 +830,9 @@ if __name__ == '__main__':
     spark = get_spark("service_feature_csv_export")
     spark.sparkContext.setLogLevel("ERROR")
 
-    itemDF = get_service_feature_df()
-    print(itemDF.columns)
-    print(itemDF.head(100))
-
-    sys.exit()
+    # itemDF = get_service_feature_df()
+    # print(itemDF.columns)
+    # print(itemDF.head(100))
 
     # 行为数据
     clickSql = getClickSql(startDay,endDay)
@@ -853,9 +851,10 @@ if __name__ == '__main__':
         .withColumn("timestamp",F.col("timestamp").cast("long"))
 
     print(ratingDF.columns)
-    print(ratingDF.show(10, truncate=False))
+    print(ratingDF.show(100, truncate=False))
 
     print("添加label...")
+    #TODO 负样本为排除点击的数据
     ratingSamplesWithLabel = addSampleLabel(ratingDF)
     df = ratingSamplesWithLabel.toPandas()
     df = pd.DataFrame(df)
@@ -885,13 +884,15 @@ if __name__ == '__main__':
 
     print("dataVocab:")
     for k, v in dataVocab.items():
-        print(k, len(v))
+        print(k, len(v), v)
 
 
     itemDF_spark = spark.createDataFrame(itemDF)
     itemDF_spark.printSchema()
     itemDF_spark.show(10, truncate=False)
 
+    sys.exit(1)
+
     # item统计特征处理
     itemStaticDF = addItemStaticFeatures(ratingSamplesWithLabel,itemDF_spark,dataVocab)