模型调试

spark/featureEngSk.py

@@ -1020,16 +1020,21 @@ if __name__ == '__main__':
 
 
     write_time_start = time.time()
+    vocab_redis_keys = []
     for categoty_field in categoty_fields:
         output_file = "file:///home/gmuser/" + categoty_field + "_vocab"
         # train_samples.select(categoty_field).where(F.col(categoty_field) != '-1').where(F.col(categoty_field) != '').distinct().write.mode("overwrite").options(header="false").csv(output_file)
         categoty_field_rows = train_samples.select(categoty_field).where(F.col(categoty_field) != '-1').where(F.col(categoty_field) != '').distinct().collect()
-        saveVocab("strategy:" + categoty_field + ":vocab", list(map(lambda row: row[categoty_field], categoty_field_rows)))
+        vocab_redis_keys.append("strategy:" + categoty_field + ":vocab")
+        saveVocab(vocab_redis_keys[-1], list(map(lambda row: row[categoty_field], categoty_field_rows)))
     for multi_categoty_field in multi_categoty_fields:
         output_file = "file:///home/gmuser/" + multi_categoty_field + "_vocab"
         # train_samples.selectExpr("explode(split({multi_categoty_field},','))".format(multi_categoty_field = multi_categoty_field)).where(F.col(multi_categoty_field) != '-1').distinct().write.mode("overwrite").options(header="false").csv(output_file)
         multi_categoty_field_rows = train_samples.selectExpr("explode(split({multi_categoty_field},',')) as {multi_categoty_field}".format(multi_categoty_field = multi_categoty_field)).where(F.col(multi_categoty_field) != '-1').distinct().collect()
-        saveVocab("strategy:" + multi_categoty_field + ":vocab", list(map(lambda row: row[multi_categoty_field], multi_categoty_field_rows)))
+        vocab_redis_keys.append("strategy:" + multi_categoty_field + ":vocab")
+        saveVocab(vocab_redis_keys[-1], list(map(lambda row: row[multi_categoty_field], multi_categoty_field_rows)))
+
+    saveVocab("strategy:all:vocab", vocab_redis_keys)
 
     output_file = "file:///home/gmuser/train_samples"
     train_samples.write.mode("overwrite").options(header="false", sep='|').csv(output_file)

train/down_vocab.py

+import redis