update viladate

word_vector/tractate.py

@@ -180,10 +180,17 @@ def validate_tractate_item2vec_by_id(tractate_id, related_tractate_ids):
 
 def validate_tractate_item2vec(tractate_ids, n=30):
     tractate_ids = random.sample(list(tractate_ids), n)
+    count = 0
+    total = 0
     for tractate_id in  tractate_ids:
         related_tractate_ids = get_tractate_item2vec_by_id(tractate_id, TRACTATE_CLICK_IDS_MODEL)
         score = validate_tractate_item2vec_by_id(tractate_id, related_tractate_ids)
+        if score != 0:
+            total += score
+            count += 1
         print("{}: {}".format(tractate_id, score))
+    if total != 0:
+        print("res: {}/{} avg: {}".format(count, n, total / count))
 
 
 if __name__ == "__main__":
@@ -214,6 +221,6 @@ if __name__ == "__main__":
 
     print("total cost: {:.2f}mins".format((time.time() - begin_time) / 60))
 
-    # validate_tractate_item2vec(TRACTATE_CLICK_IDS, 10)
+    # validate_tractate_item2vec(TRACTATE_CLICK_IDS, 100)
 
 # spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/strategy_embedding/word_vector/tractate.py