change dev and prod config and concat diary format

eda/node2vec/src/main/resources/application.properties

-dev.tidb.jdbcuri=jdbc:mysql://10.66.157.22:4000/jerry_test?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
-dev.tispark.pd.addresses=10.66.157.22:2379
+dev.tidb.jdbcuri=jdbc:mysql://192.168.15.12:4000/jerry_test?user=root&password=&rewriteBatchedStatements=true
+dev.tispark.pd.addresses=192.168.15.11:2379
 dev.mimas.jdbcuri= jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com/mimas_test?user=work&password=workwork&rewriteBatchedStatements=true
 dev.gaia.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com/zhengxing_test?user=work&password=workwork&rewriteBatchedStatements=true
 dev.gold.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com/doris_test?user=work&password=workwork&rewriteBatchedStatements=true
-dev.redis.host=10.30.50.58
-dev.redis.port=6379
+dev.tidb.database=jerry_test
 
 
 pre.tidb.jdbcuri=jdbc:mysql://192.168.16.11:4000/eagle?user=root&password=&rewriteBatchedStatements=true
 pre.tispark.pd.addresses=192.168.16.11:2379
 pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas_prod?user=mimas&password=workwork&rewriteBatchedStatements=true
 
+prod.tidb.database=jerry_prod
 prod.tidb.jdbcuri=jdbc:mysql://10.66.157.22:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
 prod.gold.jdbcuri=jdbc:mysql://rm-m5e842126ng59jrv6.mysql.rds.aliyuncs.com/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
 prod.mimas.jdbcuri=jdbc:mysql://rm-m5emg41za2w7l6au3.mysql.rds.aliyuncs.com/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true

eda/node2vec/src/main/scala/com/gmei/GmeiConfig.scala

@@ -53,7 +53,7 @@ object GmeiConfig extends Serializable {
     }
 
     if (!sparkConf.contains("spark.tispark.pd.addresses")) {
-      sparkConf.set("spark.tispark.pd.addresses", config.getString("tispark.pd.addresses"))
+      sparkConf.set("spark.tispark.pd.addresses", this.config.getString("tispark.pd.addresses"))
     }
     println(sparkConf.get("spark.tispark.pd.addresses"))
 
@@ -64,10 +64,6 @@ object GmeiConfig extends Serializable {
       .getOrCreate()
 
     val context = SparkContext.getOrCreate(sparkConf)
-
-    val ti = new TiContext(spark)
-    // mapping all tables cache it.
-    ti.tidbMapDatabase("jerry_test")
     (context, spark)
   }
 
@@ -86,7 +82,7 @@ object GmeiConfig extends Serializable {
 
 
   def writeToJDBCTable(df: DataFrame, table: String, saveMode: SaveMode): Unit = {
-    val jdbcuri = config.getString("tidb.jdbcuri")
+    val jdbcuri = this.config.getString("tidb.jdbcuri")
     println(jdbcuri, table)
     writeToJDBCTable(jdbcuri, df, table, saveMode)
   }

eda/node2vec/src/main/scala/com/gmei/Main.scala

@@ -4,8 +4,6 @@ import java.io.Serializable
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.mllib.linalg.Vectors
 import org.apache.spark.mllib.linalg.distributed.{IndexedRow, IndexedRowMatrix}
-import org.apache.spark.sql.functions.row_number
-import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.{SaveMode,TiContext}
 import org.apache.log4j.{Level, Logger}
 import scopt.OptionParser
@@ -100,7 +98,7 @@ object Main {
       val sc = GmeiConfig.getSparkSession()._2
 
       val ti = new TiContext(sc)
-      ti.tidbMapTable(dbName = "jerry_test",tableName = "data_meigou_cid")
+      ti.tidbMapTable(dbName = GmeiConfig.config.getString("tidb.database"),tableName = "data_meigou_cid")
 
       val tidb_inupt = sc.sql(
         s"""
@@ -174,8 +172,7 @@ object Main {
 
       remapSecond.take(20).foreach(println)
 
-      val w1 = Window.orderBy($"score")
-      val score_result = remapSecond.toDF("cid1","cid2","score").withColumn("id",row_number().over(w1))
+      val score_result = remapSecond.toDF("cid1","cid2","score")
       GmeiConfig.writeToJDBCTable(score_result, table="cid_pairs_cosine_distince", SaveMode.Overwrite)
 
 
@@ -189,8 +186,7 @@ object Main {
       // print out the results for the first 10 words
       result.take(20).foreach(println)
 
-      val w2 = Window.orderBy($"cid")
-      val similar_result = result.toDF("cid","similarity_cid").withColumn("id",row_number().over(w2))
+      val similar_result = result.toDF("cid","similarity_cid")
       GmeiConfig.writeToJDBCTable(similar_result, table="cid_similarity_matrix", SaveMode.Overwrite)
 
     }