update config

word_vector/tests.py

@@ -6,7 +6,7 @@ from pytispark import pytispark as pti
 # Create your tests here.
 
 
-def get_spark():
+def get_spark(name=""):
     sparkConf = SparkConf()
     sparkConf.set("spark.sql.crossJoin.enabled", True)
     sparkConf.set("spark.debug.maxToStringFields", "100")
@@ -18,17 +18,17 @@ def get_spark():
     sparkConf.set("mapreduce.output.fileoutputformat.compress", False)
     sparkConf.set("mapreduce.map.output.compress", False)
     spark = (SparkSession.builder.config(conf=sparkConf).config(
-        "spark.sql.extensions", "org.apache.spark.sql.TiExtensions").config(
-            "spark.tispark.pd.addresses",
-            "172.16.40.170:2379").appName("SERVICE FM CSV EXPORT").enableHiveSupport().getOrCreate())
+        "spark.sql.extensions",
+        "org.apache.spark.sql.TiExtensions").config("spark.tispark.pd.addresses",
+                                                    "172.16.40.170:2379").appName(name).enableHiveSupport().getOrCreate())
     ti = pti.TiContext(spark)
     ti.tidbMapDatabase("jerry_test")
     return spark
 
+
 if __name__ == "__main__":
-    spark = get_spark()
-    a = spark.sql("select * from online.bl_hdfs_maidian_updates limit 1")
+    spark = get_spark("test")
+    a = spark.sql("select cl_id from online.bl_hdfs_maidian_updates limit 1")
     a.show()
 
-
-# spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/strategy_embedding/word_vector/tests.py
+# /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/strategy_embedding/word_vector/tests.py