change test file

tensnsorflow/feature_engineering.py

@@ -35,11 +35,46 @@ def multi_hot(df,column,n):
     app_list_map = dict(zip(app_list_unique, list(range(n, number + n))))
     return number,app_list_map
 
+def get_list(db,sql,n):
+    cursor = db.cursor()
+    cursor.execute(sql)
+    result = cursor.fetchall()
+    v = list(set([i[0] for i in result]))
+    app_list_value = [str(i).split(",") for i in v]
+    app_list_unique = []
+    for i in app_list_value:
+        app_list_unique.extend(i)
+    app_list_unique = list(set(app_list_unique))
+    number = len(app_list_unique)
+    app_list_map = dict(zip(app_list_unique, list(range(n, number + n))))
+    db.close()
+    return number, app_list_map
+
 
 def get_map():
     db = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
-    sql = "select max(stat_date) from esmm_train_data"
-    validate_date = con_sql(db, sql)[0].values.tolist()[0]
+    sql = "select app_list from device_app_list"
+    a = time.time()
+    apps_number, app_list_map = get_list(db,sql,1)
+    print("applist")
+    print((time.time()-a)/60)
+    db = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    sql = "select level2_ids from diary_feat"
+    b = time.time()
+    leve2_number, leve2_map = get_list(db, sql, apps_number)
+    print("leve2")
+    print((time.time() - b) / 60)
+    db = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    sql = "select level3_ids from diary_feat"
+    c = time.time()
+    leve3_number, leve3_map = get_list(db, sql, leve2_number)
+    print((time.time() - c) / 60)
+
+
+
+
+
+
 
 def feature_engineer():
     db = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
@@ -251,25 +286,25 @@ def con_sql(db,sql):
 
 
 if __name__ == '__main__':
-    sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true") \
-        .set("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive", "true") \
-        .set("spark.tispark.plan.allow_index_double_read", "false") \
-        .set("spark.tispark.plan.allow_index_read", "true") \
-        .set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \
-        .set("spark.tispark.pd.addresses", "172.16.40.158:2379").set("spark.io.compression.codec", "lzf")\
-        .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec","snappy")
-
-    spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
-    ti = pti.TiContext(spark)
-    ti.tidbMapDatabase("jerry_test")
-    ti.tidbMapDatabase("eagle")
-    spark.sparkContext.setLogLevel("WARN")
-    path = "hdfs:///strategy/esmm/"
-    local_path = "/home/gmuser/esmm/"
-
-    validate_date, value_map, app_list_map, leve2_map, leve3_map = feature_engineer()
-    get_predict(validate_date, value_map, app_list_map, leve2_map, leve3_map)
-
+    # sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true") \
+    #     .set("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive", "true") \
+    #     .set("spark.tispark.plan.allow_index_double_read", "false") \
+    #     .set("spark.tispark.plan.allow_index_read", "true") \
+    #     .set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \
+    #     .set("spark.tispark.pd.addresses", "172.16.40.158:2379").set("spark.io.compression.codec", "lzf")\
+    #     .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec","snappy")
+    #
+    # spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
+    # ti = pti.TiContext(spark)
+    # ti.tidbMapDatabase("jerry_test")
+    # ti.tidbMapDatabase("eagle")
+    # spark.sparkContext.setLogLevel("WARN")
+    # path = "hdfs:///strategy/esmm/"
+    # local_path = "/home/gmuser/esmm/"
+    #
+    # validate_date, value_map, app_list_map, leve2_map, leve3_map = feature_engineer()
+    # get_predict(validate_date, value_map, app_list_map, leve2_map, leve3_map)
+    get_map()
 
 
 

tensnsorflow/multi.py

@@ -164,7 +164,7 @@ if __name__ == '__main__':
     cursor.execute(sql)
     result = cursor.fetchall()
     print(result)
-    a = [i[0] for i in result]
+    a = list(set([i[0] for i in result]))
     print(a)