模型上线

shell/featureeng_export.sh

 source /srv/envs/serviceRec/bin/activate
 python_os=/srv/envs/serviceRec/bin/python
 path=/srv/apps/serviceRec
-day_count=$1
-spark_mode=$2
+trainDays=$1
+itemStatisticStartDays=$2
+spark_mode=$3
 content_type="service"
-pythonFile=${path}/spark/featureEng.py
+pythonFile=${path}/spark/featureEngSk.py
 #log_file=~/${content_type}_feature_csv_export.log
 /opt/hadoop/bin/hdfs dfs -rmr /${content_type}_feature_v1_train
 /opt/hadoop/bin/hdfs dfs -rmr /${content_type}_feature_v1_test
 if [ -n "${spark_mode}" ]; then
-  /opt/spark/bin/spark-submit --master local[4] --deploy-mode client --driver-memory 8g --executor-memory 2g --executor-cores 1 --num-executors 4 --conf spark.pyspark.python=${python_os} --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar ${pythonFile} $day_count
+  /opt/spark/bin/spark-submit --master local[8] --deploy-mode client --driver-memory 16g --executor-memory 2g --executor-cores 1 --num-executors 4 --conf spark.pyspark.python=${python_os} --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar ${pythonFile} $trainDays $itemStatisticStartDays
   else
-  /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 8g --executor-memory 2g --executor-cores 1 --num-executors 8 --conf spark.pyspark.python=${python_os} --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar ${pythonFile} $day_count
+  /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 8g --executor-memory 2g --executor-cores 1 --num-executors 8 --conf spark.pyspark.python=${python_os} --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar ${pythonFile} $trainDays $itemStatisticStartDays
 fi
 
 #/opt/spark/bin/spark-submit --master local[4] --deploy-mode client --driver-memory 8g --executor-memory 2g --executor-cores 1 --num-executors 4 --conf spark.pyspark.python=/srv/envs/serviceRec/bin/python --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar ${pythonFile} $day_count

shell/models_upload.sh

-cd /srv/apps/tensorServing_models/
+cd /data/files/wideAndDeep/tensorServing_models/
 fileNum=`ls service/|wc -l`
 function pushModel() {
   #文件打包

shell/service_train.sh

-cd /srv/apps/tensorServing_models && rm -rf service_copy && mv service service_copy && mkdir service
+cd /data/files/wideAndDeep/tensorServing_models && rm -rf service_copy && mv service service_copy && mkdir service
 source /srv/envs/serviceRec/bin/activate
-python /srv/apps/serviceRec/train/train_service.py > /srv/apps/serviceRec/logs/train_service_log.log
+python /srv/apps/serviceRec/train/train_service_sk_tf2_distibute.py > /srv/apps/serviceRec/logs/train_service_log.log
\ No newline at end of file

spark/down_samples_and_vocab.py

+import redis
+import sys
+import os
+import json
+
+def getRedisConn():
+    pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
+    conn = redis.Redis(connection_pool=pool)
+    # conn = redis.Redis(host="172.16.50.145", port=6379, password="XfkMCCdWDIU%ls$h",db=0)
+    # conn = redis.Redis(host="172.18.51.10", port=6379, db=0, decode_responses = True) #test
+    return conn
+
+if len(sys.argv) == 2:
+    save_dir = sys.argv[1]
+else:
+    save_dir = '/data/files/wideAndDeep/trainData/'
+
+print('save_dir: ', save_dir)
+if not os.path.exists(save_dir):
+    print('mkdir save_dir: ', save_dir)
+    os.makedirs(save_dir)
+
+conn = getRedisConn()
+vocab_keys = conn.lrange("strategy:all:vocab", 0, -1)
+print("vocab_keys: ", vocab_keys[0])
+vocab_keys = eval(vocab_keys[0])
+
+
+
+for vocab_key in vocab_keys:
+    print('vocab_key: ', vocab_key)
+    splits = vocab_key.split(":")
+    field = splits[1]
+    filename = field + "_vocab.csv"
+
+    print('filename: ', filename)
+    with open(os.path.join(save_dir, filename), 'w') as f:
+        texts = conn.lrange(vocab_key, 0, -1)
+        texts = list(filter(lambda x: x != '', eval(texts[0])))
+        print('texts: ', len(texts))
+        f.write('\n'.join(texts))
+
+os.system("hdfs dfs -getmerge /strategy/train_samples {save_dir}train_samples.csv".format(save_dir = save_dir))
+os.system("hdfs dfs -getmerge /strategy/eval_samples {save_dir}eval_samples.csv".format(save_dir = save_dir))
+