美购精排模型

shell/featureeng_export.sh

 path=/srv/apps/serviceRec
 day_count=$1
 content_type="service"
-source /srv/envs/serviceRec/bin/activate
 pythonFile=${path}/spark/featureEng.py
 #log_file=~/${content_type}_feature_csv_export.log
 /opt/hadoop/bin/hdfs dfs -rmr /${content_type}_feature_train

spark/featureEng.py

@@ -270,15 +270,11 @@ def featureColumnsToRedis(columns):
 
 def featureToRedis(key,datas):
     conn = connUtils.getRedisConn()
-    pipeline = conn.pipeline()
     for k,v in datas.items():
         newKey = key+k
-        pipeline.set(newKey,v)
-        pipeline.expire(newKey, 60 * 60 * 24 * 7)
+        conn.set(newKey,v)
+        conn.expire(newKey, 60 * 60 * 24 * 7)
 
-    pipeline.execute()
-    pipeline.close()
-    conn.close()
 
 def collectFeaturesToDict(samples,columns,prefix):
     idCol = prefix+"id"