featureeng_down.sh

dataPath=/data/files
content_type="service"
cd $dataPath
function mergeGetFile(){
    if [ ! -n "$1" ];then
        echo "dir Doesn't exist,don't run this shell"
        exit 1
    fi
    rm -f $dataPath/$1.csv
#    logging "rm -f $dataPath/$1.csv"
    /opt/hadoop/bin/hdfs dfs -getmerge $1 $dataPath/$1.csv
#    logging "/opt/hadoop/bin/hdfs dfs -getmerge $1 $dataPath/$1.csv success"
    head -1 $1.csv > $1.csv.head
    cat $1.csv|grep -v `cat $1.csv.head` >> $1.csv.head
    mv $1.csv.head $1.csv
    /opt/hadoop/bin/hdfs dfs -rmr /$1
#    logging "/opt/hadoop/bin/hdfs dfs -rmr /$1 success"
}

mergeGetFile ${content_type}_feature_v1_train
mergeGetFile ${content_type}_feature_v1_test