change gmkv db

eda/smart_rank/dist_update_user_portrait.py

@@ -132,12 +132,19 @@ if __name__ == '__main__':
         cur_jerry_test = db_jerry_test.cursor()
 
         # 获取所有用户的设备id
-        sql_device_ids = "select distinct cl_id from user_new_tag_log"
+        # sql_device_ids = "select distinct cl_id from user_new_tag_log"
+        # 获取最近30天内的用户设备id
+        sql_device_ids = "select distinct cl_id from user_new_tag_log " \
+                         "where time > UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 30 day))"
         cur_jerry_test.execute(sql_device_ids)
         device_ids_lst = [i[0] for i in cur_jerry_test.fetchall()]
 
         # 获取所有用户的行为日志
-        sql_all_log = "select time,cl_id,score_type,tag_id,tag_referrer,action from user_new_tag_log"
+        # sql_all_log = "select time,cl_id,score_type,tag_id,tag_referrer,action from user_new_tag_log"
+        # 获取最近30天内的用户的所有行为
+        sql_all_log = "select time,cl_id,score_type,tag_id,tag_referrer,action from user_new_tag_log where cl_id in " \
+                      "(select distinct cl_id from user_new_tag_log " \
+                      "where time > UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 30 day)))"
         cur_jerry_test.execute(sql_all_log)
         all_log = cur_jerry_test.fetchall()
         db_jerry_test.close()

eda/smart_rank/start.sh

@@ -4,4 +4,6 @@
 
 mysql -u root -p3SYz54LS9#^9sBvC -h 172.16.40.158 -P 4000 -D jerry_test -e "delete from jerry_test.user_new_tag_log where id in (select a.id from jerry_test.user_new_tag_log a left join eagle.src_zhengxing_api_tag b on a.tag_id=b.id where b.tag_type+0 > '3'+0)"
 
-/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.spark --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/ffm-baseline_git/eda/smart_rank/dist_update_user_portrait.py
+mysql -u root -p3SYz54LS9#^9sBvC -h 172.16.40.158 -P 4000 -D jerry_test -e "delete from user_new_tag_log where time < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 180 day))"
\ No newline at end of file
+
+/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.spark --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 30 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/ffm-baseline_git/eda/smart_rank/dist_update_user_portrait.py
\ No newline at end of file