增加缓存视频cache_video_id

diaryQueueUpdate.py

@@ -13,7 +13,7 @@ import time
 from config import *
 
 
-def get_video_id():
+def get_video_id(cache_video_id):
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='eagle')
     cursor = db.cursor()
     sql = "select diary_id from feed_diary_boost;"
@@ -24,7 +24,7 @@ def get_video_id():
     print(df.head(1))
     db.close()
     if df.empty:
-        return False
+        return cache_video_id
     else:
         video_id = df[0].values.tolist()
         return video_id
@@ -112,7 +112,7 @@ def get_score(queue_arg):
 
 def update_dairy_queue(score_df,predict_score_df,total_video_id):
     diary_id = score_df["cid"].values.tolist()
-    if total_video_id:
+    if total_video_id != []:
         video_id = list(set(diary_id)&set(total_video_id))
         if len(video_id)>0:
             not_video = list(set(diary_id) - set(video_id))
@@ -145,7 +145,7 @@ def update_dairy_queue(score_df,predict_score_df,total_video_id):
             score_df = score_df.sort_values(by="score", ascending=False)
             # print("分数合并成功1")
             return score_df.index.tolist()
-    # 如果total_video_id是空
+    # 如果total_video_id是空列表
     else:
         score_df = score_df.set_index(["cid"])
         predict_score_df = predict_score_df.set_index(["cid"])
@@ -155,7 +155,6 @@ def update_dairy_queue(score_df,predict_score_df,total_video_id):
         return score_df.index.tolist()
 
 
-
 def update_sql_dairy_queue(queue_name, diary_id,device_id, city_id):
     db = pymysql.connect(host='rm-m5e842126ng59jrv6.mysql.rds.aliyuncs.com', port=3306, user='doris',
                          passwd='o5gbA27hXHHm', db='doris_prod')
@@ -238,9 +237,12 @@ def multi_proecess_update(device_id, city_id, data_set_cid,total_video_id):
 if __name__ == "__main__":
     warnings.filterwarnings("ignore")
     total_number = 0
+    # 增加缓存日记视频列表
+    cache_video_id = []
     while True:
         data_set_cid = pd.read_csv(DIRECTORY_PATH + "data_set_cid.csv")["cid"].values.tolist()
-        total_video_id = get_video_id()
+        total_video_id = get_video_id(cache_video_id)
+        cache_video_id = total_video_id
         device_city_list = get_active_users()
         total_number += len(device_city_list)
         print("累计预测用户总数：{}".format(total_number))

keepProcess.py

+import os
+import time
+
+
+def check():
+    out = os.popen("ps aux | grep diaryQueueUpdate.py").read()
+    flag = 1
+    for line in out.splitlines():
+        if 'python diaryQueueUpdate.py' in line:
+            flag = 2
+    return flag
+
+
+if __name__ == "__main__":
+    while True:
+        if check() == 1:
+            os.popen('python diaryQueueUpdate.py')
+            print("成功重启diaryQueueUpdate")
+        time.sleep(300)
\ No newline at end of file

train.py

@@ -17,7 +17,8 @@ if __name__ == "__main__":
     end_train = time.time()
     print("训练模型耗时{}分".format((end_train-start_train)/60))
     move_file()
-    kill_process()
+    #TODO 如果用自己写的keepProcess文件守护进程，下面在这个函数里删掉重新启动进程那行代码，因为可能会多启动一次进程
+    restart_process()
 
 
 

utils.py

@@ -55,13 +55,22 @@ def move_file():
     print("成功将文件剪切到对应路径")
 
 
-def kill_process():
+def restart_process():
     out = os.popen("ps aux | grep diaryQueueUpdate.py").read()
     for line in out.splitlines():
         if 'python diaryQueueUpdate.py' in line:
             pid = int(line.split()[1])
-            os.kill(pid, signal.SIGKILL)
-            print("已杀死python diaryQueueUpdate.py 进程")
+            # 有些进程的生命周期非常短或者随时可能结束，一定要捕捉这个异常
+            try:
+                os.kill(pid, signal.SIGKILL)
+                print("已杀死python diaryQueueUpdate.py 进程")
+            except OSError:
+                print('没有如此进程!!!')
+            os.popen('python diaryQueueUpdate.py')
+            print("成功重启diaryQueueUpdate")
+        else:
+            os.popen('python diaryQueueUpdate.py')
+            print("成功重启diaryQueueUpdate")
 
 
 # 把数据写到redis里