try predict offline

ctcvr_offline.sh

@@ -20,20 +20,20 @@
 
 if ! ps aux | grep src/main_portrait.py\ 8 | grep -v grep
 then
-  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py 8 > ~/ctcvr_8.log &
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py 8 f > ~/ctcvr_8.log &
 fi
 
 if ! ps aux | grep src/main_portrait.py\ 9 | grep -v grep
 then
-  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py 9 > ~/ctcvr_9.log &
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py 9 f > ~/ctcvr_9.log &
 fi
 
 if ! ps aux | grep src/main_portrait.py\ a | grep -v grep
 then
-  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py a > ~/ctcvr_a.log &
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py a f > ~/ctcvr_a.log &
 fi
 
 if ! ps aux | grep src/main_portrait.py\ b | grep -v grep
 then
-  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py b > ~/ctcvr_b.log &
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py b f > ~/ctcvr_b.log &
 fi

ctcvr_offline_increment.sh

+if ! ps aux | grep src/main_portrait.py\ 8 | grep -v grep
+then
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py 8 true > ~/ctcvr_8_inc.log &
+fi
+
+if ! ps aux | grep src/main_portrait.py\ 9 | grep -v grep
+then
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py 9 true > ~/ctcvr_9_inc.log &
+fi
+
+if ! ps aux | grep src/main_portrait.py\ a | grep -v grep
+then
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py a true > ~/ctcvr_a_inc.log &
+fi
+
+if ! ps aux | grep src/main_portrait.py\ b | grep -v grep
+then
+  nohup /home/gmuser/.virtualenvs/tf1/bin/python3 /srv/apps/gm_strategy_cvr/src/main_portrait.py b true > ~/ctcvr_b_inc.log &
+fi

src/main_portrait.py

@@ -11,7 +11,8 @@ from models.esmm.tractate_model import model_predict_tractate
 from utils.cache import get_essm_model_save_path, redis_client2
 from utils.grey import recommed_service_category_device_id_by_tail
 from utils.portrait import (get_user_portrait_tag3_read_v2, user_portrait_tag3_get_candidate_dict,
-                            user_portrait_tag3_get_candidate_unread_list, user_portrait_tag3_write_ctcvr_data)
+                            user_portrait_tag3_get_candidate_unread_list, user_portrait_tag3_get_ctcvr_queue,
+                            user_portrait_tag3_write_ctcvr_data)
 
 
 def user_portrait_scan_info(device_dict, diary_dict, tractate_dict, diary_predict_fn, tractate_predict_fn, tail_number):
@@ -42,6 +43,35 @@ def user_portrait_scan_info(device_dict, diary_dict, tractate_dict, diary_predic
         print(e)
 
 
+def user_portrait_increment_scan_info(device_dict, diary_dict, tractate_dict, diary_predict_fn, tractate_predict_fn, tail_number):
+    try:
+        round = 0
+        all_count = 0
+        keys = "doris:user_portrait:tag3:device_id:*"
+        cur, results = redis_client2.scan(0, keys, 3000)
+        while cur != 0:
+            round += 1
+            print("round: " + str(round))
+            cur, results = redis_client2.scan(cur, keys, 3000)
+            for key in results:
+                key = str(key, "utf-8")
+                device_id = key.split(":")[-1]
+
+                if recommed_service_category_device_id_by_tail(device_id, [tail_number]):
+                    if user_portrait_tag3_get_candidate_dict(
+                            device_id, "diary") and (not (user_portrait_tag3_get_ctcvr_queue(device_id, "diary"))):
+                        all_count += 1
+                        print(str(all_count) + ": " + device_id)
+                        offline_predict_diary(device_id, device_dict, diary_dict, diary_predict_fn, size=200)
+                        offline_predict_tractate(device_id, device_dict, tractate_dict, tractate_predict_fn, size=200)
+                        print("=========================================\n")
+
+        print("all count: " + str(all_count))
+        print("scan done " + str(datetime.datetime.now()))
+    except Exception as e:
+        print(e)
+
+
 def offline_predict_diary(device_id, device_dict, diary_dict, predict_fn, size=300):
     time_begin = time.time()
 
@@ -99,7 +129,13 @@ def main():
     # print(res[:10])
 
     tail_number = sys.argv[1]  # "c", "d", "e", "f"
-    user_portrait_scan_info(device_dict, diary_dict, tractate_dict, diary_predict_fn, tractate_predict_fn, tail_number)
+    increment_scan = sys.argv[2]
+    if (increment_scan == "true"):
+        print("aaaa")
+        # user_portrait_increment_scan_info(device_dict, diary_dict, tractate_dict, diary_predict_fn, tractate_predict_fn,
+        #                                   tail_number)
+    else:
+        user_portrait_scan_info(device_dict, diary_dict, tractate_dict, diary_predict_fn, tractate_predict_fn, tail_number)
 
 
 if __name__ == "__main__":

src/utils/portrait.py

@@ -134,6 +134,11 @@ def get_user_portrait_tag3_read_v2(device_id, content_type):
     return read_list_v2, today_read
 
 
+def user_portrait_tag3_get_ctcvr_queue(device_id, content_type):
+    key = _user_portrait_tag3_get_ctcvr_candidate_key(device_id, content_type)
+    return redis_client4.exists(key)
+
+
 def user_portrait_tag3_get_candidate_dict(device_id, content_type, latest=False):
     key = _user_portrait_tag3_get_candidate_key(device_id, content_type, latest)
     res = redis_client4.get(key)