update

crawler_sys/framework/es_target_releasers.py

@@ -9,7 +9,6 @@ import random
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import scan
 
-#  rds=redis.StrictRedis(host='192.168.17.26',port=6379,db=0)
 hosts = '172.18.52.14'
 port = 9200
 

run.sh

 #!/bin/bash
 #sudo su - gmuser
-crawler-ops
+#crawler-ops
 #conda activate crawler_env
-python  /srv/apps/crawler/crawler_sys/framework/write_releasers_to_redis.py -p weibo -d 1 -proxies 5 > /data/log/crawler/write_task.log &
+source /srv/envs/crawler/bin/activate
+python  /srv/apps/crawler_old/crawler/crawler_sys/framework/write_releasers_to_redis.py -p weibo -d 1 -proxies 5 > /data/log/crawler/write_task.log &
 #/home/gmuser/.virtualenvs/litao/bin/python3  /srv/apps/crawler/crawler_sys/framework/write_releasers_to_redis.py -p douban -d 1 -proxies 5 > /data/log/crawler/write_task.log &

start_crawler.sh

@@ -2,8 +2,9 @@
 #source /root/anaconda3/bin/activate
 #conda activate crawler_env
 sudo su - gmuser
+source /srv/envs/crawler/bin/activate
 ps aux|grep update_data_in_target_releasers_multi_process_by_date_from_redis|grep -v grep|cut -c 9-15|xargs kill -15
 ps aux|grep cal_ni_and_put_to_backend|grep -v grep|cut -c 9-15|xargs kill -15
 current_date=`date -d "-1 day" "+%Y%m%d"`
-python /srv/apps/crawler/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py > /data/log/crawler/write_task_${current_date}_.log &
-python /srv/apps/crawler/crawler_sys/scheduler/cal_ni_and_put_to_backend.py > /data/log/crawler/.log &
\ No newline at end of file
+python /srv/apps/crawler_old/crawler/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py > /data/log/crawler/write_task_${current_date}_.log &
+python /srv/apps/crawler_old/crawler/crawler_sys/scheduler/cal_ni_and_put_to_backend.py > /data/log/crawler/.log &
\ No newline at end of file

tasks/run.sh

 source /srv/envs/esmm/bin/activate
-python /srv/apps/crawler/write_data_into_es/revise_data_from_es.py
\ No newline at end of file
+python /srv/apps/crawler_old/crawler/write_data_into_es/revise_data_from_es.py
\ No newline at end of file