Skip to content

P
physical
Commit 5f58738a authored by 黄凯's avatar 黄凯
Browse files
Options

Merge branch 'test' into 'hk' 

# Conflicts:
#   linucb/views/collect_data.py
#   search/views/topic.py
parents e11006a6 f4fea10d
Hide whitespace changes
Inline Side-by-side
Showing with 714 additions and 191 deletions
app_conf.xml
View file @ 5f58738a
......@@ -17,6 +17,5 @@
<element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/>
<!--<element value="majia..tasks.tasks"/>-->
</config>
</gm_rpcd_config>
majia/get_session.py → injection/data_sync/get_session.py
View file @ 5f58738a
......@@ -8,15 +8,15 @@ import logging
DATABASES = settings.DATABASES
# host = DATABASES['default']['HOST']
#
# user = DATABASES['default']['USER']
#
# port = DATABASES['default']['PORT']
#
# db = DATABASES['default']['NAME']
#
# passwd = DATABASES['default']['PASSWORD']
host = DATABASES['default']['HOST']
user = DATABASES['default']['USER']
port = DATABASES['default']['PORT']
db = DATABASES['default']['NAME']
passwd = DATABASES['default']['PASSWORD']
auto_click_url = "http://earth.igengmei.com/api/v1/like"
auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
......@@ -26,7 +26,7 @@ auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
def get_majia():
try:
data = open("/srv/apps/physical/majia/majia_email.txt")
data = open("/srv/apps/physical/injection/data_sync/majia_email.txt")
list = []
for i in data:
list.append(i.strip('\n').strip(','))
......@@ -41,7 +41,7 @@ def get_majia():
def get_cookies(user_id):
try:
with open("/srv/apps/physical/majia/user_session.txt", 'r') as f:
with open("/srv/apps/cybertron/user_session.txt", 'r') as f:
try:
data = f.readlines()
except:
......@@ -123,7 +123,7 @@ def time_conv(minutest):
def get_comments():
try:
data = open("/srv/apps/physical/majia/guanshui.txt", "r")
data = open("/srv/apps/physical/injection/data_sync/guanshui.txt", "r")
list_guanshui = []
for i in data:
list_guanshui.append(i)
......
injection/data_sync/majia_email.txt 0 → 100644
View file @ 5f58738a
s_sgMyOm@shadow.com
s_4HRYFK@shadow.com
s_KaRMJO@shadow.com
s_07ssdt@shadow.com
s_zMc7wq@shadow.com
s_zA014B@shadow.com
s_akgPyg@shadow.com
s_vKSOXA@shadow.com
s_oi7M2a@shadow.com
s_lOTICx@shadow.com
s_i4rFhk@shadow.com
s_UPqB4b@shadow.com
s_do9K3h@shadow.com
s_gXRMNW@shadow.com
s_GGopQR@shadow.com
s_be9Zi8@shadow.com
s_hDAJDM@shadow.com
s_yQJV10@shadow.com
s_axSkdT@shadow.com
s_QI5QFa@shadow.com
s_ERIRTt@shadow.com
s_zNA0qR@shadow.com
s_by2EKN@shadow.com
s_yRDyTe@shadow.com
s_mAu8xe@shadow.com
s_xNXLax@shadow.com
s_xkRtne@shadow.com
s_Dpguo5@shadow.com
s_i7ILAu@shadow.com
s_fjgn6l@shadow.com
s_iwSVzk@shadow.com
s_u39tAC@shadow.com
s_BqeGFr@shadow.com
s_cA91fz@shadow.com
s_ESqCxg@shadow.com
s_8G7dcC@shadow.com
s_yUirRE@shadow.com
s_upU0XW@shadow.com
s_e1gsjY@shadow.com
s_ShQyke@shadow.com
s_KzuAYn@shadow.com
s_hsG5eH@shadow.com
s_cF7TuX@shadow.com
s_bNzT3y@shadow.com
s_E4I3Xs@shadow.com
s_AdUmQr@shadow.com
s_JU1HJp@shadow.com
s_e6gQH3@shadow.com
s_YSpR4I@shadow.com
s_36xYLw@shadow.com
s_b5CSos@shadow.com
s_fGrHfu@shadow.com
s_OTNbWY@shadow.com
s_BmjUyS@shadow.com
s_5gjbBq@shadow.com
s_o1KuZK@shadow.com
s_4cYuPa@shadow.com
s_ql2jNt@shadow.com
s_TlQiUV@shadow.com
s_oD1Hmy@shadow.com
s_lRgLWN@shadow.com
s_hO4VFC@shadow.com
s_AviTFG@shadow.com
s_2Lago0@shadow.com
s_IN3CIK@shadow.com
s_49T1iu@shadow.com
s_QZ5xev@shadow.com
s_f0ULrl@shadow.com
s_XubfZG@shadow.com
s_PzH35A@shadow.com
s_3chHX1@shadow.com
s_Eh80vb@shadow.com
s_XAkiWi@shadow.com
s_DsRsnp@shadow.com
s_S4WZXB@shadow.com
s_QqYjNu@shadow.com
s_FMZSoT@shadow.com
s_NTvlOB@shadow.com
s_ei0zs6@shadow.com
s_KnFg4t@shadow.com
s_ropbJR@shadow.com
s_3Q98tn@shadow.com
s_EbB7e0@shadow.com
s_NdwviG@shadow.com
s_fIH36h@shadow.com
s_HrFCH6@shadow.com
s_3ixLfv@shadow.com
s_IqssSO@shadow.com
s_V1GHJz@shadow.com
s_Lt4V85@shadow.com
s_exsZWG@shadow.com
s_Py0oZ4@shadow.com
s_W4RqbA@shadow.com
s_s1L2lv@shadow.com
s_hgZehK@shadow.com
s_KU3D7m@shadow.com
s_JwVmFg@shadow.com
s_uZRiM7@shadow.com
s_LNhnOE@shadow.com
s_YcQsmL@shadow.com
s_AAXg2x@shadow.com
s_1N8qG8@shadow.com
s_hpC1IP@shadow.com
s_VWL5Pk@shadow.com
s_y8atXB@shadow.com
s_igjXaA@shadow.com
s_axJVqe@shadow.com
s_hhSSqd@shadow.com
s_8Evf2c@shadow.com
s_EvDs7H@shadow.com
s_F6FHfH@shadow.com
s_RGun9G@shadow.com
s_qVKF9Q@shadow.com
s_i6ffeZ@shadow.com
s_E0txbx@shadow.com
s_DVwDK3@shadow.com
s_gwVryk@shadow.com
s_Cm7btb@shadow.com
s_WP3s7w@shadow.com
s_ZdLDZo@shadow.com
s_C2PiMv@shadow.com
s_GPR0C1@shadow.com
s_wnrxFo@shadow.com
s_sNyLO4@shadow.com
s_08DlWX@shadow.com
s_rTS8qV@shadow.com
s_nd6HiH@shadow.com
s_a9cIf1@shadow.com
s_5I3fN3@shadow.com
s_TWwKNi@shadow.com
s_WrVSmD@shadow.com
s_Twwxb8@shadow.com
s_DJvlFY@shadow.com
s_KTcosq@shadow.com
s_5o4RJL@shadow.com
s_hsj9pq@shadow.com
s_7TOm5v@shadow.com
s_enXoFU@shadow.com
s_F9hH2t@shadow.com
s_fJD9SK@shadow.com
s_895iAz@shadow.com
s_oniMPE@shadow.com
s_y6GymP@shadow.com
s_UGN1aM@shadow.com
s_1RCzUi@shadow.com
s_GPok7I@shadow.com
s_hX41uJ@shadow.com
s_jYDXVu@shadow.com
s_5M8FPz@shadow.com
s_0KyxOu@shadow.com
s_wWtI7X@shadow.com
s_upwHbj@shadow.com
s_kWGe6S@shadow.com
s_GtpUFG@shadow.com
s_yMfJP3@shadow.com
s_g3zIuG@shadow.com
s_bLymnh@shadow.com
s_xvwj7J@shadow.com
s_n2rgzI@shadow.com
s_kfGtFE@shadow.com
s_lK7QJJ@shadow.com
s_bOQCNQ@shadow.com
s_1mypap@shadow.com
s_ux8AJL@shadow.com
s_twoHks@shadow.com
s_xgZqN4@shadow.com
s_MaChwe@shadow.com
s_NzQnI9@shadow.com
s_87PpDu@shadow.com
s_UTQ2Lb@shadow.com
s_2ZlLCx@shadow.com
s_5FAGd4@shadow.com
s_TNhLPD@shadow.com
s_uujHEb@shadow.com
s_wBqKNY@shadow.com
s_MrbRuR@shadow.com
s_FPb33o@shadow.com
s_3c7tUk@shadow.com
s_s7uz0U@shadow.com
s_v3TI1p@shadow.com
s_djjDaF@shadow.com
s_NjoRTW@shadow.com
s_CDooAN@shadow.com
s_pkMFSS@shadow.com
s_OU8QCt@shadow.com
s_rSlV4T@shadow.com
s_MtxuFr@shadow.com
s_H36e9F@shadow.com
s_xnXhe5@shadow.com
s_7nWGkq@shadow.com
s_KtWEu9@shadow.com
s_FpzO9Q@shadow.com
s_zhOQcV@shadow.com
s_T8Gcjy@shadow.com
s_ZsMr7d@shadow.com
s_XgWX4z@shadow.com
s_1ozsiy@shadow.com
s_UPD6bx@shadow.com
s_zsvmqU@shadow.com
s_HfHwKX@shadow.com
s_FihASr@shadow.com
s_KRN7h9@shadow.com
s_Jmkcc9@shadow.com
s_d7TIgK@shadow.com
s_gtMV3b@shadow.com
s_XVsXeA@shadow.com
s_edvBYb@shadow.com
s_YiX7OV@shadow.com
s_EAYAMe@shadow.com
s_IR0Jmz@shadow.com
s_KphQ2M@shadow.com
s_lPRv0u@shadow.com
s_7FUcFf@shadow.com
s_FGgy48@shadow.com
s_ltsupK@shadow.com
s_Ee4QC1@shadow.com
s_gqBA3v@shadow.com
s_embU8E@shadow.com
s_0I4C0g@shadow.com
s_EzpzqV@shadow.com
s_98A1Z2@shadow.com
s_p5kFP1@shadow.com
s_6srtjM@shadow.com
s_FJyYaA@shadow.com
s_TSJPo5@shadow.com
s_8nvaAF@shadow.com
s_6rkYkC@shadow.com
s_rUnoMo@shadow.com
s_Z8PP5J@shadow.com
s_Q5LUMj@shadow.com
s_v26xAA@shadow.com
s_HqZEg7@shadow.com
s_8gSWmC@shadow.com
s_fCQcu1@shadow.com
s_91xnox@shadow.com
s_gvlFut@shadow.com
s_nyfbCS@shadow.com
s_uA9TOF@shadow.com
s_JLlz1L@shadow.com
s_vm9MFh@shadow.com
s_Lqxoqm@shadow.com
s_uhXH6g@shadow.com
s_Wyn2gW@shadow.com
s_rtCnmO@shadow.com
s_RyIPYo@shadow.com
s_bS8mIT@shadow.com
s_eUL2wx@shadow.com
s_rggaRa@shadow.com
s_T93ZyL@shadow.com
s_qnVpkI@shadow.com
s_hHpKxO@shadow.com
s_OgaFKI@shadow.com
s_kJWvzA@shadow.com
s_ilPCgr@shadow.com
s_0bCas9@shadow.com
s_gJ4l7J@shadow.com
s_jBC0ve@shadow.com
s_8e80N0@shadow.com
s_4jPHc1@shadow.com
s_2LjC8I@shadow.com
s_XXGEwN@shadow.com
s_VbZq9G@shadow.com
s_5rfk9c@shadow.com
s_kzuGZj@shadow.com
s_yUUJmV@shadow.com
s_Kmd5N3@shadow.com
s_wvsmbJ@shadow.com
s_ROiMbb@shadow.com
s_sAZ7L8@shadow.com
s_UJWzA6@shadow.com
s_mCazhk@shadow.com
s_cuQRSf@shadow.com
s_3Us8vf@shadow.com
s_pxACJM@shadow.com
s_jRQhe1@shadow.com
s_O8qfPf@shadow.com
s_rQC4Tn@shadow.com
s_09lKhD@shadow.com
s_J3uR9F@shadow.com
s_8vCW0n@shadow.com
s_53EyN7@shadow.com
s_jlNKoI@shadow.com
s_prpHwa@shadow.com
s_NBpqo6@shadow.com
s_0YjkyC@shadow.com
s_oD0SPa@shadow.com
s_cRZ820@shadow.com
s_QuSUc9@shadow.com
s_1aQRIO@shadow.com
s_8WpiMk@shadow.com
s_mFbRwb@shadow.com
s_P0Y9Tk@shadow.com
s_GakHBe@shadow.com
s_XQ3BZh@shadow.com
s_gJDa8J@shadow.com
s_6MhNqA@shadow.com
s_fvdNZW@shadow.com
s_5J2GxW@shadow.com
s_JluV3t@shadow.com
s_Iblrkj@shadow.com
s_335ZV4@shadow.com
s_CHiVph@shadow.com
s_nYtmvx@shadow.com
s_KPbWLr@shadow.com
s_lQKSDY@shadow.com
s_Du0nUG@shadow.com
s_DTILiI@shadow.com
s_NzzDIB@shadow.com
s_IJ4WKW@shadow.com
s_ChO3b1@shadow.com
s_kNYvxm@shadow.com
s_UJDVhN@shadow.com
s_pPYeUT@shadow.com
s_85Kgkq@shadow.com
s_uTPfeO@shadow.com
s_nO8pMn@shadow.com
s_Jsgrmu@shadow.com
s_BXqhZU@shadow.com
s_V11qVt@shadow.com
s_dIHw1i@shadow.com
s_nh6qPY@shadow.com
s_OKKdnO@shadow.com
s_68WhjX@shadow.com
s_8FXzfc@shadow.com
s_FJSZGg@shadow.com
s_EkQYST@shadow.com
s_LbXj8I@shadow.com
s_PewDvT@shadow.com
s_1wfiXX@shadow.com
s_tIpF2Z@shadow.com
s_KifGet@shadow.com
s_RpvPdS@shadow.com
s_a30n4A@shadow.com
s_KExWNT@shadow.com
s_eGi9Rv@shadow.com
s_De6CKT@shadow.com
s_o4Fr6f@shadow.com
s_vDC1eS@shadow.com
s_pl3E7A@shadow.com
s_HttzBh@shadow.com
s_vAhvMd@shadow.com
s_w1S8S8@shadow.com
s_UcMCt1@shadow.com
s_8ZpViX@shadow.com
s_UGjdzs@shadow.com
s_17kAoA@shadow.com
s_741wfB@shadow.com
s_f5Jj7b@shadow.com
s_CJiMB7@shadow.com
s_FfHEiT@shadow.com
s_VI2Imw@shadow.com
s_UdLIDf@shadow.com
s_k77N6c@shadow.com
s_zKJf4X@shadow.com
s_dGjQK4@shadow.com
s_15Du4D@shadow.com
s_oGmTi8@shadow.com
s_65gIdM@shadow.com
s_qOqqzJ@shadow.com
s_mUZqQt@shadow.com
s_aI9Ku7@shadow.com
s_PdBQu7@shadow.com
s_UhTVV5@shadow.com
s_bzRObE@shadow.com
s_ntnHXm@shadow.com
s_w3ZZJn@shadow.com
s_uMjxpT@shadow.com
s_BxSAPH@shadow.com
s_dsJUNE@shadow.com
s_sfLGzM@shadow.com
s_Tao00Z@shadow.com
s_o92L1i@shadow.com
s_rRrUE5@shadow.com
s_x5DTTo@shadow.com
s_55RYhl@shadow.com
s_2Yteeu@shadow.com
s_06fWcj@shadow.com
s_BEa7Nm@shadow.com
s_Zli3Rp@shadow.com
s_p1PEOV@shadow.com
s_ISOl1S@shadow.com
s_gFzMYy@shadow.com
s_lRjEm4@shadow.com
s_7UkikS@shadow.com
s_ZFISK2@shadow.com
s_MaqNBa@shadow.com
s_PoKDa9@shadow.com
s_hFMaHa@shadow.com
s_hveGbT@shadow.com
s_OXrBq7@shadow.com
s_IYyGo8@shadow.com
s_BP84Ls@shadow.com
s_rexvsp@shadow.com
s_86vJCD@shadow.com
s_J2n2rf@shadow.com
s_wrajLj@shadow.com
s_p7moGs@shadow.com
s_WsbZFz@shadow.com
s_0LpqA0@shadow.com
s_DXhqau@shadow.com
s_wLJpFr@shadow.com
s_elsxmB@shadow.com
s_PbDamE@shadow.com
s_gsAitD@shadow.com
s_iAO3TP@shadow.com
s_ITbTqk@shadow.com
s_4O4rSo@shadow.com
s_f4PIek@shadow.com
s_kAafF1@shadow.com
s_UlbokC@shadow.com
s_StVyIN@shadow.com
s_Qny9jd@shadow.com
s_yLXjZH@shadow.com
s_A206OU@shadow.com
s_PA9kIT@shadow.com
s_ZCN9cu@shadow.com
s_ukKc8S@shadow.com
s_MhRJYd@shadow.com
s_aYvHUZ@shadow.com
s_sQTGcP@shadow.com
s_vqyczy@shadow.com
s_lqerll@shadow.com
s_2dIy2D@shadow.com
s_sqPBhm@shadow.com
s_rxeL87@shadow.com
s_T7Zllb@shadow.com
s_A1prFK@shadow.com
s_kqy0Ti@shadow.com
s_yRxvCl@shadow.com
s_bAf12Y@shadow.com
s_mSDYSZ@shadow.com
s_XQa53g@shadow.com
s_ifZ8Nd@shadow.com
s_JnVz5P@shadow.com
s_YQAkOm@shadow.com
s_LrWglN@shadow.com
s_20d8Bf@shadow.com
s_FOhoT3@shadow.com
s_87duIN@shadow.com
s_f7Wbt1@shadow.com
s_0rotjy@shadow.com
s_95vHay@shadow.com
s_xjCKlA@shadow.com
s_roJThz@shadow.com
s_cIoVmE@shadow.com
s_F7Kobr@shadow.com
s_Ye2HV6@shadow.com
s_Zq0esd@shadow.com
s_qtIoFr@shadow.com
s_9SUFAA@shadow.com
s_wWfMdw@shadow.com
s_fSjdY4@shadow.com
s_6COY94@shadow.com
s_Y4eWHx@shadow.com
s_oLysj9@shadow.com
s_6Sve8x@shadow.com
s_fai50t@shadow.com
s_LxFO8U@shadow.com
s_rha2wA@shadow.com
s_jb6wqN@shadow.com
s_16PqdV@shadow.com
s_I89v8b@shadow.com
s_kTP2tk@shadow.com
s_58S40h@shadow.com
s_8wiQXC@shadow.com
s_2n6PoJ@shadow.com
s_gsvhGu@shadow.com
s_njbBDy@shadow.com
s_xMzp0h@shadow.com
s_4g7YHw@shadow.com
s_cgbaLZ@shadow.com
s_8hDFBn@shadow.com
s_aOIRXe@shadow.com
s_vKHZZk@shadow.com
s_Mbo1lq@shadow.com
s_1RRDLt@shadow.com
s_YbKv8b@shadow.com
s_oLPWVd@shadow.com
s_1L5VHC@shadow.com
s_dt8GRU@shadow.com
s_q8Plkv@shadow.com
s_M4qo0a@shadow.com
s_8TOaIC@shadow.com
s_Axbdwo@shadow.com
s_DsgJyQ@shadow.com
s_KcE9m4@shadow.com
s_cxUoTi@shadow.com
s_yf5F2d@shadow.com
s_WWBin1@shadow.com
s_4hNpGq@shadow.com
s_Y0DdxY@shadow.com
s_q4AaZH@shadow.com
s_GnhyAn@shadow.com
s_ULfH0b@shadow.com
s_GUt0uv@shadow.com
s_nh7lJ6@shadow.com
s_dDXEx1@shadow.com
s_d6tSOl@shadow.com
s_ZUKIJB@shadow.com
s_YQizji@shadow.com
injection/data_sync/tasks.py
View file @ 5f58738a
......@@ -2,6 +2,10 @@
import logging
import traceback
import json
import pymysql
import threading
import random
import datetime
from celery import shared_task
from django.conf import settings
from django.core import serializers
......@@ -9,8 +13,14 @@ from trans2es.type_info import get_type_info_map
# from rpc.all import get_rpc_remote_invoker
from libs.es import ESPerform
from libs.cache import redis_client
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
from linucb.utils.register_user_tag import RegisterUserTag
from injection.data_sync.get_session import get_comments, click, login, reply
from injection.data_sync.get_session import host, user, passwd, db
exec_count_click = 0
exec_count = 0
@shared_task
def write_to_es(es_type, pk_list, use_batch_query_set=False):
......@@ -67,26 +77,161 @@ def sync_face_similar_data_to_redis():
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
@shared_task
def sync_user_similar_score():
try:
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list("user_id",flat=True)
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list(
"user_id", flat=True)
redis_key_prefix = "physical:user_similar_score:user_id:"
logging.info("begin sync user_similar_score!")
for user_id in results_items:
redis_key = redis_key_prefix + str(user_id)
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False,user_id=user_id).order_by("-score")
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False, user_id=user_id).order_by(
"-score")
item_list = list()
for item in similar_results_items:
contrast_user_id = item.contrast_user_id
score = item.score
item_list.append(
[contrast_user_id,score]
[contrast_user_id, score]
)
redis_client.set(redis_key, json.dumps(item_list))
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def auto_click(pk_list):
logging.info("get--------auto_click--------------:%s" % pk_list)
try:
now = datetime.datetime.now()
yes_time_str = now.strftime('%Y-%m-%d')
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
topic_id_list = []
if len(pk_list) > 0:
if len(pk_list) == 1:
cursor.execute(
"SELECT id FROM topic WHERE id = " + str(
pk_list[
0]) + " and user_id in (select user_id from user_extra where is_shadow = 0) and create_time LIKE '%%%%%s%%%%'" % yes_time_str)
else:
cursor.execute(
"SELECT id FROM topic WHERE id in " + str(
tuple(
pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0 ) and create_time LIKE '%%%%%s%%%%'" % yes_time_str)
data = cursor.fetchall()
topic_id_list = list(data)
logging.info("Database version : %s " % topic_id_list)
pc.close()
if topic_id_list:
try:
def fun_timer():
cookies = login()
if cookies:
logging.info("get topic_id_list:%s" % topic_id_list)
click(cookies, topic_id_list[0][0])
global timer
global exec_count_click
exec_count_click += 1
if exec_count_click == 1:
logging.info("----------2-----------")
# sleep_time = random.randint(300, 540)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_timer)
timer.start()
if exec_count_click == 2:
# sleep_time = random.randint(1000, 1900)
logging.info("----------3-----------")
sleep_time = 50
timer = threading.Timer(sleep_time, fun_timer)
timer.start()
if exec_count_click > 2:
pass
sleep_time = random.randint(300, 540)
logging.info("----------1-----------")
timer = threading.Timer(10, fun_timer) # 首次启动
timer.start()
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
else:
pass
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s" % pk_list)
exec_count = 0
try:
now = datetime.datetime.now()
yes_time_str = now.strftime('%Y-%m-%d')
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
topic_id_list = []
try:
if len(pk_list) > 0:
if len(pk_list) == 1:
cursor.execute(
"SELECT id FROM topic WHERE id = " + str( #
pk_list[
0]) + " and user_id in (select user_id from user_extra where is_shadow = 0) and create_time LIKE '%%%%%s%%%%' " % yes_time_str)
else:
cursor.execute(
"SELECT id FROM topic WHERE id in " + str(tuple( # and create_time LIKE '%%%%%s%%%%'
pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0) and and create_time LIKE '%%%%%s%%%%'"%yes_time_str)
data = cursor.fetchall()
topic_id_list = list(data)
logging.info("Database version : %s " % topic_id_list)
pc.close()
except:
pass
if topic_id_list:
try:
def fun_comment():
cookies = login()
if cookies:
comment_content = get_comments()
comment = comment_content[0]
reply(cookies, topic_id_list[0][0], comment)
global timer
global exec_count
exec_count += 1
if exec_count == 1:
sleep_time = random.randint(300, 540)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_comment)
timer.start()
if exec_count == 2:
sleep_time = random.randint(1000, 1900)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_comment)
timer.start()
sleep_time = random.randint(300, 540)
timer = threading.Timer(10, fun_comment) # 首次启动
timer.start()
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
libs/es.py
View file @ 5f58738a
......@@ -288,25 +288,40 @@ class ESPerform(object):
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {"term": {"content_level": 6}},
"weight": 6000
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {"term": {"content_level": 5}},
"weight": 5000
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {"term": {"content_level": 4}},
"weight": 4000
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
......@@ -333,7 +348,7 @@ class ESPerform(object):
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
......
majia/__init__.py deleted 100644 → 0
View file @ e11006a6
default_app_config = 'majia.apps.MaJiaApp'
majia/admin.py deleted 100644 → 0
View file @ e11006a6
from django.contrib import admin
# Register your models here.
majia/apps.py deleted 100644 → 0
View file @ e11006a6
# coding=utf-8
from __future__ import unicode_literals, print_function, absolute_import
from django.apps import AppConfig
class MaJiaApp(AppConfig):
name = 'majia'
label = 'majia'
majia/models.py deleted 100644 → 0
View file @ e11006a6
from django.db import models
# Create your models here.
majia/tasks.py deleted 100644 → 0
View file @ e11006a6
# -*- coding: UTF-8 -*-
import pymysql
import random
import threading
import traceback
import logging
from celery import shared_task
from django.conf import settings
from majia.get_session import get_comments, click, login, reply
# from majia.get_session import host, user, passwd, db
exec_count = 0
def auto_click(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_timer():
# cookies = login()
# if cookies:
# click(cookies, topic_id_list[0])
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# # sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# if exec_count == 2:
# # sleep_time = random.randint(1000, 1900)
# sleep_time = 50
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_timer) # 首次启动
# timer.start()
#
# else:
# pass
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_comment():
# cookies = login()
# if cookies:
# comment_content = get_comments()
# comment = comment_content[0]
# reply(cookies, topic_id_list[0], comment)
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# if exec_count == 2:
# sleep_time = random.randint(1000, 1900)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_comment) # 首次启动
# timer.start()
#
#
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
@shared_task
def auto_majia_effect(pk_list):
auto_click(pk_list)
auto_reply(pk_list)
majia/tests.py deleted 100644 → 0
View file @ e11006a6
from django.test import TestCase
# Create your tests here.
majia/views.py deleted 100644 → 0
View file @ e11006a6
from django.shortcuts import render
# Create your views here.
physical/celery_task_router.py
View file @ 5f58738a
......@@ -9,10 +9,7 @@ class CeleryTaskRouter(object):
queue_task_map = {
"tapir-alpha": [
'injection.data_sync.tasks.write_to_es',
],
"majia-alpha": [
'majia.tasks.auto_majia_effect',
],
]
}
# Map[TaskName, QueueName]
......
search/utils/topic.py
View file @ 5f58738a
......@@ -177,24 +177,28 @@ class TopicUtils(object):
},
{
"filter": {
"term": {
"content_level": 6
"constant_score":{
"filter":{
"term": {
"content_level": 6
}
}
}
},
"weight": 1000
"weight": 600
}
]
if len(user_similar_score_list) > 0:
for item in user_similar_score_list[:100]:
score_item = 2 + item[1]
functions_list.append(
{
"filter": {"bool": {
"should": {"term": {"user_id": item[0]}}}},
"weight": score_item,
}
)
# if len(user_similar_score_list) > 0:
# for item in user_similar_score_list[:100]:
# score_item = 2 + item[1]
# functions_list.append(
# {
# "filter": {"bool": {
# "should": {"term": {"user_id": item[0]}}}},
# "weight": score_item,
# }
# )
if len(attention_user_id_list) > 0:
functions_list.append(
......
search/views/topic.py
View file @ 5f58738a
......@@ -73,10 +73,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
user_similar_score_redis_list = json.loads(
redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
attention_tag_list = list()
recommend_topic_list = list()
......
trans2es/mapping/topic-v1.json
View file @ 5f58738a
......@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment