Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
F
ffm-baseline
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ML
ffm-baseline
Commits
b6a24939
Commit
b6a24939
authored
5 years ago
by
张彦钊
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
change test
parent
e671a9bb
master
mr/beta/bug22
offic
rtt
updatedb
zhao
zhao22
1 merge request
!36
把tidb库地址由172.16.40.158改为172.16.40.170
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
538 additions
and
52 deletions
+538
-52
topic.py
copy_doris/topic.py
+506
-0
monitor.py
monitor.py
+29
-23
rerank.py
rerank.py
+3
-29
No files found.
copy_doris/topic.py
View file @
b6a24939
@staticmethod
def
fetch_user_topic
(
device_id
,
card_type
,
size
):
try
:
def
filter_topic
(
cid_list
):
try
:
if
gmkv
.
exists
(
dislike_key
):
dislike
=
gmkv
.
smembers
(
dislike_key
)
if
len
(
cid_list
)
>
0
:
if
type
(
cid_list
[
0
])
==
int
or
type
(
cid_list
[
0
])
==
str
:
cid_list
=
[
i
for
i
in
cid_list
if
str
(
i
)
.
encode
(
'utf-8'
)
not
in
dislike
]
else
:
cid_list
=
[
i
for
i
in
cid_list
if
i
not
in
dislike
]
return
cid_list
else
:
return
cid_list
except
:
return
cid_list
def
write_after_filter_tractate
(
cid_list
):
try
:
if
gmkv
.
exists
(
after_filter_key
):
gmkv
.
set
(
after_filter_key
,
json
.
dumps
(
cid_list
))
else
:
gmkv
.
set
(
after_filter_key
,
json
.
dumps
(
cid_list
),
ex
=
6
*
60
*
60
)
except
:
logging_exception
()
logger
.
error
(
"catch exception,err_log:
%
s"
%
traceback
.
format_exc
())
def
get_filter_tractate
():
try
:
return
json
.
loads
(
gmkv
.
get
(
after_filter_key
))
except
:
return
[]
def
read_history
(
cid_list
):
if
redis_client
.
exists
(
today_key
):
redis_client
.
sadd
(
today_key
,
*
cid_list
)
else
:
redis_client
.
sadd
(
today_key
,
*
cid_list
)
redis_client
.
expire
(
today_key
,
15
*
24
*
60
*
60
)
if
redis_client
.
exists
(
read_key
)
and
redis_client
.
exists
(
old_key
):
redis_client
.
sdiffstore
(
read_key
,
read_key
,
old_key
)
redis_client
.
delete
(
old_key
)
redis_client
.
expire
(
read_key
,
time
=
13
*
24
*
60
*
60
)
redis_client
.
sadd
(
read_key
,
*
cid_list
)
def
get_gmkv
(
redis_ip
,
redis_port
,
redis_db
,
redis_password
=
""
):
try
:
if
len
(
redis_password
)
==
0
:
cli_ins
=
redis
.
Redis
(
host
=
redis_ip
,
port
=
redis_port
,
db
=
redis_db
,
socket_timeout
=
2
)
else
:
cli_ins
=
redis
.
Redis
(
host
=
redis_ip
,
port
=
redis_port
,
db
=
redis_db
,
password
=
redis_password
,
socket_timeout
=
2
)
cli_ins
.
ping
()
return
cli_ins
except
:
return
None
dislike_key
=
str
(
device_id
)
+
"_dislike_tractate"
search_topic_recommend_key
=
"TS:search_recommend_tractate_queue:device_id:"
+
str
(
device_id
)
after_filter_key
=
"device_tractate_after_filter:device_id:"
+
str
(
device_id
)
tractate_key
=
"tractate_is_tail"
+
str
(
device_id
)
read_key
=
"TS:recommend_tractate_set:device_id:"
+
str
(
device_id
)
old_key
=
"TS:recommend_tractate_set:device_id:{}:{}"
\
.
format
(
device_id
,
(
datetime
.
date
.
today
()
-
datetime
.
timedelta
(
days
=
14
))
.
strftime
(
"
%
Y-
%
m-
%
d"
))
today_key
=
"TS:recommend_tractate_set:device_id:{}:{}"
\
.
format
(
device_id
,
datetime
.
date
.
today
()
.
strftime
(
"
%
Y-
%
m-
%
d"
))
search_list
=
list
()
gmkv
=
None
for
gm_kv_host_item
in
settings
.
GM_KV_HOSTS
:
gmkv
=
get_gmkv
(
redis_ip
=
gm_kv_host_item
[
"host"
],
redis_port
=
gm_kv_host_item
[
"port"
],
redis_db
=
gm_kv_host_item
[
"db"
],
redis_password
=
gm_kv_host_item
[
"password"
])
if
gmkv
:
break
if
device_id
!=
'0'
:
if
redis_client
.
exists
(
search_topic_recommend_key
):
search_topic_recommend_dict
=
redis_client
.
hgetall
(
search_topic_recommend_key
)
search_topic_recommend_list
=
json
.
loads
(
search_topic_recommend_dict
[
b
'tractate_queue'
])
search_topic_recommend_list
=
filter_topic
(
search_topic_recommend_list
)
if
len
(
search_topic_recommend_list
)
==
0
:
redis_client
.
delete
(
search_topic_recommend_key
)
elif
len
(
search_topic_recommend_list
)
<=
2
:
search_list
=
search_topic_recommend_list
size
=
size
-
len
(
search_list
)
redis_client
.
delete
(
search_topic_recommend_key
)
else
:
search_list
=
search_topic_recommend_list
[:
2
]
size
=
size
-
2
redis_client
.
hset
(
search_topic_recommend_key
,
'tractate_queue'
,
json
.
dumps
(
search_topic_recommend_list
[
2
:]))
if
gmkv
.
exists
(
tractate_key
):
if
len
(
search_list
)
>
0
:
search_list
=
list
(
map
(
int
,
search_list
))
read_history
(
search_list
)
return
search_list
elif
gmkv
.
exists
(
after_filter_key
):
que
=
get_filter_tractate
()
que
=
filter_topic
(
que
)
if
len
(
que
)
==
0
:
gmkv
.
set
(
tractate_key
,
"tail"
,
ex
=
2
*
60
*
60
)
if
len
(
search_list
)
>
0
:
search_list
=
list
(
map
(
int
,
search_list
))
read_history
(
search_list
)
return
search_list
elif
len
(
que
)
<=
size
:
search_list
.
extend
(
que
)
gmkv
.
set
(
tractate_key
,
"tail"
,
ex
=
2
*
60
*
60
)
search_list
=
list
(
map
(
int
,
search_list
))
read_history
(
search_list
)
return
search_list
else
:
search_list
.
extend
(
que
[:
size
])
write_after_filter_tractate
(
que
[
size
:])
search_list
=
list
(
map
(
int
,
search_list
))
read_history
(
search_list
)
return
search_list
else
:
try
:
que
=
DeviceUserTopicQueue
.
objects
.
get
(
device_id
=
device_id
)
except
DeviceUserTopicQueue
.
DoesNotExist
:
que
=
UserTopicQueue
.
objects
.
last
()
if
not
que
:
if
len
(
search_list
)
>
0
:
search_list
=
list
(
map
(
int
,
search_list
))
read_history
(
search_list
)
return
search_list
qa
=
list
(
filter
(
None
,
que
.
queue
.
split
(
','
)))
qa
=
filter_topic
(
qa
)
if
len
(
qa
)
==
0
:
gmkv
.
set
(
tractate_key
,
"tail"
,
ex
=
2
*
60
*
60
)
if
len
(
search_list
)
>
0
:
search_list
=
list
(
map
(
int
,
search_list
))
read_history
(
search_list
)
return
search_list
elif
len
(
qa
)
<=
size
:
search_list
.
extend
(
qa
)
search_list
=
list
(
map
(
int
,
search_list
))
gmkv
.
set
(
tractate_key
,
"tail"
,
ex
=
2
*
60
*
60
)
read_history
(
search_list
)
return
search_list
else
:
search_list
.
extend
(
qa
[:
size
])
search_list
=
list
(
map
(
int
,
search_list
))
write_after_filter_tractate
(
qa
[
size
:])
read_history
(
search_list
)
return
search_list
else
:
key
=
'{device_id}-{card_type}-{date}'
.
format
(
device_id
=
device_id
,
card_type
=
card_type
,
date
=
RecommendFeed
.
current_date
())
try
:
que
=
DeviceUserTopicQueue
.
objects
.
get
(
device_id
=
device_id
)
except
DeviceUserTopicQueue
.
DoesNotExist
:
que
=
UserTopicQueue
.
objects
.
last
()
if
not
que
:
return
[]
que
=
list
(
filter
(
None
,
que
.
queue
.
split
(
','
)))
# adjust args.
cursor
=
redis_client
.
get
(
key
)
or
0
cursor
=
int
(
cursor
)
%
len
(
que
)
size
=
min
(
size
,
len
(
que
))
data
=
list
(
islice
(
cycle
(
que
),
cursor
,
cursor
+
size
))
data
=
list
(
map
(
int
,
data
))
if
cursor
+
2
*
size
<
len
(
que
):
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
else
:
try
:
context
.
request_logger
.
app
(
reset_queue
=
True
)
cursor
=
0
redis_client
.
set
(
key
,
cursor
,
ex
=
24
*
60
*
60
)
except
:
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
return
data
except
:
logging_exception
()
return
[]
# 帖子以前的方法
def
fetch_user_topic
(
device_id
,
card_type
,
size
):
try
:
key
=
'{device_id}-{card_type}-{date}'
.
format
(
device_id
=
device_id
,
card_type
=
card_type
,
date
=
RecommendFeed
.
current_date
())
if
(
device_id
!=
'0'
)
and
size
>=
2
:
search_topic_recommend_key
=
"TS:search_recommend_tractate_queue:device_id:"
+
str
(
device_id
)
search_topic_recommend_list
=
list
()
search_cursor_ts
=
0
if
redis_client
.
exists
(
search_topic_recommend_key
):
search_topic_recommend_dict
=
redis_client
.
hgetall
(
search_topic_recommend_key
)
if
b
'cursor'
in
search_topic_recommend_dict
:
search_cursor_ts
=
json
.
loads
(
search_topic_recommend_dict
[
b
'cursor'
])
if
search_cursor_ts
<
30
:
search_topic_recommend_list
=
json
.
loads
(
search_topic_recommend_dict
[
b
'tractate_queue'
])
if
search_cursor_ts
<
len
(
search_topic_recommend_list
):
size
=
size
-
2
try
:
que
=
DeviceUserTopicQueue
.
objects
.
get
(
device_id
=
device_id
)
except
DeviceUserTopicQueue
.
DoesNotExist
:
que
=
UserTopicQueue
.
objects
.
last
()
if
not
que
:
return
[]
que
=
list
(
filter
(
None
,
que
.
queue
.
split
(
','
)))
# adjust args.
cursor
=
redis_client
.
get
(
key
)
or
0
cursor
=
int
(
cursor
)
%
len
(
que
)
size
=
min
(
size
,
len
(
que
))
data
=
list
(
islice
(
cycle
(
que
),
cursor
,
cursor
+
size
))
data
=
list
(
map
(
int
,
data
))
if
cursor
+
2
*
size
<
len
(
que
):
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
else
:
try
:
context
.
request_logger
.
app
(
reset_queue
=
True
)
cursor
=
0
redis_client
.
set
(
key
,
cursor
,
ex
=
24
*
60
*
60
)
except
:
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
if
device_id
!=
'0'
and
size
>=
2
:
if
len
(
search_topic_recommend_list
)
>
0
and
search_cursor_ts
<
len
(
search_topic_recommend_list
):
queue
=
search_topic_recommend_list
[
search_cursor_ts
:
search_cursor_ts
+
2
]
queue
.
extend
(
data
)
data
=
queue
new_search_cursor
=
search_cursor_ts
+
2
redis_client
.
hset
(
search_topic_recommend_key
,
'cursor'
,
new_search_cursor
)
redis_client
.
expire
(
search_topic_recommend_key
,
30
*
24
*
60
*
60
)
read_topic_key
=
"TS:recommend_tractate_set:device_id:"
+
str
(
device_id
)
if
len
(
data
)
>
0
:
redis_client
.
sadd
(
read_topic_key
,
*
data
)
return
data
except
:
logging_exception
()
return
[]
# 帖子老方法
def
fetch_user_topic
(
device_id
,
card_type
,
size
):
try
:
key
=
'{device_id}-{card_type}-{date}'
.
format
(
device_id
=
device_id
,
card_type
=
card_type
,
date
=
RecommendFeed
.
current_date
())
if
(
device_id
!=
'0'
)
and
size
>=
2
:
search_topic_recommend_key
=
"TS:search_recommend_tractate_queue:device_id:"
+
str
(
device_id
)
search_topic_recommend_list
=
list
()
search_cursor_ts
=
0
if
redis_client
.
exists
(
search_topic_recommend_key
):
search_topic_recommend_dict
=
redis_client
.
hgetall
(
search_topic_recommend_key
)
if
b
'cursor'
in
search_topic_recommend_dict
:
search_cursor_ts
=
json
.
loads
(
search_topic_recommend_dict
[
b
'cursor'
])
if
search_cursor_ts
<
30
:
search_topic_recommend_list
=
json
.
loads
(
search_topic_recommend_dict
[
b
'tractate_queue'
])
if
search_cursor_ts
<
len
(
search_topic_recommend_list
):
size
=
size
-
2
try
:
que
=
DeviceUserTopicQueue
.
objects
.
get
(
device_id
=
device_id
)
except
DeviceUserTopicQueue
.
DoesNotExist
:
que
=
UserTopicQueue
.
objects
.
last
()
if
not
que
:
return
[]
que
=
list
(
filter
(
None
,
que
.
queue
.
split
(
','
)))
# adjust args.
cursor
=
redis_client
.
get
(
key
)
or
0
cursor
=
int
(
cursor
)
%
len
(
que
)
size
=
min
(
size
,
len
(
que
))
data
=
list
(
islice
(
cycle
(
que
),
cursor
,
cursor
+
size
))
data
=
list
(
map
(
int
,
data
))
if
cursor
+
2
*
size
<
len
(
que
):
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
else
:
try
:
context
.
request_logger
.
app
(
reset_queue
=
True
)
cursor
=
0
redis_client
.
set
(
key
,
cursor
,
ex
=
24
*
60
*
60
)
except
:
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
if
device_id
!=
'0'
and
size
>=
2
:
if
len
(
search_topic_recommend_list
)
>
0
and
search_cursor_ts
<
len
(
search_topic_recommend_list
):
queue
=
search_topic_recommend_list
[
search_cursor_ts
:
search_cursor_ts
+
2
]
queue
.
extend
(
data
)
data
=
queue
new_search_cursor
=
search_cursor_ts
+
2
redis_client
.
hset
(
search_topic_recommend_key
,
'cursor'
,
new_search_cursor
)
redis_client
.
expire
(
search_topic_recommend_key
,
30
*
24
*
60
*
60
)
read_topic_key
=
"TS:recommend_tractate_set:device_id:"
+
str
(
device_id
)
if
len
(
data
)
>
0
:
redis_client
.
sadd
(
read_topic_key
,
*
data
)
return
data
except
:
logging_exception
()
return
[]
#9.6线上qa
def
fetch_qa
(
device_id
,
card_type
,
size
):
try
:
def
get_after_filter_qa
():
try
:
return
json
.
loads
(
gmkv
.
get
(
after_filter_key
))
except
:
return
[]
def
write_after_filter_qa
(
cid_list
):
try
:
if
gmkv
.
exists
(
after_filter_key
):
gmkv
.
set
(
after_filter_key
,
json
.
dumps
(
cid_list
))
else
:
gmkv
.
set
(
after_filter_key
,
json
.
dumps
(
cid_list
),
ex
=
6
*
60
*
60
)
except
:
logging_exception
()
logger
.
error
(
"catch exception,err_log:
%
s"
%
traceback
.
format_exc
())
def
filter_qa
(
device_id
,
cid_list
):
try
:
key
=
str
(
device_id
)
+
"_dislike_qa"
if
gmkv
.
exists
(
key
):
dislike
=
gmkv
.
smembers
(
key
)
if
len
(
cid_list
)
>
0
:
if
type
(
cid_list
[
0
])
==
int
or
type
(
cid_list
[
0
])
==
str
:
cid_list
=
[
i
for
i
in
cid_list
if
str
(
i
)
.
encode
(
'utf-8'
)
not
in
dislike
]
else
:
cid_list
=
[
i
for
i
in
cid_list
if
i
not
in
dislike
]
return
cid_list
else
:
return
cid_list
except
:
return
cid_list
def
read_history
(
cid_list
):
if
redis_client
.
exists
(
today_qa_key
):
redis_client
.
sadd
(
today_qa_key
,
*
cid_list
)
else
:
redis_client
.
sadd
(
today_qa_key
,
*
cid_list
)
redis_client
.
expire
(
today_qa_key
,
15
*
24
*
60
*
60
)
if
redis_client
.
exists
(
read_qa_key
)
and
redis_client
.
exists
(
old_qa_key
):
redis_client
.
sdiffstore
(
read_qa_key
,
read_qa_key
,
old_qa_key
)
redis_client
.
delete
(
old_qa_key
)
redis_client
.
expire
(
read_qa_key
,
time
=
13
*
24
*
60
*
60
)
redis_client
.
sadd
(
read_qa_key
,
*
cid_list
)
def
get_gmkv
(
redis_ip
,
redis_port
,
redis_db
,
redis_password
=
""
):
try
:
if
len
(
redis_password
)
==
0
:
cli_ins
=
redis
.
Redis
(
host
=
redis_ip
,
port
=
redis_port
,
db
=
redis_db
,
socket_timeout
=
2
)
else
:
cli_ins
=
redis
.
Redis
(
host
=
redis_ip
,
port
=
redis_port
,
db
=
redis_db
,
password
=
redis_password
,
socket_timeout
=
2
)
cli_ins
.
ping
()
return
cli_ins
except
:
return
None
search_qa_recommend_list
=
list
()
read_qa_key
=
"TS:recommend_answer_set:device_id:"
+
str
(
device_id
)
old_qa_key
=
"TS:recommend_answer_set:device_id:{}:{}"
\
.
format
(
device_id
,
(
datetime
.
date
.
today
()
-
datetime
.
timedelta
(
days
=
14
))
.
strftime
(
"
%
Y-
%
m-
%
d"
))
today_qa_key
=
"TS:recommend_answer_set:device_id:{}:{}"
\
.
format
(
device_id
,
datetime
.
date
.
today
()
.
strftime
(
"
%
Y-
%
m-
%
d"
))
answer_queue_key
=
"qa_is_tail:"
+
str
(
device_id
)
after_filter_key
=
"device_qa_after_filter:device_id:"
+
str
(
device_id
)
gmkv
=
None
for
gm_kv_host_item
in
settings
.
GM_KV_HOSTS
:
gmkv
=
get_gmkv
(
redis_ip
=
gm_kv_host_item
[
"host"
],
redis_port
=
gm_kv_host_item
[
"port"
],
redis_db
=
gm_kv_host_item
[
"db"
],
redis_password
=
gm_kv_host_item
[
"password"
])
if
gmkv
:
break
if
device_id
!=
'0'
:
search_qa_recommend_key
=
"TS:search_recommend_answer_queue:device_id:"
+
str
(
device_id
)
if
redis_client
.
exists
(
search_qa_recommend_key
):
search_qa_recommend_dict
=
redis_client
.
hgetall
(
search_qa_recommend_key
)
queue_list
=
json
.
loads
(
search_qa_recommend_dict
[
b
'answer_queue'
])
queue_list
=
filter_qa
(
device_id
,
queue_list
)
if
len
(
queue_list
)
==
0
:
redis_client
.
delete
(
search_qa_recommend_key
)
elif
len
(
queue_list
)
==
1
:
size
=
size
-
1
search_qa_recommend_list
=
queue_list
redis_client
.
delete
(
search_qa_recommend_key
)
else
:
size
=
size
-
1
search_qa_recommend_list
.
append
(
queue_list
[
0
])
redis_client
.
hset
(
search_qa_recommend_key
,
"answer_queue"
,
json
.
dumps
(
queue_list
[
1
:]))
if
gmkv
.
exists
(
answer_queue_key
):
if
len
(
search_qa_recommend_list
)
>
0
:
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
elif
gmkv
.
exists
(
after_filter_key
):
que
=
get_after_filter_qa
()
que
=
filter_qa
(
device_id
,
que
)
if
len
(
que
)
==
0
:
gmkv
.
set
(
answer_queue_key
,
"tail"
,
ex
=
6
*
60
*
60
)
if
len
(
search_qa_recommend_list
)
>
0
:
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
elif
len
(
que
)
<=
size
:
search_qa_recommend_list
.
extend
(
que
)
gmkv
.
set
(
answer_queue_key
,
"tail"
,
ex
=
6
*
60
*
60
)
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
else
:
search_qa_recommend_list
.
extend
(
que
[:
size
])
write_after_filter_qa
(
que
[
size
:])
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
try
:
que
=
DeviceQAQueue
.
objects
.
get
(
device_id
=
device_id
)
except
DeviceQAQueue
.
DoesNotExist
:
que
=
AnswerQueue
.
objects
.
last
()
if
not
que
:
if
len
(
search_qa_recommend_list
)
>
0
:
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
qa
=
list
(
filter
(
None
,
que
.
queue
.
split
(
','
)))
if
device_id
!=
"0"
:
qa
=
filter_qa
(
device_id
,
qa
)
if
len
(
qa
)
==
0
:
if
device_id
!=
"0"
:
gmkv
.
set
(
answer_queue_key
,
"tail"
,
ex
=
6
*
60
*
60
)
if
len
(
search_qa_recommend_list
)
>
0
:
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
elif
len
(
qa
)
<=
size
:
search_qa_recommend_list
.
extend
(
qa
)
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
if
device_id
!=
"0"
:
gmkv
.
set
(
answer_queue_key
,
"tail"
,
ex
=
6
*
60
*
60
)
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
else
:
search_qa_recommend_list
.
extend
(
qa
[:
size
])
search_qa_recommend_list
=
list
(
map
(
int
,
search_qa_recommend_list
))
if
device_id
!=
"0"
:
write_after_filter_qa
(
qa
[
size
:])
read_history
(
search_qa_recommend_list
)
return
search_qa_recommend_list
else
:
key
=
'{device_id}-{card_type}-{date}'
.
format
(
device_id
=
device_id
,
card_type
=
card_type
,
date
=
RecommendFeed
.
current_date
())
try
:
que
=
DeviceQAQueue
.
objects
.
get
(
device_id
=
device_id
)
except
DeviceQAQueue
.
DoesNotExist
:
que
=
AnswerQueue
.
objects
.
last
()
if
not
que
:
return
[]
que
=
list
(
filter
(
None
,
que
.
queue
.
split
(
','
)))
# adjust args.
cursor
=
redis_client
.
get
(
key
)
or
0
cursor
=
int
(
cursor
)
%
len
(
que
)
size
=
min
(
size
,
len
(
que
))
# redis_client.set(key, cursor + size, ex=24 * 60 * 60)
data
=
list
(
islice
(
cycle
(
que
),
cursor
,
cursor
+
size
))
data
=
list
(
map
(
int
,
data
))
if
cursor
+
2
*
size
<
len
(
que
):
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
else
:
try
:
context
.
request_logger
.
app
(
reset_answer_queue
=
True
)
cursor
=
0
redis_client
.
set
(
key
,
cursor
,
ex
=
24
*
60
*
60
)
except
:
redis_client
.
set
(
key
,
cursor
+
size
,
ex
=
24
*
60
*
60
)
return
data
except
:
logging_exception
()
return
[]
This diff is collapsed.
Click to expand it.
monitor.py
View file @
b6a24939
...
...
@@ -211,30 +211,36 @@ def make_data(device_id,city_id,key_head):
# device_id = "868663038800476"
city_id
=
"beijing"
def
topic
():
device_id
=
"78687687"
dislike_key
=
str
(
device_id
)
+
"_dislike_tractate"
r
=
redis
.
StrictRedis
.
from_url
(
"redis://redis.paas-test.env:6379/2"
)
r
.
sadd
(
dislike_key
,
*
[
1
,
2
])
print
(
r
.
smembers
(
dislike_key
))
search
=
"TS:search_recommend_tractate_queue:device_id:"
+
str
(
device_id
)
a
=
[
1
]
a
.
extend
(
list
(
range
(
36
,
50
)))
r
.
hset
(
search
,
'tractate_queue'
,
json
.
dumps
(
a
))
print
(
r
.
hgetall
(
search
))
def
black
(
x
):
db_zhengxing
=
pymysql
.
connect
(
host
=
"172.16.30.143"
,
port
=
3306
,
user
=
"work"
,
password
=
"BJQaT9VzDcuPBqkd"
,
db
=
"zhengxing"
,
cursorclass
=
pymysql
.
cursors
.
DictCursor
)
cursor
=
db_zhengxing
.
cursor
()
date_str
=
str
(
datetime
.
datetime
.
now
())
sql
=
"REPLACE INTO hippo_deviceblacklist(device_id,create_at,update_at,pull_black_type)"
\
"values('{}','{}','{}',{})"
.
format
(
x
,
date_str
,
date_str
,
1
)
cursor
.
execute
(
sql
)
db_zhengxing
.
commit
()
db_zhengxing
.
close
()
if
__name__
==
"__main__"
:
users_list
=
list
(
range
(
1
,
90
))
n
=
3
split_users_list
=
[
users_list
[
i
:
i
+
n
]
for
i
in
range
(
0
,
len
(
users_list
),
n
)]
for
child_users_list
in
split_users_list
:
total_samples
=
list
()
for
uid_city
in
child_users_list
:
# tag_list = get_user_profile(uid_city[0])
# queues = get_queues(uid_city[0], uid_city[1])
# if len(queues) > 0 and len(tag_list) > 0:
# new_native = tag_boost(queues[0], tag_list)
# new_nearby = tag_boost(queues[1], tag_list)
#
# insert_time = str(datetime.datetime.now().strftime('%Y%m%d%H%M'))
# sample = [uid_city[0], uid_city[1], new_native, new_nearby, queues[2], queues[3], insert_time]
total_samples
.
append
(
uid_city
)
if
len
(
total_samples
)
>
0
:
df
=
pd
.
DataFrame
(
total_samples
)
df
=
df
.
rename
(
columns
=
{
0
:
"device_id"
})
print
(
"df numbers"
)
print
(
df
.
shape
[
0
])
# to_data_base(df)
black
(
"hello"
)
...
...
This diff is collapsed.
Click to expand it.
rerank.py
View file @
b6a24939
...
...
@@ -183,35 +183,9 @@ def get_all_users():
if
__name__
==
"__main__"
:
# users_list = get_esmm_users()
# print("user number")
# print(len(users_list))
users_list
=
get_all_users
()
name_tag
=
get_searchworlds_to_tagid
()
n
=
500
split_users_list
=
[
users_list
[
i
:
i
+
n
]
for
i
in
range
(
0
,
len
(
users_list
),
n
)]
for
child_users_list
in
split_users_list
:
total_samples
=
list
()
for
uid_city
in
child_users_list
:
tag_list
=
get_user_profile
(
uid_city
[
0
])
queues
=
get_queues
(
uid_city
[
0
],
uid_city
[
1
])
if
len
(
queues
)
>
0
:
new_native
=
tag_boost
(
queues
[
0
],
tag_list
)
new_nearby
=
tag_boost
(
queues
[
1
],
tag_list
)
insert_time
=
str
(
datetime
.
datetime
.
now
()
.
strftime
(
'
%
Y
%
m
%
d
%
H
%
M'
))
sample
=
[
uid_city
[
0
],
uid_city
[
1
],
new_native
,
new_nearby
,
queues
[
2
],
queues
[
3
],
insert_time
]
total_samples
.
append
(
sample
)
if
len
(
total_samples
)
>
0
:
df
=
pd
.
DataFrame
(
total_samples
)
df
=
df
.
rename
(
columns
=
{
0
:
"device_id"
,
1
:
"city_id"
,
2
:
"native_queue"
,
3
:
"nearby_queue"
,
4
:
"nation_queue"
,
5
:
"megacity_queue"
,
6
:
"time"
})
print
(
"数量"
)
print
(
df
.
shape
[
0
])
to_data_base
(
df
)
device_id
=
"868663038800476"
city_id
=
"beijing"
queues
=
get_queues
(
device_id
,
city_id
)
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment