Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
P
physical
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
alpha
physical
Commits
62523ebf
Commit
62523ebf
authored
Jul 15, 2019
by
段英荣
Browse files
Options
Browse Files
Download
Plain Diff
conflict fixed
parents
6edfff1d
afa5c366
Hide whitespace changes
Inline
Side-by-side
Showing
13 changed files
with
556 additions
and
39 deletions
+556
-39
topic.py
search/utils/topic.py
+30
-12
group.py
search/views/group.py
+69
-1
topic-high-star.json
trans2es/mapping/topic-high-star.json
+4
-0
topic-star-routing.json
trans2es/mapping/topic-star-routing.json
+14
-1
topic-star.json
trans2es/mapping/topic-star.json
+15
-1
topic-v1.json
trans2es/mapping/topic-v1.json
+15
-1
topic.json
trans2es/mapping/topic.json
+1
-0
topic.py
trans2es/models/topic.py
+13
-10
tag_transfer.py
trans2es/utils/tag_transfer.py
+14
-13
tag_transfer_BACKUP_4652.py
trans2es/utils/tag_transfer_BACKUP_4652.py
+108
-0
tag_transfer_BASE_4652.py
trans2es/utils/tag_transfer_BASE_4652.py
+87
-0
tag_transfer_LOCAL_4652.py
trans2es/utils/tag_transfer_LOCAL_4652.py
+98
-0
tag_transfer_REMOTE_4652.py
trans2es/utils/tag_transfer_REMOTE_4652.py
+88
-0
No files found.
search/utils/topic.py
View file @
62523ebf
...
...
@@ -251,9 +251,9 @@ class TopicUtils(object):
]
if
query
is
not
None
:
# 搜索帖子
multi_fields
=
{
#'description': 200,
#
'description': 200,
'content'
:
300
,
#'name': 400,
#
'name': 400,
'tag_name_list'
:
300
,
}
query_fields
=
[
'^'
.
join
((
k
,
str
(
v
)))
for
(
k
,
v
)
in
multi_fields
.
items
()]
...
...
@@ -329,18 +329,18 @@ class TopicUtils(object):
'query'
:
query
,
'type'
:
'best_fields'
,
'operator'
:
'and'
,
'fields'
:
[
"content"
,
"tag_name_list"
],
'fields'
:
[
"content"
,
"tag_name_list"
],
}
functions_list
+=
[
{
"weight"
:
300
,
"filter"
:{
"bool"
:{
"must"
:{
"filter"
:
{
"bool"
:
{
"must"
:
{
"term"
:
{
"content_level"
:
6
},
},
"minimum_should_match"
:
1
,
"should"
:[
"should"
:
[
{
'multi_match'
:
multi_match
},
{
"term"
:
{
"tag_list"
:
tag_id
}},
{
"term"
:
{
"user_nick_name_pre"
:
query
.
lower
()}}
...
...
@@ -973,7 +973,18 @@ class TopicUtils(object):
"order"
:
"desc"
},
})
if
sorts_by
==
TOPIC_SEARCH_SORT
.
TOPIC_ADD_TIME
:
sort_rule
.
append
({
"related_billboard.topic_add_createtime"
:
{
"order"
:
"desc"
,
"nested_path"
:
"related_billboard"
,
"nested_filter"
:
{
"term"
:
{
"related_billboard.pictorial_id"
:
pictorial_id
}
}
},
})
elif
isinstance
(
sorts_by
,
list
):
for
sort_by
in
sorts_by
:
if
sort_by
==
TOPIC_SEARCH_SORT
.
ID_AEC
:
...
...
@@ -1067,6 +1078,7 @@ class TopicUtils(object):
}
},
})
logging
.
info
(
"get picotirial:
%
s"
%
sort_rule
)
return
sort_rule
...
...
@@ -1084,10 +1096,16 @@ class TopicUtils(object):
}
}
if
sorts_by
:
sorts
=
cls
.
process_sort
(
sorts_by
,
pictorial_id
=
None
)
if
sorts
:
q
[
"sort"
]
=
sorts
if
'pictorial_id'
in
filters
.
keys
():
if
sorts_by
:
sorts
=
cls
.
process_sort
(
sorts_by
,
filters
[
"pictorial_id"
])
if
sorts
:
q
[
"sort"
]
=
sorts
else
:
if
sorts_by
:
sorts
=
cls
.
process_sort
(
sorts_by
,
pictorial_id
=
None
)
if
sorts
:
q
[
"sort"
]
=
sorts
result_dict
=
ESPerform
.
get_search_results
(
ESPerform
.
get_cli
(),
sub_index_name
=
index_name
,
...
...
search/views/group.py
View file @
62523ebf
...
...
@@ -296,7 +296,7 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
es_cli_obj
=
ESPerform
.
get_cli
()
result_dict
=
ESPerform
.
get_search_results
(
es_cli_obj
,
"topic"
,
q
,
offset
,
size
)
logging
.
info
(
"get pictorial_topic_sort res:
%
s"
%
result_dict
)
#
logging.info("get pictorial_topic_sort res:%s" % result_dict)
for
item
in
result_dict
[
"hits"
]:
topic_id
=
item
[
"_source"
][
"id"
]
...
...
@@ -331,3 +331,71 @@ def search_physical(query="", offset=0, size=10):
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
{
"search_pictorial_ids"
:
[]}
@bind
(
"physical/search/pictorialid_topic"
)
def
pictorial_topic_sort_peoplehot
(
pictorial_id
=-
1
,
user_id
=-
1
,
offset
=
0
,
size
=
10
):
try
:
if
not
isinstance
(
pictorial_id
,
int
):
pictorial_id
=
-
1
results
=
pictorial_topic_sort
(
pictorial_id
=
pictorial_id
,
size
=
100
)
pict_pictorial_ids_list
=
results
[
"pict_pictorial_ids_list"
]
logging
.
info
(
"get pict_pictorial_ids_list res:
%
s"
%
pict_pictorial_ids_list
)
q
=
{
"query"
:
{
"bool"
:
{
"must"
:
[
{
"term"
:
{
"is_online"
:
True
}},
{
"term"
:
{
"is_deleted"
:
False
}},
{
"term"
:
{
"pictorial_id"
:
pictorial_id
}},
{
"term"
:
{
"user_id"
:
user_id
}}
]
}
},
"sort"
:
[
{
"related_billboard.total_vote_cnt"
:
{
"order"
:
"desc"
,
"nested_path"
:
"related_billboard"
,
"missing"
:
"_last"
,
"nested_filter"
:
{
"term"
:
{
"related_billboard.pictorial_id"
:
pictorial_id
}
}
}},
{
"create_time"
:
{
"order"
:
"desc"
}}
]
}
# 获取es链接对象
es_cli_obj
=
ESPerform
.
get_cli
()
result_dict
=
ESPerform
.
get_search_results
(
es_cli_obj
,
"topic"
,
q
,
offset
,
size
)
user_pict_pictorial_ids_list
=
[]
for
item
in
result_dict
[
"hits"
]:
topic_id
=
item
[
"_source"
][
"id"
]
user_pict_pictorial_ids_list
.
append
(
topic_id
)
logging
.
info
(
"get user_pict_pictorial_ids_list res:
%
s"
%
user_pict_pictorial_ids_list
)
pictorial_user_topicid_list
=
list
()
for
user_id_topic
in
user_pict_pictorial_ids_list
:
if
user_id_topic
in
pict_pictorial_ids_list
:
topic_sort
=
pict_pictorial_ids_list
.
index
(
user_id_topic
)
pictorial_user_topicid_list
.
append
({
"topic_id"
:
user_id_topic
,
"topic_sort"
:
topic_sort
+
1
})
else
:
index
=
user_pict_pictorial_ids_list
.
index
(
user_id_topic
)
pictorial_user_topicid_list
.
append
({
"topic_id"
:
user_id_topic
,
"topic_sort"
:
100
+
index
})
return
{
"pictorial_user_topicid_list"
:
pictorial_user_topicid_list
}
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
{
"pict_pictorial_ids_list"
:
[]}
trans2es/mapping/topic-high-star.json
View file @
62523ebf
...
...
@@ -60,6 +60,7 @@
"type"
:
"nested"
,
"properties"
:{
"pictorial_id"
:{
"type"
:
"long"
},
"topic_add_createtime"
:{
"type"
:
"long"
},
"real_vote_cnt"
:{
"type"
:
"long"
},
"virt_vote_cnt"
:{
"type"
:
"long"
},
"total_vote_cnt"
:{
"type"
:
"long"
}
...
...
@@ -67,3 +68,6 @@
}
}
}
trans2es/mapping/topic-star-routing.json
View file @
62523ebf
...
...
@@ -55,6 +55,19 @@
},
"is_excellent"
:{
"type"
:
"long"
},
"is_operation_home_recommend"
:
{
"type"
:
"boolean"
},
//是否首页运营推荐
"is_history"
:
{
"type"
:
"boolean"
}
//是否历史数据
"is_history"
:
{
"type"
:
"boolean"
},
//是否历史数据
"related_billboard"
:{
"type"
:
"nested"
,
"properties"
:{
"pictorial_id"
:{
"type"
:
"long"
},
"topic_add_createtime"
:{
"type"
:
"long"
},
"real_vote_cnt"
:{
"type"
:
"long"
},
"virt_vote_cnt"
:{
"type"
:
"long"
},
"total_vote_cnt"
:{
"type"
:
"long"
}
}
}
}
}
trans2es/mapping/topic-star.json
View file @
62523ebf
...
...
@@ -17,6 +17,7 @@
"user_nick_name_pre"
:
{
"type"
:
"text"
,
"analyzer"
:
"keyword"
},
//不切词的用户名
"group_id"
:{
"type"
:
"long"
},
//所在组ID
"tag_list"
:{
"type"
:
"long"
},
//标签属性
"useful_tag_list"
:{
"type"
:
"long"
},
//有用标签属性
"edit_tag_list"
:{
"type"
:
"long"
},
//编辑标签
"tag_name_list"
:{
"type"
:
"text"
,
"analyzer"
:
"gm_default_index"
,
"search_analyzer"
:
"gm_default_index"
},
"share_num"
:{
"type"
:
"long"
},
...
...
@@ -54,6 +55,19 @@
},
"is_excellent"
:{
"type"
:
"long"
},
"is_operation_home_recommend"
:
{
"type"
:
"boolean"
},
//是否首页运营推荐
"is_history"
:
{
"type"
:
"boolean"
}
//是否历史数据
"is_history"
:
{
"type"
:
"boolean"
},
//是否历史数据
"related_billboard"
:{
"type"
:
"nested"
,
"properties"
:{
"pictorial_id"
:{
"type"
:
"long"
},
"topic_add_createtime"
:{
"type"
:
"long"
},
"real_vote_cnt"
:{
"type"
:
"long"
},
"virt_vote_cnt"
:{
"type"
:
"long"
},
"total_vote_cnt"
:{
"type"
:
"long"
}
}
}
}
}
trans2es/mapping/topic-v1.json
View file @
62523ebf
...
...
@@ -17,6 +17,7 @@
"user_nick_name_pre"
:
{
"type"
:
"text"
,
"analyzer"
:
"keyword"
},
//不切词的用户名
"group_id"
:{
"type"
:
"long"
},
//所在组ID
"tag_list"
:{
"type"
:
"long"
},
//标签属性
"useful_tag_list"
:{
"type"
:
"long"
},
//有用标签属性
"edit_tag_list"
:{
"type"
:
"long"
},
//编辑标签
"tag_name_list"
:{
"type"
:
"text"
,
"analyzer"
:
"gm_default_index"
,
"search_analyzer"
:
"gm_default_index"
},
"share_num"
:{
"type"
:
"long"
},
...
...
@@ -54,6 +55,19 @@
},
"is_excellent"
:{
"type"
:
"long"
},
"is_operation_home_recommend"
:
{
"type"
:
"boolean"
},
//是否首页运营推荐
"is_history"
:
{
"type"
:
"boolean"
}
//是否历史数据
"is_history"
:
{
"type"
:
"boolean"
},
//是否历史数据
"related_billboard"
:{
"type"
:
"nested"
,
"properties"
:{
"pictorial_id"
:{
"type"
:
"long"
},
"topic_add_createtime"
:{
"type"
:
"long"
},
"real_vote_cnt"
:{
"type"
:
"long"
},
"virt_vote_cnt"
:{
"type"
:
"long"
},
"total_vote_cnt"
:{
"type"
:
"long"
}
}
}
}
}
trans2es/mapping/topic.json
View file @
62523ebf
...
...
@@ -60,6 +60,7 @@
"type"
:
"nested"
,
"properties"
:{
"pictorial_id"
:{
"type"
:
"long"
},
"topic_add_createtime"
:{
"type"
:
"long"
},
"real_vote_cnt"
:{
"type"
:
"long"
},
"virt_vote_cnt"
:{
"type"
:
"long"
},
"total_vote_cnt"
:{
"type"
:
"long"
}
...
...
trans2es/models/topic.py
View file @
62523ebf
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from
__future__
import
unicode_literals
,
absolute_import
,
print_function
import
time
from
django.conf
import
settings
from
django.core.management.base
import
BaseCommand
,
CommandError
import
traceback
...
...
@@ -203,7 +203,7 @@ class Topic(models.Model):
tag_id_list
=
TopicTag
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
topic_id
=
self
.
id
,
is_collection
=
1
)
.
values_list
(
"tag_id"
,
flat
=
True
)
return
list
(
tag_id_list
)
return
list
(
tag_id_list
)
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
[]
...
...
@@ -282,26 +282,28 @@ class Topic(models.Model):
def
get_related_billboard
(
self
):
try
:
pictorial_ids
=
Topic
.
get_pictorial_id
(
self
)
related_billboard_list
=
list
()
for
pictorial_id
in
pictorial_ids
:
pictorials
=
PictorialTopic
.
objects
.
filter
(
topic_id
=
self
.
id
)
.
values
(
"pictorial_id"
,
"create_time"
)
for
pictorial_id
in
pictorials
:
query_result
=
TopicBillBoard
.
objects
.
filter
(
pictorial_id
=
pictorial_id
,
query_result
=
TopicBillBoard
.
objects
.
filter
(
pictorial_id
=
pictorial_id
.
get
(
'pictorial_id'
,
None
)
,
topic_id
=
self
.
id
)
.
values
()
.
first
()
if
query_result
==
None
:
related_billboard_list
.
append
({
"pictorial_id"
:
pictorial_id
,
"real_vote_cnt"
:
0
,
"virt_vote_cnt"
:
0
,
"total_vote_cnt"
:
0
})
related_billboard_list
.
append
(
{
"pictorial_id"
:
pictorial_id
.
get
(
'pictorial_id'
,
None
),
"real_vote_cnt"
:
0
,
"virt_vote_cnt"
:
0
,
"total_vote_cnt"
:
0
,
"topic_add_createtime"
:
int
(
time
.
mktime
(
pictorial_id
.
get
(
"create_time"
,
None
)
.
timetuple
()))})
else
:
total_vote_cnt
=
int
(
query_result
[
"virt_vote_cnt"
])
+
int
(
query_result
[
"real_vote_cnt"
])
related_billboard_list
.
append
(
{
"pictorial_id"
:
query_result
[
"pictorial_id"
],
"real_vote_cnt"
:
query_result
[
"real_vote_cnt"
],
"virt_vote_cnt"
:
query_result
[
"virt_vote_cnt"
],
"total_vote_cnt"
:
total_vote_cnt
})
"virt_vote_cnt"
:
query_result
[
"virt_vote_cnt"
],
"total_vote_cnt"
:
total_vote_cnt
,
"topic_add_createtime"
:
int
(
time
.
mktime
(
pictorial_id
.
get
(
"create_time"
,
None
)
.
timetuple
()))})
logging
.
info
(
"product_brand_info"
%
related_billboard_list
)
...
...
@@ -342,6 +344,7 @@ class PictorialTopic(models.Model):
topic_id
=
models
.
BigIntegerField
(
verbose_name
=
u'帖子ID'
)
is_online
=
models
.
BooleanField
(
verbose_name
=
u"是否有效"
,
default
=
True
)
is_deleted
=
models
.
BooleanField
(
verbose_name
=
u'是否删除'
)
create_time
=
models
.
DateTimeField
(
verbose_name
=
u'创建时间'
,
default
=
datetime
.
datetime
.
fromtimestamp
(
0
))
class
TopicExtra
(
models
.
Model
):
...
...
trans2es/utils/tag_transfer.py
View file @
62523ebf
...
...
@@ -8,16 +8,15 @@ import logging
import
traceback
from
libs.tools
import
tzlc
from
trans2es.models.topic
import
Topic
from
trans2es.models.tag
import
TopicTag
,
CommunityTagType
,
CommunityTagTypeRelation
from
trans2es.models.tag
import
TopicTag
,
CommunityTagType
,
CommunityTagTypeRelation
import
datetime
from
django.conf
import
settings
import
copy
class
TagTransfer
(
object
):
@classmethod
def
get_tag_name_data
(
cls
,
instance
):
def
get_tag_name_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"name"
]
=
instance
.
name
...
...
@@ -28,7 +27,7 @@ class TagTransfer(object):
return
dict
()
@classmethod
def
get_tag_data
(
cls
,
instance
):
def
get_tag_data
(
cls
,
instance
):
try
:
res
=
dict
()
...
...
@@ -37,7 +36,7 @@ class TagTransfer(object):
begin_tag_name_terms_list
=
list
()
tag_name_terms_list
=
list
()
for
i
in
range
(
len
(
instance
.
name
)):
for
j
in
range
(
i
,
len
(
instance
.
name
)
+
1
):
for
j
in
range
(
i
,
len
(
instance
.
name
)
+
1
):
name_term
=
instance
.
name
[
i
:
j
]
.
strip
()
if
name_term
:
if
i
==
0
:
...
...
@@ -59,24 +58,27 @@ class TagTransfer(object):
topic_num
=
0
res
[
"near_new_topic_num"
]
=
topic_num
if
instance
.
is_online
==
True
and
instance
.
is_deleted
==
False
:
if
instance
.
is_online
==
True
and
instance
.
is_deleted
==
False
:
topic_id_list
=
list
()
sql_result_results
=
list
(
TopicTag
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"topic_id"
,
"is_online"
))
for
topic_id
,
is_online
in
sql_result_results
:
for
topic_id
,
is_online
in
sql_result_results
:
if
is_online
:
topic_id_list
.
append
(
topic_id
)
time_base_val
=
datetime
.
datetime
.
strftime
(
datetime
.
datetime
.
now
()
+
datetime
.
timedelta
(
-
7
),
"
%
Y-
%
m-
%
d"
)
time_base_val
=
datetime
.
datetime
.
strftime
(
datetime
.
datetime
.
now
()
+
datetime
.
timedelta
(
-
7
),
"
%
Y-
%
m-
%
d"
)
for
topic_begin_index
in
range
(
0
,
len
(
topic_id_list
),
100
):
cur_topic_num
=
Topic
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
id__in
=
topic_id_list
[
topic_begin_index
:
topic_begin_index
+
100
],
create_time__gte
=
time_base_val
)
.
count
()
for
topic_begin_index
in
range
(
0
,
len
(
topic_id_list
),
100
):
cur_topic_num
=
Topic
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
id__in
=
topic_id_list
[
topic_begin_index
:
topic_begin_index
+
100
],
create_time__gte
=
time_base_val
)
.
count
()
topic_num
+=
cur_topic_num
res
[
"near_new_topic_num"
]
=
topic_num
tag_type_sql_list
=
CommunityTagTypeRelation
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"tag_type_id"
,
flat
=
True
)
tag_type_sql_list
=
CommunityTagTypeRelation
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"tag_type_id"
,
flat
=
True
)
tag_type_list
=
list
()
for
tag_type_id
in
tag_type_sql_list
:
tag_type_list
.
append
(
tag_type_id
)
...
...
@@ -94,4 +96,4 @@ class TagTransfer(object):
return
(
res
,
begin_res
)
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
(
dict
(),
dict
())
\ No newline at end of file
return
(
dict
(),
dict
())
trans2es/utils/tag_transfer_BACKUP_4652.py
0 → 100644
View file @
62523ebf
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from
__future__
import
unicode_literals
,
absolute_import
,
print_function
import
os
import
sys
import
logging
import
traceback
from
libs.tools
import
tzlc
from
trans2es.models.topic
import
Topic
from
trans2es.models.tag
import
TopicTag
,
CommunityTagType
,
CommunityTagTypeRelation
import
datetime
from
django.conf
import
settings
import
copy
class
TagTransfer
(
object
):
@classmethod
def
get_tag_name_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"name"
]
=
instance
.
name
return
res
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
dict
()
@classmethod
def
get_tag_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"id"
]
=
instance
.
id
begin_tag_name_terms_list
=
list
()
tag_name_terms_list
=
list
()
for
i
in
range
(
len
(
instance
.
name
)):
for
j
in
range
(
i
,
len
(
instance
.
name
)
+
1
):
name_term
=
instance
.
name
[
i
:
j
]
.
strip
()
if
name_term
:
if
i
==
0
:
begin_tag_name_terms_list
.
append
(
name_term
.
lower
())
else
:
tag_name_terms_list
.
append
(
name_term
.
lower
())
res
[
"suggest"
]
=
{
<<<<<<<
HEAD
"input"
:
tag_name_terms_list
,
"weight"
:
1
,
"contexts"
:{
=======
"input"
:
tag_name_terms_list
,
"contexts"
:
{
>>>>>>>
afa5c366e2b0bbba18f896f1f32b8d57b63c6bee
"is_online"
:
[
instance
.
is_online
],
"is_deleted"
:
[
instance
.
is_deleted
]
}
}
res
[
"name"
]
=
instance
.
name
res
[
"is_online"
]
=
instance
.
is_online
res
[
"is_deleted"
]
=
instance
.
is_deleted
topic_num
=
0
res
[
"near_new_topic_num"
]
=
topic_num
if
instance
.
is_online
==
True
and
instance
.
is_deleted
==
False
:
topic_id_list
=
list
()
sql_result_results
=
list
(
TopicTag
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"topic_id"
,
"is_online"
))
for
topic_id
,
is_online
in
sql_result_results
:
if
is_online
:
topic_id_list
.
append
(
topic_id
)
time_base_val
=
datetime
.
datetime
.
strftime
(
datetime
.
datetime
.
now
()
+
datetime
.
timedelta
(
-
7
),
"
%
Y-
%
m-
%
d"
)
for
topic_begin_index
in
range
(
0
,
len
(
topic_id_list
),
100
):
cur_topic_num
=
Topic
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
id__in
=
topic_id_list
[
topic_begin_index
:
topic_begin_index
+
100
],
create_time__gte
=
time_base_val
)
.
count
()
topic_num
+=
cur_topic_num
res
[
"near_new_topic_num"
]
=
topic_num
tag_type_sql_list
=
CommunityTagTypeRelation
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"tag_type_id"
,
flat
=
True
)
tag_type_list
=
list
()
for
tag_type_id
in
tag_type_sql_list
:
tag_type_list
.
append
(
tag_type_id
)
res
[
"tag_type"
]
=
tag_type_list
res
[
"collection"
]
=
instance
.
collection
res
[
"is_ai"
]
=
instance
.
is_ai
res
[
"is_own"
]
=
instance
.
is_own
begin_res
=
copy
.
deepcopy
(
res
)
begin_res
[
"id"
]
=
str
(
begin_res
[
"id"
])
+
"_begin"
begin_res
[
"suggest"
][
"input"
]
=
begin_tag_name_terms_list
begin_res
[
"suggest"
][
"weight"
]
=
10
return
(
res
,
begin_res
)
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
<<<<<<<
HEAD
return
(
dict
(),
dict
())
=======
return
dict
()
>>>>>>>
afa5c366e2b0bbba18f896f1f32b8d57b63c6bee
trans2es/utils/tag_transfer_BASE_4652.py
0 → 100644
View file @
62523ebf
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from
__future__
import
unicode_literals
,
absolute_import
,
print_function
import
os
import
sys
import
logging
import
traceback
from
libs.tools
import
tzlc
from
trans2es.models.topic
import
Topic
from
trans2es.models.tag
import
TopicTag
,
CommunityTagType
,
CommunityTagTypeRelation
import
datetime
from
django.conf
import
settings
class
TagTransfer
(
object
):
@classmethod
def
get_tag_name_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"name"
]
=
instance
.
name
return
res
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
dict
()
@classmethod
def
get_tag_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"id"
]
=
instance
.
id
tag_name_terms_list
=
list
()
for
i
in
range
(
len
(
instance
.
name
)):
for
j
in
range
(
i
,
len
(
instance
.
name
)
+
1
):
name_term
=
instance
.
name
[
i
:
j
]
.
strip
()
if
name_term
:
tag_name_terms_list
.
append
(
name_term
.
lower
())
res
[
"suggest"
]
=
{
"input"
:
tag_name_terms_list
,
"contexts"
:{
"is_online"
:
[
instance
.
is_online
],
"is_deleted"
:
[
instance
.
is_deleted
]
}
}
res
[
"name"
]
=
instance
.
name
res
[
"is_online"
]
=
instance
.
is_online
res
[
"is_deleted"
]
=
instance
.
is_deleted
topic_num
=
0
res
[
"near_new_topic_num"
]
=
topic_num
if
instance
.
is_online
==
True
and
instance
.
is_deleted
==
False
:
topic_id_list
=
list
()
sql_result_results
=
list
(
TopicTag
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"topic_id"
,
"is_online"
))
for
topic_id
,
is_online
in
sql_result_results
:
if
is_online
:
topic_id_list
.
append
(
topic_id
)
time_base_val
=
datetime
.
datetime
.
strftime
(
datetime
.
datetime
.
now
()
+
datetime
.
timedelta
(
-
7
),
"
%
Y-
%
m-
%
d"
)
for
topic_begin_index
in
range
(
0
,
len
(
topic_id_list
),
100
):
cur_topic_num
=
Topic
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
id__in
=
topic_id_list
[
topic_begin_index
:
topic_begin_index
+
100
],
create_time__gte
=
time_base_val
)
.
count
()
topic_num
+=
cur_topic_num
res
[
"near_new_topic_num"
]
=
topic_num
tag_type_sql_list
=
CommunityTagTypeRelation
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"tag_type_id"
,
flat
=
True
)
tag_type_list
=
list
()
for
tag_type_id
in
tag_type_sql_list
:
tag_type_list
.
append
(
tag_type_id
)
res
[
"tag_type"
]
=
tag_type_list
res
[
"collection"
]
=
instance
.
collection
res
[
"is_ai"
]
=
instance
.
is_ai
res
[
"is_own"
]
=
instance
.
is_own
return
res
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
dict
()
\ No newline at end of file
trans2es/utils/tag_transfer_LOCAL_4652.py
0 → 100644
View file @
62523ebf
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from
__future__
import
unicode_literals
,
absolute_import
,
print_function
import
os
import
sys
import
logging
import
traceback
from
libs.tools
import
tzlc
from
trans2es.models.topic
import
Topic
from
trans2es.models.tag
import
TopicTag
,
CommunityTagType
,
CommunityTagTypeRelation
import
datetime
from
django.conf
import
settings
import
copy
class
TagTransfer
(
object
):
@classmethod
def
get_tag_name_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"name"
]
=
instance
.
name
return
res
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
dict
()
@classmethod
def
get_tag_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"id"
]
=
instance
.
id
begin_tag_name_terms_list
=
list
()
tag_name_terms_list
=
list
()
for
i
in
range
(
len
(
instance
.
name
)):
for
j
in
range
(
i
,
len
(
instance
.
name
)
+
1
):
name_term
=
instance
.
name
[
i
:
j
]
.
strip
()
if
name_term
:
if
i
==
0
:
begin_tag_name_terms_list
.
append
(
name_term
.
lower
())
else
:
tag_name_terms_list
.
append
(
name_term
.
lower
())
res
[
"suggest"
]
=
{
"input"
:
tag_name_terms_list
,
"weight"
:
1
,
"contexts"
:{
"is_online"
:
[
instance
.
is_online
],
"is_deleted"
:
[
instance
.
is_deleted
]
}
}
res
[
"name"
]
=
instance
.
name
res
[
"is_online"
]
=
instance
.
is_online
res
[
"is_deleted"
]
=
instance
.
is_deleted
topic_num
=
0
res
[
"near_new_topic_num"
]
=
topic_num
if
instance
.
is_online
==
True
and
instance
.
is_deleted
==
False
:
topic_id_list
=
list
()
sql_result_results
=
list
(
TopicTag
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"topic_id"
,
"is_online"
))
for
topic_id
,
is_online
in
sql_result_results
:
if
is_online
:
topic_id_list
.
append
(
topic_id
)
time_base_val
=
datetime
.
datetime
.
strftime
(
datetime
.
datetime
.
now
()
+
datetime
.
timedelta
(
-
7
),
"
%
Y-
%
m-
%
d"
)
for
topic_begin_index
in
range
(
0
,
len
(
topic_id_list
),
100
):
cur_topic_num
=
Topic
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
id__in
=
topic_id_list
[
topic_begin_index
:
topic_begin_index
+
100
],
create_time__gte
=
time_base_val
)
.
count
()
topic_num
+=
cur_topic_num
res
[
"near_new_topic_num"
]
=
topic_num
tag_type_sql_list
=
CommunityTagTypeRelation
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"tag_type_id"
,
flat
=
True
)
tag_type_list
=
list
()
for
tag_type_id
in
tag_type_sql_list
:
tag_type_list
.
append
(
tag_type_id
)
res
[
"tag_type"
]
=
tag_type_list
res
[
"collection"
]
=
instance
.
collection
res
[
"is_ai"
]
=
instance
.
is_ai
res
[
"is_own"
]
=
instance
.
is_own
begin_res
=
copy
.
deepcopy
(
res
)
begin_res
[
"id"
]
=
str
(
begin_res
[
"id"
])
+
"_begin"
begin_res
[
"suggest"
][
"input"
]
=
begin_tag_name_terms_list
begin_res
[
"suggest"
][
"weight"
]
=
10
return
(
res
,
begin_res
)
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
(
dict
(),
dict
())
\ No newline at end of file
trans2es/utils/tag_transfer_REMOTE_4652.py
0 → 100644
View file @
62523ebf
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from
__future__
import
unicode_literals
,
absolute_import
,
print_function
import
os
import
sys
import
logging
import
traceback
from
libs.tools
import
tzlc
from
trans2es.models.topic
import
Topic
from
trans2es.models.tag
import
TopicTag
,
CommunityTagType
,
CommunityTagTypeRelation
import
datetime
from
django.conf
import
settings
class
TagTransfer
(
object
):
@classmethod
def
get_tag_name_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"name"
]
=
instance
.
name
return
res
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
dict
()
@classmethod
def
get_tag_data
(
cls
,
instance
):
try
:
res
=
dict
()
res
[
"id"
]
=
instance
.
id
tag_name_terms_list
=
list
()
for
i
in
range
(
len
(
instance
.
name
)):
for
j
in
range
(
i
,
len
(
instance
.
name
)
+
1
):
name_term
=
instance
.
name
[
i
:
j
]
.
strip
()
if
name_term
:
tag_name_terms_list
.
append
(
name_term
.
lower
())
res
[
"suggest"
]
=
{
"input"
:
tag_name_terms_list
,
"contexts"
:
{
"is_online"
:
[
instance
.
is_online
],
"is_deleted"
:
[
instance
.
is_deleted
]
}
}
res
[
"name"
]
=
instance
.
name
res
[
"is_online"
]
=
instance
.
is_online
res
[
"is_deleted"
]
=
instance
.
is_deleted
topic_num
=
0
res
[
"near_new_topic_num"
]
=
topic_num
if
instance
.
is_online
==
True
and
instance
.
is_deleted
==
False
:
topic_id_list
=
list
()
sql_result_results
=
list
(
TopicTag
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"topic_id"
,
"is_online"
))
for
topic_id
,
is_online
in
sql_result_results
:
if
is_online
:
topic_id_list
.
append
(
topic_id
)
time_base_val
=
datetime
.
datetime
.
strftime
(
datetime
.
datetime
.
now
()
+
datetime
.
timedelta
(
-
7
),
"
%
Y-
%
m-
%
d"
)
for
topic_begin_index
in
range
(
0
,
len
(
topic_id_list
),
100
):
cur_topic_num
=
Topic
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
id__in
=
topic_id_list
[
topic_begin_index
:
topic_begin_index
+
100
],
create_time__gte
=
time_base_val
)
.
count
()
topic_num
+=
cur_topic_num
res
[
"near_new_topic_num"
]
=
topic_num
tag_type_sql_list
=
CommunityTagTypeRelation
.
objects
.
using
(
settings
.
SLAVE_DB_NAME
)
.
filter
(
tag_id
=
instance
.
id
)
.
values_list
(
"tag_type_id"
,
flat
=
True
)
tag_type_list
=
list
()
for
tag_type_id
in
tag_type_sql_list
:
tag_type_list
.
append
(
tag_type_id
)
res
[
"tag_type"
]
=
tag_type_list
res
[
"collection"
]
=
instance
.
collection
res
[
"is_ai"
]
=
instance
.
is_ai
res
[
"is_own"
]
=
instance
.
is_own
return
res
except
:
logging
.
error
(
"catch exception,err_msg:
%
s"
%
traceback
.
format_exc
())
return
dict
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment