Skip to content

C
cybertron
Commit 66ffbe75 authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parent 21f9e748
Hide whitespace changes
Inline Side-by-side
Showing with 164 additions and 629 deletions
.idea/workspace.xml
View file @ 66ffbe75
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="7b835479-ee84-40d6-9b97-0e9285d92119" name="Default Changelist" comment="" />
<list default="true" id="7b835479-ee84-40d6-9b97-0e9285d92119" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/index_contrast/pictorial.py" beforeDir="false" afterPath="$PROJECT_DIR$/index_contrast/pictorial.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/index_contrast/user.py" beforeDir="false" afterPath="$PROJECT_DIR$/index_contrast/user.py" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
......@@ -12,26 +15,26 @@
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="README" value="1" />
<entry key="py" value="26" />
<entry key="py" value="30" />
<entry key="txt" value="3" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="PLAIN_TEXT" value="4" />
<entry key="Python" value="26" />
<entry key="Python" value="30" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="dummy" value="1" />
<entry key="py" value="1503" />
<entry key="py" value="1504" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="1503" />
<entry key="Python" value="1504" />
</counts>
</usages-collector>
<usages-collector id="statistics.lifecycle.project">
......@@ -44,10 +47,9 @@
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/auto_request.py">
<entry file="file://$PROJECT_DIR$/auto_follow.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6315">
<caret line="421" lean-forward="true" selection-start-line="421" selection-end-line="421" />
<state>
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
......@@ -55,18 +57,12 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/index_contrast/tag.py">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/auto_follow2.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="343">
<caret line="473" column="21" lean-forward="true" selection-start-line="473" selection-start-column="21" selection-end-line="473" selection-end-column="21" />
<state>
<folding>
<element signature="e#46#55#0" expanded="true" />
<element signature="e#378#16711#0" />
<element signature="e#8854#9452#0" />
<element signature="e#9522#10831#0" />
<element signature="e#10934#13999#0" />
<element signature="e#14107#17208#0" />
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
......@@ -74,47 +70,36 @@
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/reply_comment1.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="39">
<caret line="5" selection-start-line="5" selection-end-line="5" selection-end-column="26" />
<state relative-caret-position="45">
<caret line="13" selection-start-line="13" selection-end-line="19" selection-end-column="27" />
<folding>
<element signature="e#382#16835#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/reply_comment2.py">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="315">
<caret line="21" lean-forward="true" selection-start-line="21" selection-end-line="21" />
<state relative-caret-position="413">
<caret line="479" column="93" selection-start-line="479" selection-start-column="93" selection-end-line="479" selection-end-column="93" />
<folding>
<element signature="e#0#16#0" expanded="true" />
<element signature="e#384#17208#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/answer_reply1.py">
<entry file="file://$PROJECT_DIR$/index_contrast/tag.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="536">
<caret line="80" selection-start-line="80" selection-end-line="80" />
<state relative-caret-position="315">
<caret line="21" selection-start-line="21" selection-end-line="448" />
<folding>
<element signature="e#0#14#0" expanded="true" />
<element signature="e#46#55#0" expanded="true" />
<element signature="e#349#16802#0" />
</folding>
</state>
</provider>
......@@ -152,6 +137,8 @@
<option value="$PROJECT_DIR$/reply_comment1.py" />
<option value="$PROJECT_DIR$/answer_reply1.py" />
<option value="$PROJECT_DIR$/index_contrast/tag.py" />
<option value="$PROJECT_DIR$/index_contrast/pictorial.py" />
<option value="$PROJECT_DIR$/index_contrast/user.py" />
</list>
</option>
</component>
......@@ -173,6 +160,11 @@
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" />
<item name="index_contrast" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
......@@ -234,12 +226,12 @@
<servers />
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="1280" height="720" extended-state="1" />
<frame x="0" y="23" width="1280" height="720" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Structure" order="0" sideWeight="0.21188119" side_tool="true" weight="0.19112115" />
<window_info id="Favorites" order="1" side_tool="true" />
<window_info content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.23272438" />
<window_info active="true" content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.23272438" />
<window_info id="DB Browser" order="3" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.3284974" />
......@@ -249,7 +241,7 @@
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info active="true" anchor="bottom" id="Terminal" order="8" sideWeight="0.49960285" visible="true" weight="0.0" />
<window_info anchor="bottom" id="Terminal" order="8" sideWeight="0.49960285" visible="true" weight="0.0" />
<window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.50039715" side_tool="true" weight="0.32890365" />
<window_info anchor="bottom" id="Python Console" order="10" />
<window_info anchor="bottom" id="DB Execution Console" order="11" />
......@@ -358,8 +350,11 @@
</entry>
<entry file="file://$PROJECT_DIR$/auto_lunch_app2.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="630">
<caret line="42" column="36" selection-start-line="42" selection-start-column="36" selection-end-line="42" selection-end-column="36" />
<state relative-caret-position="145">
<caret line="25" column="12" lean-forward="true" selection-start-line="25" selection-start-column="12" selection-end-line="25" selection-end-column="12" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
......@@ -379,8 +374,11 @@
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_one.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="420">
<state relative-caret-position="404">
<caret line="28" selection-start-line="28" selection-end-line="28" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
......@@ -461,25 +459,13 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/README">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/reply_comment2.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="315">
<caret line="21" lean-forward="true" selection-start-line="21" selection-end-line="21" />
<folding>
<element signature="e#0#16#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
......@@ -510,17 +496,72 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_comment_51.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<folding>
<element signature="e#0#11#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_every_day.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_new.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-100">
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_majia.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-69">
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/tag.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="343">
<caret line="473" column="21" lean-forward="true" selection-start-line="473" selection-start-column="21" selection-end-line="473" selection-end-column="21" />
<state relative-caret-position="315">
<caret line="21" selection-start-line="21" selection-end-line="448" />
<folding>
<element signature="e#46#55#0" expanded="true" />
<element signature="e#378#16711#0" />
<element signature="e#8854#9452#0" />
<element signature="e#9522#10831#0" />
<element signature="e#10934#13999#0" />
<element signature="e#14107#17208#0" />
<element signature="e#349#16802#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45">
<caret line="13" selection-start-line="13" selection-end-line="19" selection-end-column="27" />
<folding>
<element signature="e#382#16835#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="413">
<caret line="479" column="93" selection-start-line="479" selection-start-column="93" selection-end-line="479" selection-end-column="93" />
<folding>
<element signature="e#384#17208#0" />
</folding>
</state>
</provider>
......
index_contrast/pictorial.py
View file @ 66ffbe75
......@@ -13,7 +13,7 @@ from django.conf import settings
ES_INFO_LIST = [
{
"host": "62.234.191.183",
"host": "172.21.40.14",
"port": 9200
}
]
......@@ -150,9 +150,14 @@ class ESPerform(object):
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
......@@ -163,29 +168,15 @@ class ESPerform(object):
bulk_actions = []
if sub_index_name == "topic" or \
sub_index_name == "topic-star-routing" or \
sub_index_name == "topic-high-star":
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
'routing': data["content_level"]
})
else:
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
......@@ -199,8 +190,10 @@ class ESPerform(object):
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
# if sub_index_name != "mv-alpha-user-test-190711901":
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
......@@ -244,13 +237,13 @@ class ESPerform(object):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body)
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -292,170 +285,6 @@ class ESPerform(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
@classmethod
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=size, routing="4,5,6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("topic_id_list:%s" % str(topic_id_list))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
......@@ -484,7 +313,7 @@ for old_item in result_dict["hits"]:
"is_online": True
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "pictorial", q, 0, 1)
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-pictorial-test-190716901", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
......
index_contrast/tag.py
View file @ 66ffbe75
......@@ -18,7 +18,6 @@ ES_INFO_LIST = [
]
ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object):
cli_obj = None
cli_info_list = ES_INFO_LIST
......@@ -190,8 +189,8 @@ class ESPerform(object):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "read")
# if sub_index_name != "mv-alpha-user-test-190711901":
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
......@@ -236,13 +235,13 @@ class ESPerform(object):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body)
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -284,170 +283,6 @@ class ESPerform(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
@classmethod
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=size, routing="4,5,6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("topic_id_list:%s" % str(topic_id_list))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
......
index_contrast/user.py
View file @ 66ffbe75
......@@ -13,7 +13,7 @@ from django.conf import settings
ES_INFO_LIST = [
{
"host": "62.234.191.183",
"host": "172.21.40.14",
"port": 9200
}
]
......@@ -150,9 +150,14 @@ class ESPerform(object):
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
......@@ -163,29 +168,15 @@ class ESPerform(object):
bulk_actions = []
if sub_index_name == "topic" or \
sub_index_name == "topic-star-routing" or \
sub_index_name == "topic-high-star":
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
'routing': data["content_level"]
})
else:
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
......@@ -199,8 +190,10 @@ class ESPerform(object):
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
# if sub_index_name != "mv-alpha-user-test-190711901":
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
......@@ -244,13 +237,13 @@ class ESPerform(object):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body)
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -292,169 +285,6 @@ class ESPerform(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
@classmethod
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=size, routing="4,5,6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("topic_id_list:%s" % str(topic_id_list))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
......@@ -484,7 +314,7 @@ for old_item in result_dict["hits"]:
"is_online": True
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "user-test", q, 0, 1)
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-user-test-190711901", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment