Skip to content

C
cybertron
Commit 66ffbe75 authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parent 21f9e748
Hide whitespace changes
Inline Side-by-side
Showing with 164 additions and 629 deletions
.idea/workspace.xml
View file @ 66ffbe75
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="7b835479-ee84-40d6-9b97-0e9285d92119" name="Default Changelist" comment="" /> <list default="true" id="7b835479-ee84-40d6-9b97-0e9285d92119" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/index_contrast/pictorial.py" beforeDir="false" afterPath="$PROJECT_DIR$/index_contrast/pictorial.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/index_contrast/user.py" beforeDir="false" afterPath="$PROJECT_DIR$/index_contrast/user.py" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
...@@ -12,26 +15,26 @@ ...@@ -12,26 +15,26 @@
<usages-collector id="statistics.file.extensions.open"> <usages-collector id="statistics.file.extensions.open">
<counts> <counts>
<entry key="README" value="1" /> <entry key="README" value="1" />
<entry key="py" value="26" /> <entry key="py" value="30" />
<entry key="txt" value="3" /> <entry key="txt" value="3" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.open"> <usages-collector id="statistics.file.types.open">
<counts> <counts>
<entry key="PLAIN_TEXT" value="4" /> <entry key="PLAIN_TEXT" value="4" />
<entry key="Python" value="26" /> <entry key="Python" value="30" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.edit"> <usages-collector id="statistics.file.extensions.edit">
<counts> <counts>
<entry key="dummy" value="1" /> <entry key="dummy" value="1" />
<entry key="py" value="1503" /> <entry key="py" value="1504" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.edit"> <usages-collector id="statistics.file.types.edit">
<counts> <counts>
<entry key="PLAIN_TEXT" value="1" /> <entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="1503" /> <entry key="Python" value="1504" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.lifecycle.project"> <usages-collector id="statistics.lifecycle.project">
...@@ -44,10 +47,9 @@ ...@@ -44,10 +47,9 @@
<component name="FileEditorManager"> <component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300"> <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/auto_request.py"> <entry file="file://$PROJECT_DIR$/auto_follow.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6315"> <state>
<caret line="421" lean-forward="true" selection-start-line="421" selection-end-line="421" />
<folding> <folding>
<element signature="e#0#15#0" expanded="true" /> <element signature="e#0#15#0" expanded="true" />
</folding> </folding>
...@@ -55,18 +57,12 @@ ...@@ -55,18 +57,12 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/tag.py"> <entry file="file://$PROJECT_DIR$/auto_follow2.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="343"> <state>
<caret line="473" column="21" lean-forward="true" selection-start-line="473" selection-start-column="21" selection-end-line="473" selection-end-column="21" />
<folding> <folding>
<element signature="e#46#55#0" expanded="true" /> <element signature="e#0#15#0" expanded="true" />
<element signature="e#378#16711#0" />
<element signature="e#8854#9452#0" />
<element signature="e#9522#10831#0" />
<element signature="e#10934#13999#0" />
<element signature="e#14107#17208#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
...@@ -74,47 +70,36 @@ ...@@ -74,47 +70,36 @@
</file> </file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py"> <entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/reply_comment1.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="39"> <state relative-caret-position="45">
<caret line="5" selection-start-line="5" selection-end-line="5" selection-end-column="26" /> <caret line="13" selection-start-line="13" selection-end-line="19" selection-end-column="27" />
<folding>
<element signature="e#382#16835#0" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/reply_comment2.py"> <entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="315"> <state relative-caret-position="413">
<caret line="21" lean-forward="true" selection-start-line="21" selection-end-line="21" /> <caret line="479" column="93" selection-start-line="479" selection-start-column="93" selection-end-line="479" selection-end-column="93" />
<folding> <folding>
<element signature="e#0#16#0" expanded="true" /> <element signature="e#384#17208#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/answer_reply1.py"> <entry file="file://$PROJECT_DIR$/index_contrast/tag.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="536"> <state relative-caret-position="315">
<caret line="80" selection-start-line="80" selection-end-line="80" /> <caret line="21" selection-start-line="21" selection-end-line="448" />
<folding> <folding>
<element signature="e#0#14#0" expanded="true" /> <element signature="e#46#55#0" expanded="true" />
<element signature="e#349#16802#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
...@@ -152,6 +137,8 @@ ...@@ -152,6 +137,8 @@
<option value="$PROJECT_DIR$/reply_comment1.py" /> <option value="$PROJECT_DIR$/reply_comment1.py" />
<option value="$PROJECT_DIR$/answer_reply1.py" /> <option value="$PROJECT_DIR$/answer_reply1.py" />
<option value="$PROJECT_DIR$/index_contrast/tag.py" /> <option value="$PROJECT_DIR$/index_contrast/tag.py" />
<option value="$PROJECT_DIR$/index_contrast/pictorial.py" />
<option value="$PROJECT_DIR$/index_contrast/user.py" />
</list> </list>
</option> </option>
</component> </component>
...@@ -173,6 +160,11 @@ ...@@ -173,6 +160,11 @@
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" /> <item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" /> <item name="cybertron" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path>
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" />
<item name="index_contrast" type="462c0819:PsiDirectoryNode" />
</path>
</expand> </expand>
<select /> <select />
</subPane> </subPane>
...@@ -234,12 +226,12 @@ ...@@ -234,12 +226,12 @@
<servers /> <servers />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="0" y="23" width="1280" height="720" extended-state="1" /> <frame x="0" y="23" width="1280" height="720" extended-state="0" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info id="Structure" order="0" sideWeight="0.21188119" side_tool="true" weight="0.19112115" /> <window_info id="Structure" order="0" sideWeight="0.21188119" side_tool="true" weight="0.19112115" />
<window_info id="Favorites" order="1" side_tool="true" /> <window_info id="Favorites" order="1" side_tool="true" />
<window_info content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.23272438" /> <window_info active="true" content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.23272438" />
<window_info id="DB Browser" order="3" /> <window_info id="DB Browser" order="3" />
<window_info anchor="bottom" id="Message" order="0" /> <window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.3284974" /> <window_info anchor="bottom" id="Find" order="1" weight="0.3284974" />
...@@ -249,7 +241,7 @@ ...@@ -249,7 +241,7 @@
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" /> <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" /> <window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" /> <window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info active="true" anchor="bottom" id="Terminal" order="8" sideWeight="0.49960285" visible="true" weight="0.0" /> <window_info anchor="bottom" id="Terminal" order="8" sideWeight="0.49960285" visible="true" weight="0.0" />
<window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.50039715" side_tool="true" weight="0.32890365" /> <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.50039715" side_tool="true" weight="0.32890365" />
<window_info anchor="bottom" id="Python Console" order="10" /> <window_info anchor="bottom" id="Python Console" order="10" />
<window_info anchor="bottom" id="DB Execution Console" order="11" /> <window_info anchor="bottom" id="DB Execution Console" order="11" />
...@@ -358,8 +350,11 @@ ...@@ -358,8 +350,11 @@
</entry> </entry>
<entry file="file://$PROJECT_DIR$/auto_lunch_app2.py"> <entry file="file://$PROJECT_DIR$/auto_lunch_app2.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="630"> <state relative-caret-position="145">
<caret line="42" column="36" selection-start-line="42" selection-start-column="36" selection-end-line="42" selection-end-column="36" /> <caret line="25" column="12" lean-forward="true" selection-start-line="25" selection-start-column="12" selection-end-line="25" selection-end-column="12" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
...@@ -379,8 +374,11 @@ ...@@ -379,8 +374,11 @@
</entry> </entry>
<entry file="file://$PROJECT_DIR$/auto_follow_one.py"> <entry file="file://$PROJECT_DIR$/auto_follow_one.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="420"> <state relative-caret-position="404">
<caret line="28" selection-start-line="28" selection-end-line="28" /> <caret line="28" selection-start-line="28" selection-end-line="28" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
...@@ -461,25 +459,13 @@ ...@@ -461,25 +459,13 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/README"> <entry file="file://$PROJECT_DIR$/README">
<provider selected="true" editor-type-id="text-editor" /> <provider selected="true" editor-type-id="text-editor" />
</entry> </entry>
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/reply_comment2.py"> <entry file="file://$PROJECT_DIR$/reply_comment2.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="315"> <state relative-caret-position="315">
<caret line="21" lean-forward="true" selection-start-line="21" selection-end-line="21" /> <caret line="21" lean-forward="true" selection-start-line="21" selection-end-line="21" />
<folding>
<element signature="e#0#16#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
...@@ -510,17 +496,72 @@ ...@@ -510,17 +496,72 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/auto_comment_51.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<folding>
<element signature="e#0#11#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_every_day.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_new.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-100">
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_majia.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-69">
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/tag.py"> <entry file="file://$PROJECT_DIR$/index_contrast/tag.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="343"> <state relative-caret-position="315">
<caret line="473" column="21" lean-forward="true" selection-start-line="473" selection-start-column="21" selection-end-line="473" selection-end-column="21" /> <caret line="21" selection-start-line="21" selection-end-line="448" />
<folding> <folding>
<element signature="e#46#55#0" expanded="true" /> <element signature="e#46#55#0" expanded="true" />
<element signature="e#378#16711#0" /> <element signature="e#349#16802#0" />
<element signature="e#8854#9452#0" /> </folding>
<element signature="e#9522#10831#0" /> </state>
<element signature="e#10934#13999#0" /> </provider>
<element signature="e#14107#17208#0" /> </entry>
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45">
<caret line="13" selection-start-line="13" selection-end-line="19" selection-end-column="27" />
<folding>
<element signature="e#382#16835#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="413">
<caret line="479" column="93" selection-start-line="479" selection-start-column="93" selection-end-line="479" selection-end-column="93" />
<folding>
<element signature="e#384#17208#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
......
index_contrast/pictorial.py
View file @ 66ffbe75
...@@ -13,7 +13,7 @@ from django.conf import settings ...@@ -13,7 +13,7 @@ from django.conf import settings
ES_INFO_LIST = [ ES_INFO_LIST = [
{ {
"host": "62.234.191.183", "host": "172.21.40.14",
"port": 9200 "port": 9200
} }
] ]
...@@ -150,9 +150,14 @@ class ESPerform(object): ...@@ -150,9 +150,14 @@ class ESPerform(object):
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"): def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists: if not index_exists:
if not auto_create_index: if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name) logging.error("index:%s is not existing,bulk data error!" % official_index_name)
...@@ -163,29 +168,15 @@ class ESPerform(object): ...@@ -163,29 +168,15 @@ class ESPerform(object):
bulk_actions = [] bulk_actions = []
if sub_index_name == "topic" or \ for data in data_list:
sub_index_name == "topic-star-routing" or \ if data:
sub_index_name == "topic-high-star": bulk_actions.append({
for data in data_list: '_op_type': 'index',
if data: '_index': official_index_name,
bulk_actions.append({ '_type': doc_type,
'_op_type': 'index', '_id': data['id'],
'_index': official_index_name, '_source': data,
'_type': doc_type, })
'_id': data['id'],
'_source': data,
'routing': data["content_level"]
})
else:
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions) elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True return True
...@@ -199,8 +190,10 @@ class ESPerform(object): ...@@ -199,8 +190,10 @@ class ESPerform(object):
batch_search=False, routing=None): batch_search=False, routing=None):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = sub_index_name
# if sub_index_name != "mv-alpha-user-test-190711901":
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(official_index_name)
if not index_exists: if not index_exists:
if not auto_create_index: if not auto_create_index:
...@@ -244,13 +237,13 @@ class ESPerform(object): ...@@ -244,13 +237,13 @@ class ESPerform(object):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read") # official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists: if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name) logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body) res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -292,170 +285,6 @@ class ESPerform(object): ...@@ -292,170 +285,6 @@ class ESPerform(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True return True
@classmethod
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=size, routing="4,5,6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("topic_id_list:%s" % str(topic_id_list))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# 先获取一部分数据 # 先获取一部分数据
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
...@@ -484,7 +313,7 @@ for old_item in result_dict["hits"]: ...@@ -484,7 +313,7 @@ for old_item in result_dict["hits"]:
"is_online": True "is_online": True
} }
} }
result_dict_test = ESPerform.get_search_results(es_cli_obj, "pictorial", q, 0, 1) result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-pictorial-test-190716901", q, 0, 1)
for new_item in result_dict_test["hits"]: for new_item in result_dict_test["hits"]:
new_source = new_item["_source"] new_source = new_item["_source"]
new_id = new_source["id"] new_id = new_source["id"]
......
index_contrast/tag.py
View file @ 66ffbe75
...@@ -18,7 +18,6 @@ ES_INFO_LIST = [ ...@@ -18,7 +18,6 @@ ES_INFO_LIST = [
] ]
ES_INDEX_PREFIX = "gm-dbmw" ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object): class ESPerform(object):
cli_obj = None cli_obj = None
cli_info_list = ES_INFO_LIST cli_info_list = ES_INFO_LIST
...@@ -190,8 +189,8 @@ class ESPerform(object): ...@@ -190,8 +189,8 @@ class ESPerform(object):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = sub_index_name official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901": # if sub_index_name != "mv-alpha-user-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "read") # official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(official_index_name)
if not index_exists: if not index_exists:
...@@ -236,13 +235,13 @@ class ESPerform(object): ...@@ -236,13 +235,13 @@ class ESPerform(object):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read") # official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists: if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name) logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body) res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -284,170 +283,6 @@ class ESPerform(object): ...@@ -284,170 +283,6 @@ class ESPerform(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True return True
@classmethod
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=size, routing="4,5,6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("topic_id_list:%s" % str(topic_id_list))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# 先获取一部分数据 # 先获取一部分数据
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
......
index_contrast/user.py
View file @ 66ffbe75
...@@ -13,7 +13,7 @@ from django.conf import settings ...@@ -13,7 +13,7 @@ from django.conf import settings
ES_INFO_LIST = [ ES_INFO_LIST = [
{ {
"host": "62.234.191.183", "host": "172.21.40.14",
"port": 9200 "port": 9200
} }
] ]
...@@ -150,9 +150,14 @@ class ESPerform(object): ...@@ -150,9 +150,14 @@ class ESPerform(object):
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"): def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists: if not index_exists:
if not auto_create_index: if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name) logging.error("index:%s is not existing,bulk data error!" % official_index_name)
...@@ -163,29 +168,15 @@ class ESPerform(object): ...@@ -163,29 +168,15 @@ class ESPerform(object):
bulk_actions = [] bulk_actions = []
if sub_index_name == "topic" or \ for data in data_list:
sub_index_name == "topic-star-routing" or \ if data:
sub_index_name == "topic-high-star": bulk_actions.append({
for data in data_list: '_op_type': 'index',
if data: '_index': official_index_name,
bulk_actions.append({ '_type': doc_type,
'_op_type': 'index', '_id': data['id'],
'_index': official_index_name, '_source': data,
'_type': doc_type, })
'_id': data['id'],
'_source': data,
'routing': data["content_level"]
})
else:
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions) elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True return True
...@@ -199,8 +190,10 @@ class ESPerform(object): ...@@ -199,8 +190,10 @@ class ESPerform(object):
batch_search=False, routing=None): batch_search=False, routing=None):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = sub_index_name
# if sub_index_name != "mv-alpha-user-test-190711901":
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(official_index_name)
if not index_exists: if not index_exists:
if not auto_create_index: if not auto_create_index:
...@@ -244,13 +237,13 @@ class ESPerform(object): ...@@ -244,13 +237,13 @@ class ESPerform(object):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read") # official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists: if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name) logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body) res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -292,169 +285,6 @@ class ESPerform(object): ...@@ -292,169 +285,6 @@ class ESPerform(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True return True
@classmethod
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=size, routing="4,5,6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("topic_id_list:%s" % str(topic_id_list))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# 先获取一部分数据 # 先获取一部分数据
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
...@@ -484,7 +314,7 @@ for old_item in result_dict["hits"]: ...@@ -484,7 +314,7 @@ for old_item in result_dict["hits"]:
"is_online": True "is_online": True
} }
} }
result_dict_test = ESPerform.get_search_results(es_cli_obj, "user-test", q, 0, 1) result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-user-test-190711901", q, 0, 1)
for new_item in result_dict_test["hits"]: for new_item in result_dict_test["hits"]:
new_source = new_item["_source"] new_source = new_item["_source"]
new_id = new_source["id"] new_id = new_source["id"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment