Skip to content

P
physical
Commit 7940fdf4 authored by lixiaofang's avatar lixiaofang
Browse files
Options

update

parents 8f76ef94 338c6f92
Show whitespace changes
Inline Side-by-side
Showing with 918 additions and 36 deletions
.DS_Store 0 → 100644
View file @ 7940fdf4
File added
.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ 7940fdf4
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="gm-config==0.1.3" />
</list>
</value>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/workspace.xml 0 → 100644
View file @ 7940fdf4
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="dbf86a46-510b-4304-905e-cf96c87ff27d" name="Default Changelist" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FUSProjectUsageTrigger">
<session id="-127591351">
<usages-collector id="statistics.lifecycle.project">
<counts>
<entry key="project.closed" value="1" />
<entry key="project.open.time.0" value="1" />
<entry key="project.open.time.1" value="1" />
<entry key="project.open.time.5" value="1" />
<entry key="project.opened" value="3" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="json" value="11" />
<entry key="py" value="72" />
<entry key="txt" value="1" />
<entry key="xml" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="JSON" value="11" />
<entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="72" />
<entry key="XML" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="dummy" value="9" />
<entry key="json" value="100" />
<entry key="py" value="545" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="JSON" value="100" />
<entry key="PLAIN_TEXT" value="9" />
<entry key="Python" value="545" />
</counts>
</usages-collector>
</session>
</component>
<component name="FileEditorManager">
<leaf>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/physical/settings.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="116">
<caret line="133" column="36" selection-start-line="133" selection-end-line="134" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="385">
<caret line="78" column="43" selection-start-line="78" selection-start-column="43" selection-end-line="78" selection-end-column="43" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/trans2es/type_info.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="257">
<caret line="303" column="34" selection-start-line="303" selection-start-column="18" selection-end-line="303" selection-end-column="34" />
<folding>
<element signature="e#15#87#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/mapping/contrast_similar.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="9" column="26" selection-start-line="9" selection-start-column="5" selection-end-line="9" selection-end-column="26" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/search/views/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230">
<caret line="116" lean-forward="true" selection-start-line="93" selection-end-line="116" />
<folding>
<element signature="e#47#119#0" expanded="true" />
<element signature="e#418#3802#0" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>content_level</find>
<find>FaceUserContrastSimilar</find>
<find>mappin</find>
<find>topic</find>
<find>__load_mapping</find>
<find>handle</find>
<find>face_user_contrast_similar</find>
<find>alpha</find>
</findStrings>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/search/utils/topic.py" />
<option value="$PROJECT_DIR$/trans2es/models/topic.py" />
<option value="$PROJECT_DIR$/trans2es/mapping/contrast_similar.json" />
<option value="$PROJECT_DIR$/trans2es/models/face_user_contrast_similar_three.py" />
<option value="$PROJECT_DIR$/trans2es/management/commands/trans2es_mapping2es.py" />
<option value="$PROJECT_DIR$/libs/es.py" />
<option value="$PROJECT_DIR$/trans2es/utils/contrast_similar_transfer.py" />
<option value="$PROJECT_DIR$/trans2es/type_info.py" />
<option value="$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py" />
<option value="$PROJECT_DIR$/trans2es/models/contrast_similar.py" />
<option value="$PROJECT_DIR$/physical/settings.py" />
<option value="$PROJECT_DIR$/search/views/contrast_similar.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="y" value="23" />
<option name="width" value="1280" />
<option name="height" value="735" />
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
<profile-state>
<expanded-state>
<State />
<State>
<id>General</id>
</State>
</expanded-state>
<selected-state>
<State>
<id>Buildout</id>
</State>
</selected-state>
</profile-state>
</entry>
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="search" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="search" type="462c0819:PsiDirectoryNode" />
<item name="views" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
<item name="mapping" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
</panes>
</component>
<component name="PropertiesComponent">
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="RecentsManager">
<key name="MoveFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/trans2es/models" />
<recent name="$PROJECT_DIR$/trans2es/utils" />
<recent name="$PROJECT_DIR$/trans2es/mapping" />
</key>
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/trans2es/mapping" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="dbf86a46-510b-4304-905e-cf96c87ff27d" name="Default Changelist" comment="" />
<created>1547796386098</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1547796386098</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="1280" height="736" extended-state="6" />
<editor active="true" />
<layout>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.28911835" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info id="DB Browser" order="3" weight="0.32962668" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.3268921" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" weight="0.017770598" />
<window_info anchor="bottom" id="DB Execution Console" order="8" />
<window_info anchor="bottom" id="Terminal" order="9" visible="true" weight="0.093397744" />
<window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
<window_info anchor="bottom" id="Python Console" order="11" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
<layout-to-restore>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.2494043" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info id="DB Browser" order="3" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.3268921" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" />
<window_info anchor="bottom" id="DB Execution Console" order="8" />
<window_info anchor="bottom" id="Terminal" order="9" visible="true" weight="0.31239936" />
<window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
<window_info anchor="bottom" id="Python Console" order="11" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout-to-restore>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/trans2es/management/commands/trans2es_mapping2es.py</url>
<line>31</line>
<option name="timeStamp" value="1" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/search/utils/pick.py</url>
<line>49</line>
<option name="timeStamp" value="2" />
</line-breakpoint>
</breakpoints>
</breakpoint-manager>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/requirements.txt">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/manage.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/app_conf.xml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/wsgi.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/views.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/celery.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/celery_task_router.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/django_init.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/log_settings.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/urls.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/libs/cache.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<caret line="6" selection-start-line="6" selection-end-line="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/utils/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="310">
<caret line="134" column="21" selection-start-line="134" selection-start-column="20" selection-end-line="134" selection-end-column="21" />
<folding>
<element signature="e#278#958#0" />
<element signature="e#1969#2175#0" />
<element signature="e#8364#10837#0" />
<element signature="e#10909#11686#0" />
<element signature="e#11789#15127#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/tag_transfer.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/topic.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" column="34" selection-start-line="19" selection-start-column="28" selection-end-line="19" selection-end-column="34" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/injection/data_sync/tasks.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="321">
<caret line="44" lean-forward="true" selection-start-line="44" selection-end-line="45" />
</state>
</provider>
</entry>
<entry file="file:///Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/logging/__init__.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="166">
<caret line="1515" column="25" selection-start-line="1515" selection-start-column="25" selection-end-line="1515" selection-end-column="25" />
</state>
</provider>
</entry>
<entry file="file:///Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/test/test_winreg.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="166">
<caret line="247" column="31" selection-start-line="247" selection-start-column="31" selection-end-line="247" selection-end-column="31" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/topic_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-270">
<caret line="11" lean-forward="true" selection-start-line="11" selection-end-line="11" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/utils/pick.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<caret line="10" column="24" selection-start-line="10" selection-start-column="24" selection-end-line="10" selection-end-column="24" />
<folding>
<element signature="e#204#5735#0" />
<element signature="e#318#953#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/_celery.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-118" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/management/commands/trans2es_data2es_parallel.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="32" lean-forward="true" selection-start-line="32" selection-end-line="32" />
<folding>
<element signature="e#1486#3018#0" />
<element signature="e#3207#4010#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/pick_celebrity_transfer.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file:///Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/optparse.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="472">
<caret line="1680" selection-start-line="1680" selection-end-line="1680" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/venv/lib/python3.6/site-packages/django/core/management/base.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="5173">
<caret line="440" column="26" selection-start-line="440" selection-start-column="26" selection-end-line="440" selection-end-column="26" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="235">
<caret line="20" column="40" selection-start-line="20" selection-start-column="18" selection-end-line="20" selection-end-column="40" />
<folding>
<element signature="e#47#79#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="60">
<caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/management/commands/trans2es_mapping2es.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="31" column="14" selection-start-line="31" selection-start-column="8" selection-end-line="31" selection-end-column="14" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/contrast_similar_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-142">
<caret column="21" lean-forward="true" selection-start-column="21" selection-end-column="21" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="101">
<caret line="17" column="46" selection-start-line="17" selection-start-column="20" selection-end-line="17" selection-end-column="46" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/libs/es.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="227">
<caret line="27" lean-forward="true" selection-start-line="27" selection-end-line="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230">
<caret line="116" lean-forward="true" selection-start-line="93" selection-end-line="116" />
<folding>
<element signature="e#47#119#0" expanded="true" />
<element signature="e#418#3802#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/physical/settings.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="116">
<caret line="133" column="36" selection-start-line="133" selection-end-line="134" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/contrast_similar.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="9" column="26" selection-start-line="9" selection-start-column="5" selection-end-line="9" selection-end-column="26" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="385">
<caret line="78" column="43" selection-start-line="78" selection-start-column="43" selection-end-line="78" selection-end-column="43" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/type_info.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="257">
<caret line="303" column="34" selection-start-line="303" selection-start-column="18" selection-end-line="303" selection-end-column="34" />
<folding>
<element signature="e#15#87#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="ScopeChooserConfigurable.UI">
<settings>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>
\ No newline at end of file
libs/es.py
View file @ 7940fdf4
......@@ -165,9 +165,14 @@ class ESPerform(object):
return False
@classmethod
<<<<<<< HEAD
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False,
is_suggest_request=False):
=======
def get_search_results(cls, es_cli,sub_index_name,query_body,offset=0,size=10,
auto_create_index=False,doc_type="_doc",aggregations_query=False,is_suggest_request=False,batch_search=False):
>>>>>>> 338c6f92836042068a01d3670f4b581b206fd71d
try:
assert (es_cli is not None)
......@@ -182,11 +187,26 @@ class ESPerform(object):
cls.put_index_mapping(es_cli, sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
<<<<<<< HEAD
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset, size=size)
=======
>>>>>>> 338c6f92836042068a01d3670f4b581b206fd71d
if not batch_search:
res = es_cli.search(index=official_index_name,doc_type=doc_type,body=query_body,from_=offset,size=size)
if is_suggest_request:
return res
else:
result_dict = {
"total_count":res["hits"]["total"],
"hits":res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
<<<<<<< HEAD
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
......@@ -194,6 +214,12 @@ class ESPerform(object):
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
=======
res = es_cli.msearch(body=query_body,index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
>>>>>>> 338c6f92836042068a01d3670f4b581b206fd71d
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
requirements.txt
View file @ 7940fdf4
......@@ -11,6 +11,7 @@ PyMySQL==0.9.2
gunicorn==19.9.0
gevent==1.3.7
pypinyin==0.34.1
git+ssh://git@git.wanmeizhensuo.com/backend/gm-rpcd.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/helios.git@master
......@@ -19,5 +20,6 @@ git+ssh://git@git.wanmeizhensuo.com/backend/gm-config.git@v0.1.3#egg=gm-config==
git+ssh://git@git.wanmeizhensuo.com/backend/gm-protocol.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/gm-upload.git@master
git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@dev
git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/gm-types.git@master
search/utils/common.py
View file @ 7940fdf4
......@@ -26,7 +26,12 @@ class TopicDocumentField(object):
IS_ONLINE = "is_online",
TAG_LIST = "tag_list"
class TopicPageType(object):
# 首页推荐
HOME_RECOMMEND=1
# 发现页面
FIND_PAGE=2
def time_consuming_decorator(func):
def time_consuming(*args, **kwargs):
......
search/utils/topic.py
View file @ 7940fdf4
......@@ -6,6 +6,8 @@ import traceback
import json
from libs.es import ESPerform
from .common import TopicDocumentField
from search.utils.common import *
class TopicUtils(object):
......@@ -87,7 +89,7 @@ class TopicUtils(object):
return []
@classmethod
def get_recommend_topic_ids(cls,user_id,offset,size,query=None):
def get_recommend_topic_ids(cls,user_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[]):
"""
:需增加打散逻辑
:remark:获取首页推荐帖子列表
......@@ -171,18 +173,28 @@ class TopicUtils(object):
}
)
low_content_level = 4 if query_type==TopicPageType.FIND_PAGE else 3
query_function_score = {
"query": {
"bool": {
"must": {
"range": {"content_level": {"gte": 3, "lte": 5}}
}
"must": [
{"range": {"content_level": {"gte": low_content_level, "lte": 5}}},
{"term": {"has_image":True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
]
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
if query_type==TopicPageType.FIND_PAGE and len(filter_topic_id_list)>0:
query_function_score["query"]["bool"]["must_not"] = {
"terms":{
"id": filter_topic_id_list
}
}
if query is not None:#搜索帖子
multi_fields = {
......@@ -231,7 +243,7 @@ class TopicUtils(object):
return []
@classmethod
def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,offset,size):
def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -274,9 +286,9 @@ class TopicUtils(object):
query_function_score = {
"query":{
"bool":{
"must": {
"range": {"content_level": {"gte": 3, "lte": 5}}
},
"must": [
{"range": {"content_level": {"gte": 3, "lte": 5}}}
],
"must_not":{
"term":{
"id":topic_id
......@@ -289,6 +301,8 @@ class TopicUtils(object):
"functions": functions_list
}
if filter_topic_user_id:
query_function_score["query"]["bool"]["must"].append({"term": {"user_id": topic_user_id}})
if len(topic_tag_list)>0:
query_function_score["query"]["bool"]["should"]={
"terms":{
......
search/utils/user.py
View file @ 7940fdf4
......@@ -5,10 +5,57 @@ import logging
import traceback
import json
import time
import copy
from libs.es import ESPerform
class UserUtils(object):
@classmethod
def get_batch_attention_user_list(cls,user_id_list,self_user_id):
"""
:remark 批量用户 关注的 用户列表
:param user_id_list:
:param self_user_id:
:return:
"""
try:
user_id_list.append(self_user_id)
q = dict()
q["query"] = {
"terms":{
"user_id":user_id_list
}
}
q["_source"] = {
"include":["attention_user_id_list","user_id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset=0, size=len(user_id_list))
self_attention_user_id_list = []
attention_user_dict_list = list()
ret_attention_user_id_list = list()
for hit_item in result_dict["hits"]:
attention_user_dict = dict()
user_id = hit_item["_source"]["user_id"]
if user_id == self_user_id:
self_attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
else:
attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
attention_user_dict[user_id] = attention_user_id_list
attention_user_dict_list.append(attention_user_dict)
ret_attention_user_id_list.append(user_id)
return [self_attention_user_id_list,attention_user_dict_list,ret_attention_user_id_list]
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([],[],[])
@classmethod
def get_attention_user_list(cls,user_id_list,self_user_id):
"""
......@@ -131,3 +178,106 @@ class UserUtils(object):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_batch_recommend_user_dict(cls,need_filter_attention_user_id_list,attention_user_id_list,attention_user_dict_list,self_user_id,offset,size):
"""
:remark 获取批量推荐用户
:param need_filter_attention_user_id_list:
:param attention_user_dict:
:param self_user_id:
:param offset:
:param size:
:return:
"""
try:
batch_query_list = list()
for interesting_user_item_dict in attention_user_dict_list:
for interesting_user_id in interesting_user_item_dict:
if interesting_user_id != self_user_id:
logging.info("duan add,interesting_user_id:%d" % interesting_user_id)
filter_user_id_list = copy.deepcopy(need_filter_attention_user_id_list)
filter_user_id_list.append(interesting_user_id)
q = dict()
q["from"] = offset
q["size"] = size
q["query"] = dict()
functions_list = [
{
"gauss": {
"latest_topic_time_val": {
"origin": int(time.time()),
"scale": "600",
"decay": 0.1
}
}
}
]
if len(interesting_user_item_dict[interesting_user_id]) > 0:
functions_list.append(
{
"filter":{
"bool":{
"should":{"terms":{"user_id":interesting_user_item_dict[interesting_user_id]}}
}
},
"weight":10
}
)
query_function_score = {
"query": {
"bool": {
"must": [
{"term": {"is_recommend": True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"is_shadow": False}}
],
"must_not":{
"terms":{
"user_id":filter_user_id_list
}
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
q["query"]["function_score"] = query_function_score
q["_source"] = {
"include":["user_id"]
}
batch_query_list.append(q)
index_name = ESPerform.get_official_index_name("user","read")
search_header_dict = {'index': index_name, 'type': "_doc"}
query_body = ""
for query_item in batch_query_list:
query_body += "{}\n{}\n".format(json.dumps(search_header_dict),json.dumps(query_item))
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="user", query_body=query_body,
batch_search=True)
ret_dict = dict()
user_index = 0
for res_item in result_dict["responses"]:
recommend_user_list = list()
for item in res_item["hits"]["hits"]:
recommend_user_list.append(item["_source"]["user_id"])
logging.info("duan add,attention_user_id_list:%s,user_index:%d" % (str(attention_user_id_list), user_index))
ret_dict[str(attention_user_id_list[user_index])] = recommend_user_list
user_index += 1
logging.info("duan add,ret_dict:%s" % str(ret_dict))
return ret_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
search/views/topic.py
View file @ 7940fdf4
......@@ -12,12 +12,47 @@ from libs.cache import redis_client
from search.utils.common import *
def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE):
try:
if user_id == -1:
redis_key = "physical:discover_page" + ":user_id:" + str(user_id) + ":device_id:" + device_id
else:
redis_key = "physical:discover_page" + ":user_id:" + str(user_id)
redis_field_list = [b'have_read_topic_id']
redis_field_val_list = redis_client.hmget(redis_key,redis_field_list)
have_read_topic_id_list = json.loads(redis_field_val_list[0]) if redis_field_val_list[0] else []
recommend_topic_ids = []
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, offset=0, size=size,query_type=query_type,filter_topic_id_list=have_read_topic_id_list)
for item in topic_id_list:
recommend_topic_ids.append(item["id"])
have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = {
"have_read_topic_id": json.dumps(have_read_topic_id_list)
}
redis_client.hmset(redis_key,redis_dict)
return recommend_topic_ids
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_type=TopicPageType.HOME_RECOMMEND):
try:
if query is None:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":session_id:" + session_id
if user_id == -1:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":session_id:" + session_id + ":query:" + str(query)
if user_id == -1:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
redis_field_list = [b'last_offset_num', b'unread_topic_id']
for page_id in range(0,offset,size):
......@@ -27,7 +62,7 @@ def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
last_offset_num = int(redis_field_val_list[0]) if redis_field_val_list[0] else -1
recommend_topic_ids = []
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query)
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query,query_type=query_type)
have_read_group_id_set = set()
unread_topic_id_list = list()
have_read_topic_id_set = set()
......@@ -90,9 +125,32 @@ def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@bind("physical/search/home_recommend")
def home_recommend(session_id="",user_id=-1,offset=0,size=10):
def home_recommend(device_id="",user_id=-1,offset=0,size=10,query_type=TopicPageType.HOME_RECOMMEND):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
:param user_id:
:param offset:
:param size:
:return:
"""
try:
if not user_id:
user_id=-1
if not isinstance(device_id,str):
device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset,size,query_type=query_type)
return {"recommend_topic_ids":recommend_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
@bind("physical/search/discover_page")
def discover_page(device_id="",user_id=-1,size=10):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
......@@ -104,10 +162,10 @@ def home_recommend(session_id="",user_id=-1,offset=0,size=10):
try:
if not user_id:
user_id=-1
if not isinstance(session_id,str):
session_id = ""
if not isinstance(device_id,str):
device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,session_id,offset,size)
recommend_topic_ids = get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE)
return {"recommend_topic_ids":recommend_topic_ids}
except:
......@@ -116,7 +174,7 @@ def home_recommend(session_id="",user_id=-1,offset=0,size=10):
@bind("physical/search/home_query")
def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
def home_query(device_id="",user_id=-1,query="",offset=0,size=10):
"""
:remark:首页搜索，目前只推荐日记
:param session_id:
......@@ -129,10 +187,10 @@ def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
try:
if not user_id:
user_id=-1
if not isinstance(session_id,str):
session_id = ""
if not isinstance(device_id,str):
device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,session_id,offset,size,query)
recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset,size,query)
return {"recommend_topic_ids":recommend_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -140,7 +198,7 @@ def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
@bind("physical/search/topic_detail_page_recommend")
def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,offset=0,size=10):
def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,filter_topic_user_id=False,offset=0,size=10):
"""
:remark:帖子详情页推荐策略，缺少第一个卡片策略
:param user_id:
......@@ -156,7 +214,7 @@ def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_u
# 获取帖子标签列表
topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id)
result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,offset,size)
result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size)
recommend_topic_ids_list = list()
if len(result_list)>0:
recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
......
search/views/user.py
View file @ 7940fdf4
......@@ -38,4 +38,33 @@ def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
return recommend_user_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_user_ids":[]}
\ No newline at end of file
return []
@bind("physical/search/batch_recommend_user")
def batch_recommend_user(self_user_id,interesting_user_id_list,offset=0,size=10):
"""
:remark 点关注推荐用户 - 批量接口
:param self_user_id:
:param interesting_user_id:
:param offset:
:param size:
:return:
"""
try:
if not isinstance(self_user_id,int):
self_user_id = -1
#获取关注用户列表
(need_filter_attention_user_id_list, attention_user_dict_list,attention_user_id_list) = UserUtils.get_batch_attention_user_list(interesting_user_id_list,self_user_id)
#去除自身及感兴趣的用户ID
need_filter_attention_user_id_list.append(self_user_id)
recommend_user_dict = UserUtils.get_batch_recommend_user_dict(need_filter_attention_user_id_list=need_filter_attention_user_id_list,attention_user_id_list=attention_user_id_list,attention_user_dict_list=attention_user_dict_list,self_user_id=self_user_id,offset=offset,size=size)
logging.info("duan add,recommend_user_dict:%s" % str(recommend_user_dict))
return recommend_user_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {}
\ No newline at end of file
trans2es/mapping/topic.json
View file @ 7940fdf4
......@@ -18,6 +18,7 @@
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
"manual_score":{"type":"double"},//人工赋分
"has_image":{"type":"boolean"},//是否有图
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
......
trans2es/models/topic.py
View file @ 7940fdf4
......@@ -27,32 +27,58 @@ class ActionSumAboutTopic(models.Model):
data_type = models.IntegerField(verbose_name=u"动作类型")
data_value = models.BigIntegerField(verbose_name=u"值")
class TopicImage(models.Model):
class Meta:
verbose_name = u'日记图片'
db_table = 'topic_image'
id = models.IntegerField(verbose_name='日记图片ID',primary_key=True)
topic_id = models.IntegerField(verbose_name=u'日记ID')
url = models.CharField(verbose_name=u'图片URL',max_length=300)
is_online = models.BooleanField(verbose_name='是否上线')
is_deleted = models.BooleanField(verbose_name='是否删除')
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class Topic(models.Model):
class Meta:
verbose_name = u'日记'
db_table = 'topic'
id = models.IntegerField(verbose_name='日记ID',primary_key=True)
name = models.CharField(verbose_name='日记名称',max_length=100)
id = models.IntegerField(verbose_name=u'日记ID',primary_key=True)
name = models.CharField(verbose_name=u'日记名称',max_length=100)
#group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
group = models.ForeignKey(
Group, verbose_name=u"关联的小组", related_name=u"group_topics",null=True, blank=True, default=None, on_delete=models.CASCADE)
user_id = models.IntegerField(verbose_name='用户ID')
user_id = models.IntegerField(verbose_name=u'用户ID')
drop_score = models.IntegerField(verbose_name=u'人工赋分',default=0)
description = models.CharField(verbose_name='日记本描述',max_length=200)
content = models.CharField(verbose_name='日记本内容',max_length=1000)
description = models.CharField(verbose_name=u'日记本描述',max_length=200)
content = models.CharField(verbose_name=u'日记本内容',max_length=1000)
share_num = models.IntegerField(verbose_name='')
vote_num = models.IntegerField(verbose_name='点赞数')
reply_num = models.IntegerField(verbose_name='回复数')
vote_num = models.IntegerField(verbose_name=u'点赞数')
reply_num = models.IntegerField(verbose_name=u'回复数')
cover = models.CharField(verbose_name='',max_length=200)
is_online = models.BooleanField(verbose_name='是否上线')
is_deleted = models.BooleanField(verbose_name='是否删除')
content_level = models.CharField(verbose_name='内容等级',max_length=3)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
content_level = models.CharField(verbose_name=u'内容等级',max_length=3)
create_time = models.DateTimeField(verbose_name=u'日记创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
def topic_has_image(self):
try:
has_image = False
query_list = TopicImage.objects.filter(topic_id=self.id,is_deleted=False,is_online=True)
if len(query_list)>0:
has_image=True
return has_image
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
def get_pick_id_info(self):
try:
pick_id_list = list()
......
trans2es/utils/topic_transfer.py
View file @ 7940fdf4
......@@ -38,6 +38,7 @@ class TopicTransfer(object):
res["offline_score"] = instance.get_topic_offline_score()
res["manual_score"] = instance.drop_score
res["has_image"] = instance.topic_has_image()
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment