Skip to content

P
physical
Commit 0ad7f0ca authored by lixiaofang's avatar lixiaofang
Browse files
Options

Merge branch 'test' of git.wanmeizhensuo.com:alpha/physical into test

parents 8419b60e 5ccf3977
Hide whitespace changes
Inline Side-by-side
Showing with 362 additions and 710 deletions
.gitignore
View file @ 0ad7f0ca
......@@ -58,6 +58,7 @@ docs/_build/
# config
fabfile.py
settings.online.py
/gaia/settings.py
settings_local.py
media/
log/
......@@ -69,3 +70,16 @@ Vagrantfile
*.DS_Store
dump.rdb
# .gitignore for yangchuncheng
api/management/commands/ycc*
settings_override*
.script/
.tmp.sql
.env
*.pem
/gaia/hospital_list_settings.py
coverage_html/
gaia/rpcd.json
*.swp
dbmw_deploy/config.dir/
.idea/workspace.xml
View file @ 0ad7f0ca
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="d7dd36ca-85ef-4a59-9db5-8b1ee4993a4e" name="Default Changelist" comment="" />
<list default="true" id="d7dd36ca-85ef-4a59-9db5-8b1ee4993a4e" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -53,174 +55,34 @@
</counts>
</usages-collector>
</session>
<session id="-609148713">
<usages-collector id="statistics.lifecycle.project">
<counts>
<entry key="project.closed" value="1" />
<entry key="project.open.time.2" value="1" />
<entry key="project.open.time.8" value="1" />
<entry key="project.opened" value="2" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="json" value="8" />
<entry key="py" value="69" />
<entry key="template" value="3" />
<entry key="xml" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="JSON" value="8" />
<entry key="PLAIN_TEXT" value="3" />
<entry key="Python" value="69" />
<entry key="XML" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="json" value="56" />
<entry key="py" value="888" />
<entry key="template" value="5" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="JSON" value="56" />
<entry key="PLAIN_TEXT" value="5" />
<entry key="Python" value="888" />
</counts>
</usages-collector>
</session>
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/search/views/topic.py">
<provider selected="true" editor-type-id="text-editor">
<<<<<<< HEAD
<state relative-caret-position="15">
=======
<state relative-caret-position="823">
>>>>>>> huabao
<caret line="103" column="37" selection-start-line="103" selection-start-column="32" selection-end-line="103" selection-end-column="37" />
<state relative-caret-position="209">
<caret line="85" column="46" selection-start-line="85" selection-start-column="46" selection-end-line="85" selection-end-column="46" />
<folding>
<element signature="e#466#1607#0" />
<element signature="e#7074#7716#0" />
<element signature="e#7816#8386#0" />
<element signature="e#8511#9086#0" />
<element signature="e#463#1573#0" />
<element signature="e#6049#6240#0" />
<element signature="e#7051#7614#0" />
<element signature="e#8492#10033#0" />
<element signature="e#10162#10820#0" />
<element signature="e#10934#11363#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/search/views/group.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="510">
<caret line="42" column="16" lean-forward="true" selection-start-line="42" selection-start-column="16" selection-end-line="42" selection-end-column="16" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/type_info.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="265">
<caret line="337" column="24" selection-start-line="337" selection-start-column="24" selection-end-line="337" selection-end-column="24" />
<folding>
<element signature="e#15#87#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/models/pictorial.py">
<provider selected="true" editor-type-id="text-editor">
<<<<<<< HEAD
<state relative-caret-position="307">
<caret line="264" column="36" lean-forward="true" selection-start-line="264" selection-start-column="36" selection-end-line="264" selection-end-column="36" />
=======
<state>
<caret line="31" column="4" selection-start-line="31" selection-start-column="4" selection-end-line="31" selection-end-column="4" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/utils/pictorial_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="322">
<caret line="28" column="48" lean-forward="true" selection-start-line="28" selection-start-column="48" selection-end-line="28" selection-end-column="48" />
>>>>>>> huabao
<folding>
<element signature="e#46#55#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/mapping/topic.json">
<provider selected="true" editor-type-id="text-editor">
<<<<<<< HEAD
<state relative-caret-position="215">
<caret line="24" column="16" selection-start-line="24" selection-start-column="5" selection-end-line="24" selection-end-column="16" />
=======
<state relative-caret-position="319">
<caret line="27" column="36" lean-forward="true" selection-start-line="27" selection-start-column="36" selection-end-line="27" selection-end-column="36" />
>>>>>>> huabao
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/trans2es/utils/topic_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<<<<<<< HEAD
<state relative-caret-position="1068">
<caret line="75" column="19" lean-forward="true" selection-start-line="75" selection-start-column="19" selection-end-line="75" selection-end-column="19" />
=======
<state relative-caret-position="1129">
<caret line="88" column="48" lean-forward="true" selection-start-line="88" selection-start-column="48" selection-end-line="88" selection-end-column="48" />
>>>>>>> huabao
<folding>
<element signature="e#46#55#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/models/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="698">
<caret line="98" column="80" selection-start-line="98" selection-start-column="12" selection-end-line="98" selection-end-column="80" />
<folding>
<element signature="e#46#118#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/mapping/pictorial.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="16" column="1" lean-forward="true" selection-start-line="16" selection-start-column="1" selection-end-line="16" selection-end-column="1" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/search/utils/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1290">
<caret line="94" column="8" selection-start-line="94" selection-start-column="8" selection-end-line="94" selection-end-column="8" />
<state relative-caret-position="1760">
<caret line="293" column="66" selection-start-line="293" selection-start-column="66" selection-end-line="293" selection-end-column="66" />
<folding>
<element signature="e#47#61#0" expanded="true" />
<element signature="e#363#1043#0" />
</folding>
</state>
</provider>
......@@ -228,72 +90,25 @@
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>grop</find>
<find>contrast_similar</find>
<find>topic</find>
<find>GroupTransfer</find>
<find>get_group_query_result</find>
<find>get_group_ids_by_aggs</find>
<find>pictorial_id</find>
<find>pictorial_name</find>
<find>get_recommend_topic_ids</find>
<find>ES_INDEX_PREFIX</find>
<find>physical/search/query_tag_id_by_topic</find>
<find>tag_name_list</find>
<find>pictorial</find>
<find>group</find>
<find>update_time</find>
</findStrings>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py" />
<option value="$PROJECT_DIR$/physical/settings.py" />
<option value="$PROJECT_DIR$/trans2es/utils/pictorial_transfer.py" />
<option value="$PROJECT_DIR$/trans2es/mapping/group.json" />
<option value="$PROJECT_DIR$/trans2es/mapping/pictorial.py" />
<option value="$PROJECT_DIR$/trans2es/mapping/pictorial.json" />
<option value="$PROJECT_DIR$/trans2es/type_info.py" />
<option value="$PROJECT_DIR$/trans2es/models/group.py" />
<option value="$PROJECT_DIR$/trans2es/utils/group_transfer.py" />
<option value="$PROJECT_DIR$/trans2es/models/pictorial.py" />
<option value="$PROJECT_DIR$/trans2es/mapping/user.json" />
<option value="$PROJECT_DIR$/trans2es/utils/user_transfer.py" />
<option value="$PROJECT_DIR$/trans2es/models/user.py" />
<option value="$PROJECT_DIR$/trans2es/management/commands/trans2es_data2es_parallel.py" />
<option value="$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py" />
<option value="$PROJECT_DIR$/search/views/topic.py" />
<option value="$PROJECT_DIR$/search/views/group.py" />
<option value="$PROJECT_DIR$/search/utils/topic.py" />
<option value="$PROJECT_DIR$/physical/settings_local.py.template" />
<option value="$PROJECT_DIR$/search/utils/group.py" />
<option value="$PROJECT_DIR$/trans2es/models/topic.py" />
<option value="$PROJECT_DIR$/trans2es/mapping/topic.json" />
<option value="$PROJECT_DIR$/trans2es/utils/topic_transfer.py" />
<option value="$PROJECT_DIR$/physical/settings.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<<<<<<< HEAD
<option name="x" value="-89" />
=======
<option name="x" value="-56" />
>>>>>>> huabao
<option name="x" value="279" />
<option name="y" value="23" />
<option name="width" value="1920" />
<option name="height" value="724" />
<option name="width" value="1280" />
<option name="height" value="738" />
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
......@@ -328,7 +143,7 @@
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="injection" type="462c0819:PsiDirectoryNode" />
<item name="linucb" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
......@@ -343,13 +158,14 @@
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
<item name="search" type="462c0819:PsiDirectoryNode" />
<item name="utils" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
<item name="mapping" type="462c0819:PsiDirectoryNode" />
<item name="search" type="462c0819:PsiDirectoryNode" />
<item name="views" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
......@@ -362,9 +178,6 @@
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/trans2es/mapping" />
</key>
<key name="MoveFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/search/views" />
</key>
......@@ -395,14 +208,9 @@
<servers />
</component>
<component name="ToolWindowManager">
<<<<<<< HEAD
<frame x="-89" y="23" width="1920" height="724" extended-state="0" />
=======
<frame x="-56" y="23" width="1920" height="724" extended-state="0" />
>>>>>>> huabao
<editor active="true" />
<frame x="0" y="23" width="1280" height="724" extended-state="6" />
<layout>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.18114798" />
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.22160445" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="DB Browser" order="2" />
<window_info id="Favorites" order="3" side_tool="true" />
......@@ -415,11 +223,7 @@
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="DB Execution Console" order="8" />
<<<<<<< HEAD
<window_info anchor="bottom" id="Terminal" order="9" visible="true" weight="0.56978655" />
=======
<window_info anchor="bottom" id="Terminal" order="9" visible="true" weight="0.0" />
>>>>>>> huabao
<window_info active="true" anchor="bottom" id="Terminal" order="9" visible="true" weight="0.44663382" />
<window_info anchor="bottom" id="Python Console" order="10" />
<window_info anchor="bottom" id="Event Log" order="11" side_tool="true" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
......@@ -430,62 +234,7 @@
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/trans2es/models/user.py</url>
<line>148</line>
<option name="timeStamp" value="1" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/trans2es/models/user.py</url>
<line>139</line>
<option name="timeStamp" value="2" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/search/views/group.py</url>
<line>36</line>
<option name="timeStamp" value="3" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/search/utils/group.py</url>
<line>11</line>
<option name="timeStamp" value="4" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/trans2es/models/topic.py</url>
<line>86</line>
<option name="timeStamp" value="5" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/trans2es/models/topic.py</url>
<line>110</line>
<option name="timeStamp" value="7" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/trans2es/models/topic.py</url>
<line>136</line>
<option name="timeStamp" value="8" />
</line-breakpoint>
</breakpoints>
</breakpoint-manager>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/pictorial.py" />
<entry file="file://$PROJECT_DIR$/trans2es/mapping/group.json">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret column="1" lean-forward="true" selection-start-column="1" selection-end-column="1" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/management/commands/trans2es_mapping2es.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
......@@ -493,319 +242,57 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/group_user_role.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/pick_celebrity.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/pick_topic.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/pickuserrecord.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/user_extra.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/user_follow.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/group_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150">
<caret line="14" column="25" selection-start-line="14" selection-start-column="25" selection-end-line="14" selection-end-column="25" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/group.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" column="26" lean-forward="true" selection-start-line="14" selection-start-column="26" selection-end-line="14" selection-end-column="26" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/user_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="644">
<caret line="76" selection-start-line="76" selection-end-line="76" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/user.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="448">
<caret line="51" column="32" selection-start-line="51" selection-start-column="5" selection-end-line="51" selection-end-column="32" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/django_init.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/settings.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180">
<caret line="12" column="9" lean-forward="true" selection-start-line="12" selection-start-column="9" selection-end-line="12" selection-end-column="9" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/management/commands/trans2es_data2es_parallel.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="585">
<caret line="97" lean-forward="true" selection-start-line="97" selection-end-line="97" />
<state relative-caret-position="364">
<caret line="129" column="63" selection-start-line="129" selection-start-column="54" selection-end-line="129" selection-end-column="63" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/user.py">
<entry file="file://$PROJECT_DIR$/search/utils/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" column="25" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
<state relative-caret-position="1760">
<caret line="293" column="66" selection-start-line="293" selection-start-column="66" selection-end-line="293" selection-end-column="66" />
<folding>
<element signature="e#47#79#0" expanded="true" />
<element signature="e#460#7466#0" />
<element signature="e#1396#1736#0" />
<element signature="e#1787#2318#0" />
<element signature="e#2367#3238#0" />
<element signature="e#3291#3907#0" />
<element signature="e#3960#4576#0" />
<element signature="e#3954#5060#0" />
<element signature="e#5114#6318#0" />
<element signature="e#6364#6780#0" />
<element signature="e#47#61#0" expanded="true" />
<element signature="e#363#1043#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-395">
<caret line="1" column="12" selection-start-line="1" selection-start-column="12" selection-end-line="1" selection-end-column="12" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/tag.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/search/views/tag.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-28" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/pick.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/app_conf.xml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/search/utils/common.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="72">
<caret line="6" column="6" selection-start-line="6" selection-start-column="6" selection-end-line="6" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/physical/celery.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/celery_task_router.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="25" column="23" lean-forward="true" selection-start-line="25" selection-start-column="23" selection-end-line="25" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/physical/urls.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/wsgi.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/settings_local.py.template">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1837">
<caret line="146" column="26" selection-start-line="146" selection-start-column="26" selection-end-line="146" selection-end-column="26" />
</state>
<state relative-caret-position="-1987" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/physical/views.py">
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/libs/es.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="17" column="43" selection-start-line="17" selection-start-column="28" selection-end-line="17" selection-end-column="43" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/utils/group.py">
<<<<<<< HEAD
=======
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="147">
<caret line="115" column="21" selection-start-line="115" selection-start-column="21" selection-end-line="115" selection-end-column="21" />
<folding>
<element signature="e#218#1347#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/injection/data_sync/tasks.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180">
<caret line="22" column="22" selection-start-line="22" selection-start-column="22" selection-end-line="22" selection-end-column="38" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/utils/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1290">
<caret line="94" column="8" selection-start-line="94" selection-start-column="8" selection-end-line="94" selection-end-column="8" />
<folding>
<element signature="e#47#61#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/group.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="510">
<caret line="42" column="16" lean-forward="true" selection-start-line="42" selection-start-column="16" selection-end-line="42" selection-end-column="16" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/topic.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="319">
<caret line="27" column="36" lean-forward="true" selection-start-line="27" selection-start-column="36" selection-end-line="27" selection-end-column="36" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="823">
<caret line="103" column="37" selection-start-line="103" selection-start-column="32" selection-end-line="103" selection-end-column="37" />
<folding>
<element signature="e#466#1607#0" />
<element signature="e#7074#7716#0" />
<element signature="e#7816#8386#0" />
<element signature="e#8511#9086#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/pictorial.json">
>>>>>>> huabao
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="16" column="1" lean-forward="true" selection-start-line="16" selection-start-column="1" selection-end-line="16" selection-end-column="1" />
</state>
</provider>
</entry>
<<<<<<< HEAD
=======
<entry file="file://$PROJECT_DIR$/trans2es/type_info.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="265">
<caret line="337" column="24" selection-start-line="337" selection-start-column="24" selection-end-line="337" selection-end-column="24" />
<folding>
<element signature="e#15#87#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/pictorial_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="322">
<caret line="28" column="48" lean-forward="true" selection-start-line="28" selection-start-column="48" selection-end-line="28" selection-end-column="48" />
<folding>
<element signature="e#46#55#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
>>>>>>> huabao
<entry file="file://$PROJECT_DIR$/trans2es/models/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="698">
<caret line="98" column="80" selection-start-line="98" selection-start-column="12" selection-end-line="98" selection-end-column="80" />
<folding>
<element signature="e#46#118#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/pictorial.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret line="31" column="4" selection-start-line="31" selection-start-column="4" selection-end-line="31" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/topic_transfer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1129">
<caret line="88" column="48" lean-forward="true" selection-start-line="88" selection-start-column="48" selection-end-line="88" selection-end-column="48" />
<folding>
<element signature="e#46#55#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<<<<<<< HEAD
<entry file="file://$PROJECT_DIR$/trans2es/mapping/topic.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="215">
<caret line="24" column="16" selection-start-line="24" selection-start-column="5" selection-end-line="24" selection-end-column="16" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/group.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="510">
<caret line="42" column="16" lean-forward="true" selection-start-line="42" selection-start-column="16" selection-end-line="42" selection-end-column="16" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="15">
<caret line="103" column="37" selection-start-line="103" selection-start-column="32" selection-end-line="103" selection-end-column="37" />
<folding>
<element signature="e#466#1607#0" />
<element signature="e#7074#7716#0" />
<element signature="e#7816#8386#0" />
<element signature="e#8511#9086#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/type_info.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="307">
<caret line="264" column="36" lean-forward="true" selection-start-line="264" selection-start-column="36" selection-end-line="264" selection-end-column="36" />
<state relative-caret-position="209">
<caret line="85" column="46" selection-start-line="85" selection-start-column="46" selection-end-line="85" selection-end-column="46" />
<folding>
<element signature="e#15#87#0" expanded="true" />
<element signature="e#463#1573#0" />
<element signature="e#6049#6240#0" />
<element signature="e#7051#7614#0" />
<element signature="e#8492#10033#0" />
<element signature="e#10162#10820#0" />
<element signature="e#10934#11363#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/utils/topic_transfer.py">
<entry file="file://$PROJECT_DIR$/physical/settings.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1068">
<caret line="75" column="19" lean-forward="true" selection-start-line="75" selection-start-column="19" selection-end-line="75" selection-end-column="19" />
<folding>
<element signature="e#46#55#0" expanded="true" />
</folding>
<state relative-caret-position="-2448">
<caret line="12" column="9" lean-forward="true" selection-start-line="12" selection-start-column="9" selection-end-line="12" selection-end-column="9" />
</state>
</provider>
</entry>
=======
>>>>>>> huabao
</component>
<component name="masterDetails">
<states>
......
injection/data_sync/tasks.py
View file @ 0ad7f0ca
......@@ -21,7 +21,6 @@ def write_to_es(es_type, pk_list, use_batch_query_set=False):
type_info = type_info_map[es_type]
logging.info("duan add,es_type:%s" % str(es_type))
logging.info("get es_type:%s"%es_type)
type_info.insert_table_by_pk_list(
sub_index_name=es_type,
pk_list=pk_list,
......
libs/es.py
View file @ 0ad7f0ca
......@@ -111,10 +111,6 @@ class ESPerform(object):
return False
mapping_dict = cls.__load_mapping(sub_index_name)
logging.info("get write_alias_name:%s"%write_alias_name)
logging.info("get mapping_dict:%s"%mapping_dict)
logging.info("get mapping_type:%s"%mapping_type)
es_cli.indices.put_mapping(index=write_alias_name,body=mapping_dict,doc_type=mapping_type)
return True
......@@ -226,17 +222,22 @@ class ESPerform(object):
for item in es_nodes_info_list:
try:
item_list = item.split(" ")
if len(item_list)>4:
if len(item_list)==11:
cpu_load = item_list[4]
elif len(item_list)==10:
cpu_load = item_list[3]
if int(cpu_load) > 60:
high_num += 1
es_nodes_list.append(int(cpu_load))
else:
continue
int_cpu_load = int(cpu_load)
if int_cpu_load > 60:
high_num += 1
es_nodes_list.append(int_cpu_load)
except:
logging.error("catch exception,item:%s,err_msg:%s" % (str(item),traceback.format_exc()))
return True
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s" % str(es_nodes_info_list))
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
......
linucb/views/collect_data.py
View file @ 0ad7f0ca
......@@ -98,9 +98,10 @@ class CollectData(object):
logging.info("consume topic_id:%s,device_id:%s" % (str(topic_id), str(device_id)))
tag_list = list()
sql_query_results = TopicTag.objects.filter(is_online=True, topic_id=topic_id)
for sql_item in sql_query_results:
tag_list.append(sql_item.tag_id)
click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online")
for tag_id,is_online in click_sql_query_results:
if is_online:
tag_list.append(tag_id)
is_click = 1
is_vote = 0
......@@ -130,17 +131,19 @@ class CollectData(object):
exposure_topic_id = item["card_id"]
logging.info(
"consume exposure topic_id:%s,device_id:%s" % (str(exposure_topic_id), str(device_id)))
exposure_topic_id_list.append(exposure_topic_id)
if exposure_topic_id:
exposure_topic_id_list.append(exposure_topic_id)
topic_tag_id_dict = dict()
tag_list = list()
sql_query_results = TopicTag.objects.filter(is_online=True, topic_id__in=exposure_topic_id_list)
for sql_item in sql_query_results:
tag_list.append(sql_item.tag_id)
if sql_item.topic_id not in topic_tag_id_dict:
topic_tag_id_dict[sql_item.topic_id] = list()
topic_tag_id_dict[sql_item.topic_id].append(sql_item.tag_id)
exposure_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id__in=exposure_topic_id_list).values_list("topic_id","tag_id","is_online")
for topic_id,tag_id,is_online in exposure_sql_query_results:
if is_online:
tag_list.append(tag_id)
if topic_id not in topic_tag_id_dict:
topic_tag_id_dict[topic_id] = list()
topic_tag_id_dict[topic_id].append(tag_id)
is_click = 0
is_vote = 0
......
linucb/views/linucb.py
View file @ 0ad7f0ca
......@@ -9,6 +9,8 @@ import logging
import traceback
import json
import pickle
from django.conf import settings
class LinUCB:
d = 2
......@@ -22,12 +24,9 @@ class LinUCB:
try:
if len(cls.default_tag_list) == 0:
query_item_results = Tag.objects.filter(is_online=True)
for item in query_item_results:
cls.default_tag_list.append(item.id)
cls.default_tag_list = Tag.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True,collection=1).values_list("id",flat=True)[0:100]
return cls.default_tag_list[:20]
return cls.default_tag_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
......@@ -87,6 +86,8 @@ class LinUCB:
for top_score in sorted_np_score_list:
for top_score_index in np_score_dict[top_score]:
top_tag_set.add(str(tag_list[top_score_index], encoding="utf-8"))
if len(top_tag_set) >= 10:
break
if len(top_tag_set) >= 10:
break
......
search/utils/topic.py
View file @ 0ad7f0ca
......@@ -34,8 +34,6 @@ class TopicUtils(object):
q["_source"] = {
"include": ["tag_list", "attention_user_id_list", "pick_user_id_list", "same_pictorial_user_id_list"]
}
logging.info("get query-------:%s"%q)
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset, size)
return result_dict
......@@ -122,10 +120,9 @@ class TopicUtils(object):
return {}
@classmethod
def get_recommend_topic_ids(cls, user_id, tag_id, offset, size, single_size, query=None,
query_type=TopicPageType.FIND_PAGE,
filter_topic_id_list=[], test_score=False, must_topic_id_list=[], recommend_tag_list=[],
user_similar_score_list=[]):
def get_recommend_topic_ids(cls,user_id,tag_id,offset,size,single_size,query=None,query_type=TopicPageType.FIND_PAGE,
filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],
user_similar_score_list=[],index_type="topic"):
"""
:需增加打散逻辑
:remark:获取首页推荐帖子列表
......@@ -139,7 +136,6 @@ class TopicUtils(object):
attention_user_id_list = list()
pick_user_id_list = list()
# same_group_id_list = list()
same_pictorial_id_list = list()
user_tag_list = list()
result_dict = TopicUtils.get_related_user_info(user_id, 0, 1)
......@@ -152,19 +148,13 @@ class TopicUtils(object):
pick_user_info_list = result_dict["hits"][0]["_source"]["pick_user_id_list"]
pick_user_id_list = [item["user_id"] for item in pick_user_info_list]
logging.info("get result_dict_hits :%s" % result_dict["hits"])
# same_group_user_info_list = result_dict["hits"][0]["_source"]["same_group_user_id_list"]
# same_group_id_list = [item["user_id"] for item in same_group_user_info_list]
# same_group_id_list = same_group_id_list[:100]
same_pictorial_user_info_list = result_dict["hits"][0]["_source"]["same_pictorial_user_id_list"]
logging.info("get same_pictorial_user_info_list :%s" % same_pictorial_user_info_list)
same_pictorial_id_list = [item["user_id"] for item in same_pictorial_user_info_list]
same_pictorial_id_list = same_pictorial_id_list[:100]
# same_pictorial_user_info_list = result_dict["hits"][0]["_source"]["same_pictorial_user_id_list"]
#
# same_pictorial_id_list = [item["user_id"] for item in same_pictorial_user_info_list]
# same_pictorial_id_list = same_pictorial_id_list[:100]
user_tag_list = result_dict["hits"][0]["_source"]["tag_list"]
logging.info("get same_pictorial_id_list :%s" % same_pictorial_id_list)
q = dict()
q["query"] = dict()
......@@ -215,22 +205,15 @@ class TopicUtils(object):
"weight": 2
}
)
# if len(same_group_id_list)>0:
# if len(same_pictorial_id_list) > 0:
# functions_list.append(
# {
# "filter": {"bool": {
# "should": {"terms":{"user_id":same_group_id_list}}}},
# "should": {"terms": {"user_id": same_pictorial_id_list}}}},
# "weight": 1
# }
# )
if len(same_pictorial_id_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"user_id": same_pictorial_id_list}}}},
"weight": 1
}
)
# query_tag_term_list = cls.___get_should_term_list(user_tag_list)
if len(user_tag_list) > 0:
......@@ -255,8 +238,8 @@ class TopicUtils(object):
"query": {
"bool": {
"filter": [
{"range": {"content_level": {"gte": low_content_level, "lte": 5}}},
{"term": {"has_image": True}},
# {"range": {"content_level": {"gte": low_content_level, "lte": 5}}},
{"term": {"has_image":True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
......@@ -336,14 +319,13 @@ class TopicUtils(object):
},
"order": "desc"
}
},
"_score"
}
]
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q,
offset=offset, size=size)
topic_id_list = list()
same_pictorial_id_set = set()
same_group_id_set = set()
same_user_id_set = set()
for item in result_dict["hits"]:
......@@ -381,7 +363,7 @@ class TopicUtils(object):
@classmethod
def get_topic_detail_recommend_list(cls, user_id, topic_id, topic_tag_list, topic_pictorial_id, topic_user_id,
filter_topic_user_id, have_read_topic_list, offset, size, es_cli_obj=None):
filter_topic_user_id, have_read_topic_list, offset, size, es_cli_obj=None,index_type="topic"):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -428,7 +410,7 @@ class TopicUtils(object):
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 3, "lte": 5}}},
# {"range": {"content_level": {"gte": 3, "lte": 5}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
......@@ -457,7 +439,7 @@ class TopicUtils(object):
"includes": ["id", "pictorial_id", "user_id", "_score"]
}
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="topic", query_body=q,
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q,
offset=offset, size=size)
return result_dict["hits"]
......@@ -598,10 +580,22 @@ class TopicUtils(object):
{"term": {"is_deleted": False}},
]
logging.info("get filters:%s"%filters)
if not filters:
return f
for k, v in filters.items():
if k == "is_complaint":
f.append({
"term": {k: v},
})
if k == "is_complaint":
f.append({
"term": {k: v},
})
if not v:
continue
......@@ -635,6 +629,29 @@ class TopicUtils(object):
}
}
})
elif k.startswith("__gte") or k.startswith("__lte") or \
k.startswith("__gt") or k.startswith("__lt"):
if k.startswith("__gte"):
op = "gte"
filed = k[:-5]
elif k == "__lte":
op = "lte"
filed = k[:-5]
elif k == "__gt":
op = "lte"
filed = k[:-4]
elif k == "__lt":
op = "lte"
filed = k[:-4]
f.append({
"range": {
filed: {
op: v,
}
}
})
else:
if isinstance(v, list):
f.append({
......@@ -726,14 +743,74 @@ class TopicUtils(object):
if sorts:
q["sort"] = sorts
logging.info("get query:%s" % q)
try:
result_dict = ESPerform.get_search_results(
ESPerform.get_cli(), sub_index_name=index_name,
query_body=q, offset=offset, size=size
)
return {
"hits": result_dict["hits"],
"total_count": result_dict["total_count"]
}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {
"hits": [],
"total_count": 0
}
@classmethod
def business_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
must = cls.process_filters(filters, filter_online=filter_online)
query = ''
for k, v in filters.items():
if k == "content":
query = filters[k]
q = {}
q["query"] = {
"function_score": {
"functions": [{
"filter": {
"bool": {
"must": must,
"must_not": cls.process_nfilters(nfilters),
}
},
"weight": 1
}],
"query": {
"multi_match": {
"fields":["content"],
"type": "cross_fields",
"operator": "and",
"query": query
}
}
}
}
if query == '':
q["query"] = {
"bool": {
"must": must,
"must_not": cls.process_nfilters(nfilters),
}
}
if sorts_by:
sorts = cls.process_sort(sorts_by)
if sorts:
q["sort"] = sorts
try:
result_dict = ESPerform.get_search_results(
ESPerform.get_cli(), sub_index_name=index_name,
query_body=q, offset=offset, size=size
)
logging.info("get result_dicts:%s" % result_dict)
return {
"hits": result_dict["hits"],
......
search/views/business_topic.py
View file @ 0ad7f0ca
......@@ -18,11 +18,14 @@ def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=
"""帖子搜索。"""
try:
result_list = TopicUtils.list_topic_ids(
result_list = TopicUtils.business_topic_ids(
filters=filters, nfilters=nfilters, sorts_by=sorts_by,
offset=offset, size=size, filter_online=False,
index_name="topic"
)
logging.info("get result_lsit:%s"%result_list)
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
except:
......
search/views/group.py
View file @ 0ad7f0ca
......@@ -13,7 +13,7 @@ from libs.es import ESPerform
@bind("physical/search/query_pictorial")
def query_group(query="", offset=0, size=10):
def query_pictorial(query="", offset=0, size=10):
"""
:remark:小组搜索排序策略，缺少排序策略
:param query:
......@@ -103,7 +103,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
}
}
q["_source"] = {
"include": ["id", "pictorial_id", "tag_list"]
"includes": ["id", "pictorial_id", "tag_list"]
}
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size)
logging.info("get result_dict:%s" % result_dict)
......@@ -140,7 +140,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
}
q["_source"] = {
"include": ["id", "update_time"]
"includes": ["id", "update_time"]
}
q["sort"] = {
'update_time': {
......@@ -153,8 +153,6 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
for item in result_dict["hits"]:
pictorial_id = item["_source"]["id"]
pictorial_id_list.append(pictorial_id)
logging.info("get pictorial_id_list:%s" % pictorial_id_list)
logging.info("get topic_tag_list:%s" % topic_tag_list)
if len(pictorial_id_list) < 10:
num = 10 - len(pictorial_id_list)
......@@ -175,7 +173,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
}
}
q["_source"] = {
"include": ["id", "tag_id"]}
"includes": ["id", "tag_id"]}
q["sort"] = {
'update_time': {
'order': 'desc'
......@@ -192,9 +190,6 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
if id not in pictorial_id_list:
pictorial_id_list.append(id) #
logging.info("get result_dict tag:%s" % result_dict)
logging.info("get pictorial_id_list tag:%s" % pictorial_id_list)
pictorial_list = pictorial_id_list if len(pictorial_id_list) < 10 else pictorial_id_list[:10]
return {"pictorial_ids_list": pictorial_list}
......
search/views/topic.py
View file @ 0ad7f0ca
......@@ -27,7 +27,7 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list)
filter_topic_id_list=have_read_topic_id_list,index_type="topic-high-star")
have_read_topic_id_list.extend(recommend_topic_ids)
......@@ -46,18 +46,11 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
query_type=TopicPageType.HOME_RECOMMEND):
try:
if query is None:
if user_id == -1:
redis_key = "physical:home_recommend" + ":user_id:" + str(
user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
redis_key = "physical:home_recommend" + ":user_id:" + str(
user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
if user_id == -1:
redis_key = "physical:home_query" + ":user_id:" + str(
user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_key = "physical:home_query" + ":user_id:" + str(
user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
......@@ -72,8 +65,12 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
recommend_topic_ids = []
have_read_topic_id_list = list()
if redis_field_val_list[0] and query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if redis_field_val_list[0]:
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset>0:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
......@@ -84,7 +81,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
recommend_tag_list=recommend_tag_list,
user_similar_score_list=user_similar_score_redis_list)
user_similar_score_list=user_similar_score_redis_list,index_type="topic-high-star")
have_read_group_id_set = set()
have_read_user_id_set = set()
unread_topic_id_dict = dict()
......@@ -170,11 +167,11 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
recommend_topic_ids = list()
es_node_load_high_flag = False
try:
es_node_load_high_flag = ESPerform.if_es_node_load_high(ESPerform.get_cli())
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
es_node_load_high_flag = True
# try:
# es_node_load_high_flag = ESPerform.if_es_node_load_high(ESPerform.get_cli())
# except:
# logging.error("catch exception,err_msg:%s" % traceback.format_exc())
# es_node_load_high_flag = True
if es_node_load_high_flag:
temp_downgrading_key = "physical:home_recommend:user_id:241407656:query_type:1"
......@@ -273,7 +270,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
result_list = TopicUtils.get_topic_detail_recommend_list(user_id, topic_id, topic_tag_list, topic_pictorial_id,
topic_user_id, filter_topic_user_id,
have_read_topic_list, offset, size, es_cli_obj)
have_read_topic_list, offset, size, es_cli_obj,index_type="topic-high-star")
recommend_topic_ids_list = list()
if len(result_list) > 0:
recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
......@@ -348,7 +345,7 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size,
must_topic_id_list=must_topic_id_list)
must_topic_id_list=must_topic_id_list,index_type="topic-high-star")
return {"recommend_topic_ids": topic_id_list}
except:
......
trans2es/management/commands/trans2es_data2es_parallel.py
View file @ 0ad7f0ca
......@@ -49,6 +49,7 @@ class Job(object):
def __call__(self):
type_info = get_type_info_map()[self._type_name]
assert isinstance(type_info, TypeInfo)
result = type_info.insert_table_chunk(
sub_index_name=self._sub_index_name,
table_chunk=self._chunk,
......
trans2es/mapping/topic-high-star.json 0 → 100644
View file @ 0ad7f0ca
{
"dynamic":"strict",
"properties": {
"id":{"type":"long"},
"is_online":{"type":"boolean"},//上线
"is_deleted":{"type":"boolean"},
"vote_num":{"type":"long"},
"reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"},
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
"manual_score":{"type":"double"},//人工赋分
"has_image":{"type":"boolean"},//是否有图
"has_video":{"type":"boolean"},//是否是视频
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
"update_time_val":{"type":"long"},
"language_type":{"type":"long"},
"is_shadow": {"type": "boolean"},
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"virtual_content_level":{"type": "text"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
}
}
}
\ No newline at end of file
trans2es/models/group.py
View file @ 0ad7f0ca
......@@ -27,15 +27,15 @@ class Group(models.Model):
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
# #获取前一天4，5星发帖数
# def get_high_quality_topic_num(self):
# yesterday = datetime.datetime.now()-datetime.timedelta(days=1)
# yesterday_begin_time = "%s-%s-%s 00:00:00" % (yesterday.year, yesterday.month, yesterday.day)
# yesterday_end_time = "%s-%s-%s 23:59:59" % (yesterday.year, yesterday.month, yesterday.day)
#
# topic_num = self.group_topics.filter(content_level__in=("4","5"),create_time__gte=yesterday_begin_time,create_time__lte=yesterday_end_time).count()
#
# return topic_num
#获取前一天4，5星发帖数
def get_high_quality_topic_num(self):
yesterday = datetime.datetime.now()-datetime.timedelta(days=1)
yesterday_begin_time = "%s-%s-%s 00:00:00" % (yesterday.year, yesterday.month, yesterday.day)
yesterday_end_time = "%s-%s-%s 23:59:59" % (yesterday.year, yesterday.month, yesterday.day)
topic_num = self.group_topics.filter(content_level__in=("4","5"),create_time__gte=yesterday_begin_time,create_time__lte=yesterday_end_time).count()
return topic_num
def detail(self):
result = {
......
trans2es/models/pictorial.py
View file @ 0ad7f0ca
from django.db import models
import datetime
from django.db import models
import logging
import traceback
......@@ -81,13 +81,9 @@ class Pictorial(models.Model):
topic_id_list = PictorialTopics.objects.filter(pictorial_id=self.id).values_list("topic_id", flat=True)
logging.info("get topic_id_list:%s" % topic_id_list)
topic_num = Topic.objects.filter(content_level__in=["4", "5"], create_time__gte=yesterday_begin_time,
create_time__lte=yesterday_end_time, id__in=topic_id_list).count()
logging.info("get topic_num:%s" % topic_num)
return topic_num
except:
......@@ -98,9 +94,7 @@ class Pictorial(models.Model):
try:
tag_id_list = []
tags = PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True)
logging.info("get tags:%s" % tags)
for i in tags:
logging.info("get tags i:%s" % i)
tag_id_list.append(i)
return tag_id_list
......@@ -112,13 +106,9 @@ class Pictorial(models.Model):
def get_tag_by_name(self, tag_id):
try:
tag_name_list = []
logging.info("get tag_id:%s" % tag_id)
tags = Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True)
logging.info("get tags name :%s" % tags)
for i in tags:
tag_name_list.append(i)
logging.info("get tags name i:%s" % i)
return tag_name_list
......
trans2es/models/topic.py
View file @ 0ad7f0ca
......@@ -52,9 +52,9 @@ class Topic(models.Model):
id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
name = models.CharField(verbose_name=u'日记名称', max_length=100)
# group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
# group = models.ForeignKey(
# Group, verbose_name=u"关联的小组", related_name=u"group_topics", null=True, blank=True, default=None,
# on_delete=models.CASCADE)
group = models.ForeignKey(
Group, verbose_name=u"关联的小组", related_name=u"group_topics", null=True, blank=True, default=None,
on_delete=models.CASCADE)
user_id = models.IntegerField(verbose_name=u'用户ID')
has_video = models.BooleanField(verbose_name=u'是否是视频日记')
drop_score = models.IntegerField(verbose_name=u'人工赋分', default=0)
......@@ -106,8 +106,7 @@ class Topic(models.Model):
try:
has_image = False
query_list = TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, is_deleted=False,
is_online=True)
query_list = TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, is_deleted=False, is_online=True)
if len(query_list) > 0:
has_image = True
......@@ -133,13 +132,12 @@ class Topic(models.Model):
topic_tag_id_list = list()
edit_tag_id_list = list()
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id).values_list("tag_id",
flat=True)
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id).values_list("tag_id", flat=True)
tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=tag_id_list)
for tag_item in tag_query_results:
is_online = tag_item.is_online
is_deleted = tag_item.is_deleted
collection = tag_item.collection
is_online=tag_item.is_online
is_deleted=tag_item.is_deleted
collection=tag_item.collection
if is_online and not is_deleted:
topic_tag_id_list.append(tag_item.id)
......@@ -149,7 +147,7 @@ class Topic(models.Model):
return (topic_tag_id_list, edit_tag_id_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([], [])
return ([],[])
def get_tag_name_list(self, tag_id_list):
try:
......@@ -188,12 +186,9 @@ class Topic(models.Model):
elif self.content_level == '3':
offline_score += 2.0
exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id,
data_type=1).count()
click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id,
data_type=2).count()
uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id,
data_type=3).count()
exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
if exposure_count > 0:
offline_score += click_count / exposure_count
......@@ -224,8 +219,7 @@ class TopicComplaint(models.Model):
id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
user_id = models.BigIntegerField(verbose_name=u'用户ID', db_index=True)
topic = models.ForeignKey(
Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE,
related_name='complaints')
Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE, related_name='complaints')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
......
trans2es/type_info.py
View file @ 0ad7f0ca
......@@ -81,6 +81,9 @@ class TypeInfo(object):
def bulk_get_data(self, instance_iterable):
data_list = []
# 4，5星帖子单独索引
topic_data_high_star_list = list()
if self.batch_get_data_func:
_pk_list = [getattr(instance, 'pk', None) for instance in instance_iterable]
not_found_pk_list = []
......@@ -137,7 +140,10 @@ class TypeInfo(object):
))
else:
data_list.append(data)
return data_list
if self.type=="topic" and instance.content_level and int(instance.content_level)>=4:
topic_data_high_star_list.append(data)
return (data_list,topic_data_high_star_list)
def elasticsearch_bulk_insert_data(self, sub_index_name, data_list, es=None):
......@@ -162,10 +168,10 @@ class TypeInfo(object):
# traceback.print_exc()
# es_result = 'error'
return ESPerform.es_helpers_bulk(es, data_list, sub_index_name, True)
return ESPerform.es_helpers_bulk(es, data_list, sub_index_name)
def elasticsearch_bulk_insert(self, sub_index_name, instance_iterable, es=None):
data_list = self.bulk_get_data(instance_iterable)
data_list, topic_data_high_star_list = self.bulk_get_data(instance_iterable)
return self.elasticsearch_bulk_insert_data(
sub_index_name=sub_index_name,
data_list=data_list,
......@@ -188,7 +194,7 @@ class TypeInfo(object):
time1=end-begin
begin = time.time()
data_list = self.bulk_get_data(instance_list)
data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
end = time.time()
time2=end-begin
......@@ -201,6 +207,14 @@ class TypeInfo(object):
data_list=data_list,
es=es,
)
# 同时写4星及以上的帖子
if len(topic_data_high_star_list)>0:
self.elasticsearch_bulk_insert_data(
sub_index_name="topic-high-star",
data_list=topic_data_high_star_list,
es=es,
)
end = time.time()
time3=end-begin
logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0,time1,time2,time3))
......@@ -215,7 +229,7 @@ class TypeInfo(object):
stage_1_time = time.time()
data_list = self.bulk_get_data(instance_list)
data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
stage_2_time = time.time()
......@@ -260,7 +274,16 @@ def get_type_info_map():
return _get_type_info_map_result
type_info_list = [
TypeInfo(
name='topic-high-star', # >=4星日记
type='topic-high-star',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,
get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='topic', # 日记
type='topic',
......
trans2es/utils/group_transfer.py
View file @ 0ad7f0ca
......@@ -33,7 +33,7 @@ class GroupTransfer(object):
update_time = instance.update_time
tzlc_udpate_time = tzlc(update_time)
res["update_time"] = tzlc_udpate_time
# res["high_quality_topic_num"] = instance.get_high_quality_topic_num()
res["high_quality_topic_num"] = instance.get_high_quality_topic_num()
return res
except:
......
trans2es/utils/topic_transfer.py
View file @ 0ad7f0ca
......@@ -28,11 +28,11 @@ class TopicTransfer(object):
res["content_level"] = instance.content_level
res["user_id"] = instance.user_id
# if instance.group:
# res["group_id"] = instance.group.id
# else:
# res["group_id"] = -1
if instance.group:
res["group_id"] = instance.group.id
else:
res["group_id"] = -1
res["share_num"] = instance.share_num
res["pictorial_id"] = instance.get_pictorial_id()
......@@ -82,6 +82,20 @@ class TopicTransfer(object):
# else:
# res["language_type"] = instance.language_type
res["is_shadow"] = instance.is_shadow
res["is_recommend"] = True if instance.is_recommend else False
res["is_complaint"] = instance.is_complaint
res["virtual_content_level"] = instance.virtual_content_level
res["like_num_crawl"] = instance.like_num_crawl
res["comment_num_crawl"]= instance.comment_num_crawl
res["is_crawl"] = instance.is_crawl
res["platform"] = instance.platform
res["platform_id"] = instance.platform_id
res["drop_score"] = instance.drop_score
res["sort_score"] = instance.sort_score
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
......@@ -91,11 +105,11 @@ class TopicTransfer(object):
update_time = instance.update_time
tzlc_update_time = tzlc(update_time)
# res["update_time"] = tzlc_update_time
res["update_time"] = tzlc_update_time
res["update_time_val"] = int(time.mktime(tzlc_update_time.timetuple()))
logging.info("test topic transfer time cost,time0:%d,time1:%d,time2:%d,time3:%d,time4:%d" % (time0,time1,time2,time3,time4))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
\ No newline at end of file
return None
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment