Skip to content

C
cybertron
Commit 5686973a authored by lixiaofang's avatar lixiaofang
Browse files
Options

S

parents b9e8c2b3 44b068ac
Hide whitespace changes
Inline Side-by-side
Showing with 2767 additions and 127 deletions
.idea/workspace.xml
View file @ 5686973a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<<<<<<< HEAD
<list default="true" id="7b835479-ee84-40d6-9b97-0e9285d92119" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
=======
<list default="true" id="47bd7678-47c9-4681-b4c9-928e6d933349" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/index_contrast/product.py" afterDir="false" />
>>>>>>> type_majia
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
......@@ -12,46 +17,75 @@
</component>
<component name="FUSProjectUsageTrigger">
<session id="-334786743">
<usages-collector id="statistics.file.extensions.open">
<usages-collector id="statistics.file.extensions.edit">
<counts>
<<<<<<< HEAD
<entry key="README" value="1" />
<entry key="batch_user_gen" value="1" />
<entry key="guanshui" value="1" />
<entry key="py" value="61" />
<entry key="txt" value="7" />
=======
<entry key="py" value="158" />
>>>>>>> type_majia
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<usages-collector id="statistics.file.types.edit">
<counts>
<<<<<<< HEAD
<entry key="PLAIN_TEXT" value="10" />
<entry key="Python" value="61" />
=======
<entry key="Python" value="158" />
>>>>>>> type_majia
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<usages-collector id="statistics.file.extensions.open">
<counts>
<<<<<<< HEAD
<entry key="dummy" value="1" />
<entry key="py" value="2480" />
=======
<entry key="batch_user_gen" value="2" />
<entry key="guanshui" value="4" />
<entry key="py" value="78" />
<entry key="txt" value="8" />
>>>>>>> type_majia
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<usages-collector id="statistics.file.types.open">
<counts>
<<<<<<< HEAD
<entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="2480" />
=======
<entry key="PLAIN_TEXT" value="14" />
<entry key="Python" value="78" />
>>>>>>> type_majia
</counts>
</usages-collector>
<usages-collector id="statistics.lifecycle.project">
<counts>
<<<<<<< HEAD
<entry key="project.closed" value="7" />
<entry key="project.open.time.1" value="1" />
<entry key="project.open.time.17" value="1" />
<entry key="project.open.time.2" value="1" />
<entry key="project.open.time.3" value="1" />
=======
<entry key="project.closed" value="6" />
<entry key="project.open.time.1" value="1" />
<entry key="project.open.time.11" value="1" />
<entry key="project.open.time.19" value="1" />
<entry key="project.open.time.2" value="1" />
>>>>>>> type_majia
<entry key="project.opened" value="4" />
</counts>
</usages-collector>
</session>
</component>
<component name="FileEditorManager">
<<<<<<< HEAD
<splitter split-orientation="horizontal" split-proportion="0.2859412">
<split-first>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
......@@ -95,16 +129,32 @@
</leaf>
</split-second>
</splitter>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
=======
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="101">
<caret line="197" column="69" selection-start-line="197" selection-start-column="12" selection-end-line="197" selection-end-column="69" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/index_contrast/product.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-439">
<caret line="373" column="39" lean-forward="true" selection-start-line="373" selection-start-column="39" selection-end-line="373" selection-end-column="39" />
</state>
</provider>
</entry>
</file>
</leaf>
>>>>>>> type_majia
</component>
<component name="FindInProjectRecents">
<findStrings>
<<<<<<< HEAD
<find>/da</find>
<find>existing</find>
<find>mv-alpha-pictorial-test-190716901</find>
......@@ -122,6 +172,9 @@
<find>/api/v1/reply/create</find>
<find>print</find>
<find>email</find>
=======
<find>get_error</find>
>>>>>>> type_majia
</findStrings>
</component>
<component name="Git.Settings">
......@@ -130,14 +183,9 @@
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/answer2.txt" />
<option value="$PROJECT_DIR$/answer1.txt" />
<option value="$PROJECT_DIR$/answer_reply.py" />
<option value="$PROJECT_DIR$/reply_comment1.py" />
<option value="$PROJECT_DIR$/test.py" />
<option value="$PROJECT_DIR$/index_contrast/tag.py" />
<option value="$PROJECT_DIR$/index_contrast/user.py" />
<option value="$PROJECT_DIR$/one_by_one.py" />
<option value="$PROJECT_DIR$/index_contrast/pictorial.py" />
<<<<<<< HEAD
<option value="$PROJECT_DIR$/answer_reply1.py" />
<option value="$PROJECT_DIR$/reply_comment2.py" />
<option value="$PROJECT_DIR$/answer_reply3.py" />
......@@ -156,6 +204,23 @@
<option name="y" value="-1080" />
<option name="width" value="1920" />
<option name="height" value="1080" />
=======
<option value="$PROJECT_DIR$/index_contrast/user.py" />
<option value="$PROJECT_DIR$/index_contrast/tag.py" />
<option value="$PROJECT_DIR$/answer_reply1.py" />
<option value="$PROJECT_DIR$/index_contrast/topic.py" />
<option value="$PROJECT_DIR$/auto_request.py" />
<option value="$PROJECT_DIR$/auto_click_51.py" />
<option value="$PROJECT_DIR$/index_contrast/product.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="x" value="-11" />
<option name="y" value="23" />
<option name="width" value="1280" />
<option name="height" value="727" />
>>>>>>> type_majia
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
......@@ -170,15 +235,26 @@
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" />
<item name="index_contrast" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
</panes>
</component>
<component name="PropertiesComponent">
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<<<<<<< HEAD
<recent name="$PROJECT_DIR$" />
=======
>>>>>>> type_majia
<recent name="$PROJECT_DIR$/index_contrast" />
</key>
</component>
......@@ -194,35 +270,12 @@
</list>
</option>
</component>
<component name="RunManager">
<configuration name="Unnamed" type="PythonConfigurationType" factoryName="Python">
<module name="cybertron" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="$USER_HOME$/LiXiaoFang/bin/python3.6" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="7b835479-ee84-40d6-9b97-0e9285d92119" name="Default Changelist" comment="" />
<changelist id="47bd7678-47c9-4681-b4c9-928e6d933349" name="Default Changelist" comment="" />
<created>1542787882519</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
......@@ -236,7 +289,11 @@
<layout>
<window_info id="Structure" order="0" sideWeight="0.21188119" side_tool="true" weight="0.19112115" />
<window_info id="Favorites" order="1" side_tool="true" />
<<<<<<< HEAD
<window_info content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.072143234" />
=======
<window_info active="true" content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.21169037" />
>>>>>>> type_majia
<window_info id="DB Browser" order="3" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.3284974" />
......@@ -246,9 +303,15 @@
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<<<<<<< HEAD
<window_info active="true" anchor="bottom" id="Terminal" order="8" sideWeight="0.4988086" visible="true" weight="0.27253887" />
<window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.50119144" side_tool="true" weight="0.32843137" />
<window_info anchor="bottom" id="Python Console" order="10" weight="0.32953367" />
=======
<window_info anchor="bottom" id="Terminal" order="8" visible="true" weight="0.06321243" />
<window_info anchor="bottom" id="Event Log" order="9" side_tool="true" />
<window_info anchor="bottom" id="Python Console" order="10" />
>>>>>>> type_majia
<window_info anchor="bottom" id="DB Execution Console" order="11" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
......@@ -257,7 +320,11 @@
<layout-to-restore>
<window_info id="Structure" order="0" sideWeight="0.21188119" side_tool="true" weight="0.19112115" />
<window_info id="Favorites" order="1" side_tool="true" />
<<<<<<< HEAD
<window_info active="true" content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.29626688" />
=======
<window_info active="true" content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.20333599" />
>>>>>>> type_majia
<window_info id="DB Browser" order="3" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.3284974" />
......@@ -267,8 +334,13 @@
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<<<<<<< HEAD
<window_info anchor="bottom" id="Terminal" order="8" sideWeight="0.4988086" visible="true" weight="0.0" />
<window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.50119144" side_tool="true" weight="0.32843137" />
=======
<window_info anchor="bottom" id="Terminal" order="8" visible="true" weight="0.0" />
<window_info anchor="bottom" id="Event Log" order="9" side_tool="true" />
>>>>>>> type_majia
<window_info anchor="bottom" id="Python Console" order="10" />
<window_info anchor="bottom" id="DB Execution Console" order="11" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
......@@ -297,20 +369,10 @@
<option name="timeStamp" value="9" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/index_contrast/tag.py</url>
<line>170</line>
<url>file://$PROJECT_DIR$/auto_request.py</url>
<line>178</line>
<option name="timeStamp" value="10" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/index_contrast/tag.py</url>
<line>21</line>
<option name="timeStamp" value="12" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/index_contrast/pictorial.py</url>
<line>190</line>
<option name="timeStamp" value="14" />
</line-breakpoint>
</breakpoints>
<breakpoints-dialog>
<breakpoints-dialog />
......@@ -325,21 +387,30 @@
</breakpoint-manager>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/auto_urge_three1.py">
<entry file="file://$PROJECT_DIR$/test.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="340">
<caret line="44" column="36" selection-start-line="44" selection-start-column="36" selection-end-line="44" selection-end-column="36" />
<state relative-caret-position="390">
<caret line="28" column="12" selection-start-line="28" selection-start-column="12" selection-end-line="28" selection-end-column="12" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_urge_three.py">
<entry file="file://$PROJECT_DIR$/auto_click2.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="570">
<caret line="49" column="29" selection-start-line="49" selection-start-column="29" selection-end-line="49" selection-end-column="29" />
<state relative-caret-position="165">
<caret line="25" column="31" lean-forward="true" selection-start-line="25" selection-start-column="31" selection-end-line="25" selection-end-column="31" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_one2.py">
<entry file="file://$PROJECT_DIR$/auto_click4.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_majia.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_star_topic.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<folding>
......@@ -348,20 +419,27 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_lunch_app.py">
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/answer_reply2.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="495">
<caret line="42" column="45" selection-start-line="42" selection-start-column="45" selection-end-line="42" selection-end-column="45" />
<state relative-caret-position="120">
<caret line="15" column="67" selection-start-line="15" selection-start-column="67" selection-end-line="15" selection-end-column="67" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_lunch_app2.py">
<entry file="file://$PROJECT_DIR$/answer_reply3.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="145">
<caret line="25" column="12" lean-forward="true" selection-start-line="25" selection-start-column="12" selection-end-line="25" selection-end-column="12" />
<state relative-caret-position="262">
<caret line="201" column="12" selection-start-line="201" selection-start-column="12" selection-end-line="205" selection-end-column="69" />
</state>
</provider>
</entry>
<<<<<<< HEAD
<entry file="file://$PROJECT_DIR$/auto_follow_one.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="330">
......@@ -370,72 +448,148 @@
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
=======
<entry file="file://$PROJECT_DIR$/auto_click3.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_click_fan.py">
<entry file="file://$PROJECT_DIR$/auto_comment1_fan.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_comment1.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_comment2.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/reply_comment1.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/reply_comment2.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/reply_comment3.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_comment.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="547">
<caret line="56" column="73" selection-start-line="56" selection-start-column="73" selection-end-line="56" selection-end-column="73" />
<state>
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_star_topic.py">
<entry file="file://$PROJECT_DIR$/auto_comment3.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180">
<caret line="21" column="61" selection-start-line="21" selection-start-column="61" selection-end-line="21" selection-end-column="61" />
<state relative-caret-position="228">
<caret line="34" column="68" lean-forward="true" selection-start-line="34" selection-start-column="68" selection-end-line="34" selection-end-column="68" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_comment_51.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_every_day.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-4" />
>>>>>>> type_majia
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow2.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_lunch_app.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-117" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_lunch_app2.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-97" />
</provider>
</entry>
<<<<<<< HEAD
<entry file="file://$PROJECT_DIR$/auto_comment.py">
=======
<entry file="file://$PROJECT_DIR$/auto_urge1.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="195">
<caret line="13" column="13" selection-end-line="90" />
</state>
<state relative-caret-position="-126" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_urge2.py">
>>>>>>> type_majia
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-207" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/answer2.txt">
<entry file="file://$PROJECT_DIR$/auto_follow_new.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" column="6" lean-forward="true" selection-start-line="17" selection-start-column="6" selection-end-line="17" selection-end-column="6" />
<state relative-caret-position="165">
<caret line="20" selection-start-line="20" selection-end-line="20" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<<<<<<< HEAD
<entry file="file://$PROJECT_DIR$/comment.txt">
<provider selected="true" editor-type-id="text-editor" />
=======
<entry file="file://$PROJECT_DIR$/auto_star_urge.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="645">
<caret line="52" selection-start-line="52" selection-end-line="52" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
>>>>>>> type_majia
</entry>
<entry file="file://$PROJECT_DIR$/answer1.txt">
<entry file="file://$PROJECT_DIR$/user_id.txt">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_one.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="11" column="10" lean-forward="true" selection-start-line="11" selection-start-column="10" selection-end-line="11" selection-end-column="10" />
<state relative-caret-position="330">
<caret line="31" selection-start-line="31" selection-end-line="32" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_comment1.py">
<entry file="file://$PROJECT_DIR$/auto_follow_one2.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="105">
<caret line="7" lean-forward="true" selection-start-line="7" selection-end-line="7" />
<state>
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/README">
<entry file="file://$PROJECT_DIR$/log_settings.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_comment_51.py">
<entry file="file://$PROJECT_DIR$/offline_click.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_every_day.py">
<entry file="file://$PROJECT_DIR$/batch_user_gen">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_new.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-100" />
</provider>
<entry file="file://$PROJECT_DIR$/user_session.txt">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow_majia.py">
<entry file="file://$PROJECT_DIR$/comment.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-69" />
<state relative-caret-position="-8488" />
</provider>
</entry>
<<<<<<< HEAD
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py" />
<entry file="file://$USER_HOME$/.local/lib/python3.6/site-packages/requests/__init__.py">
<provider selected="true" editor-type-id="text-editor" />
......@@ -444,18 +598,26 @@
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/auto_follow2.py">
=======
<entry file="file://$PROJECT_DIR$/guanshui">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-8488">
<caret selection-end-column="5" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/answer1.txt">
>>>>>>> type_majia
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/test.py">
<entry file="file://$PROJECT_DIR$/answer_reply1.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="8" column="8" lean-forward="true" selection-start-line="8" selection-start-column="8" selection-end-line="8" selection-end-column="8" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
<state relative-caret-position="-255">
<caret line="30" column="22" lean-forward="true" selection-start-line="30" selection-start-column="22" selection-end-line="30" selection-end-column="22" />
</state>
</provider>
</entry>
<<<<<<< HEAD
<entry file="file://$PROJECT_DIR$/index_contrast/tag.py" />
<entry file="file://$PROJECT_DIR$/index_contrast/user.py" />
<entry file="file://$PROJECT_DIR$/index_contrast/pictorial.py" />
......@@ -549,10 +711,69 @@
<caret line="22" column="45" selection-start-line="22" selection-start-column="45" selection-end-line="22" selection-end-column="45" />
<folding>
<element signature="e#0#15#0" expanded="true" />
=======
<entry file="file://$PROJECT_DIR$/auto_click_instant.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<folding>
<element signature="e#0#11#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_click.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="73">
<caret line="14" column="23" lean-forward="true" selection-start-line="14" selection-start-column="23" selection-end-line="14" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_click5.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/one_by_one.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-717">
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_request.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="375">
<caret line="210" column="57" selection-start-line="210" selection-start-column="57" selection-end-line="210" selection-end-column="57" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_click_fan.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-260" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_click1.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="18" column="35" lean-forward="true" selection-start-line="17" selection-start-column="12" selection-end-line="18" selection-end-column="35" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_click_51.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="540">
<caret line="36" selection-start-line="36" selection-end-line="37" />
<folding>
<element signature="e#0#11#0" expanded="true" />
>>>>>>> type_majia
</folding>
</state>
</provider>
</entry>
<<<<<<< HEAD
<entry file="file://$PROJECT_DIR$/auto_star_urge.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="645">
......@@ -577,16 +798,27 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="15">
<caret line="18" column="12" lean-forward="true" selection-start-line="18" selection-start-column="12" selection-end-line="18" selection-end-column="12" />
=======
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="101">
<caret line="197" column="69" selection-start-line="197" selection-start-column="12" selection-end-line="197" selection-end-column="69" />
>>>>>>> type_majia
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/auto_request.py">
<entry file="file://$PROJECT_DIR$/index_contrast/product.py">
<provider selected="true" editor-type-id="text-editor">
<<<<<<< HEAD
<state relative-caret-position="5130">
<caret line="342" selection-start-line="342" selection-end-line="342" />
<folding>
<element signature="e#0#15#0" expanded="true" />
</folding>
=======
<state relative-caret-position="-439">
<caret line="373" column="39" lean-forward="true" selection-start-line="373" selection-start-column="39" selection-end-line="373" selection-end-column="39" />
>>>>>>> type_majia
</state>
</provider>
</entry>
......
auto_click_51.py
View file @ 5686973a
......@@ -4,7 +4,7 @@ import random
import traceback
from log_settings import *
import logging
from auto_request import login, click, time_conv
from auto_request import login, click, time_convs
from auto_request import host, user, db, passwd
......@@ -13,24 +13,37 @@ def get_data(numtime, numtime2):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
cursor.execute(
"select id from topic where content_level in (4,5,6) and create_time > '2019-05-01 00:00:00.0000' and id not in (select topic_id from reply )")
"SELECT distinct(user_id) FROM topic WHERE is_online=1 and content_level in (4,5,6) and (create_time >= '%s' and create_time <= '%s')" % (
numtime, numtime2))
data = cursor.fetchall()
topic_id = list(data)
topic_id_list = []
for i in topic_id and topic_id:
cursor.execute(
"select user_id,is_shadow from user_extra where user_id =" + str(
i[0]) + " and is_online =1 and is_deleted =0")
data = cursor.fetchall()
user_id = list(data)
if (user_id and user_id[0][1] == 0) or len(user_id) == 0:
topic_id_list.append(i)
logging.info("Database version : %s " % topic_id_list)
pc.close()
return topic_id
return topic_id_list
except:
logging.error("catch exception,get_data:%s" % traceback.format_exc())
if __name__ == "__main__":
try:
numtime, numtime2 = time_conv(15, 5)
numtime, numtime2 = time_convs(1, 7)
user_id = get_data(numtime, numtime2)
dicts = {}
logging.info("get user_id:%s" % user_id)
for i in user_id:
cook = login()
if cook is not None:
click(cook, i)
rand_num = random.randint(0, 1)
if rand_num == 1:
cook = login()
if cook is not None:
click(cook, i[1])
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
auto_request.py
View file @ 5686973a
......@@ -10,28 +10,29 @@ import redis
import smtplib
from email.mime.text import MIMEText
from email.utils import formataddr
from collections import Counter
my_sender = 'lixiaofang@igengmei.com'
my_pass = 'tg5AVKBB8jLQGBET'
my_user6 = "lixiaofang@igengmei.com"
auto_click_url = "http://earth.gmapp.env/api/v1/like"
auto_reply_url = "http://earth.gmapp.env/api/v1/reply/create"
auto_follow_url = "http://earth.gmapp.env/api/v1/follow"
auto_urge_url = "http://earth.gmapp.env/api/v1/user/urge"
auto_click_url = "http://earth.iyanzhi.com/api/v1/like"
auto_reply_url = "http://earth.iyanzhi.com/api/v1/reply/create"
auto_follow_url = "http://earth.iyanzhi.com/api/v1/follow"
auto_urge_url = "http://earth.iyanzhi.com/api/v1/user/urge"
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'alpha',
'USER': 'work',
'PASSWORD': 'Gengmei123!',
'HOST': '172.21.36.16',
'PASSWORD': 'oars152!traipseize738',
'HOST': '172.17.40.136',
'PORT': '3306',
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4",
},
}
}
}
......@@ -136,7 +137,8 @@ def click(cookies_get, id):
data=post_dict)
logging.info("response.text:%s" % response.text)
get_error(response.text)
get_error(response.text, "click", id)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
......@@ -154,7 +156,7 @@ def reply(cookies_get, id, content):
data=post_dict)
logging.info("response.text:%s" % response.text)
get_error(response.text)
get_error(response.text, "reply", id)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
......@@ -219,6 +221,9 @@ def time_convs(numtime, numtime2):
yes_time_str2 = yes_time2.strftime('%Y-%m-%d')
yes_time_str2 = yes_time_str2 + ' 00:00:00.000000'
logging.info("get yes_time_str:%s"%yes_time_str)
logging.info("get yes_time_str2:%s"%yes_time_str2)
return yes_time_str, yes_time_str2
except:
return None
......@@ -232,9 +237,7 @@ def get_comment():
list_guanshui.append(i)
num = random.randint(0, len(list_guanshui))
comment = list_guanshui[num - 1]
return comment
except:
......@@ -295,7 +298,7 @@ def follow(cookies_get, id):
data=post_dict)
logging.info("response.text:%s" % response.text)
get_error(response.text)
get_error(response.text, "follow", id)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
......@@ -311,7 +314,7 @@ def Urge(cookies_get, id):
data=post_dict)
logging.info("response.text:%s" % response.text)
get_error(response.text)
get_error(response.text, "Urge", id)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
......@@ -387,12 +390,12 @@ def send_email(stat_data):
logging.error("catch exception,main:%s" % traceback.format_exc())
def get_error(line):
def get_error(line, type, id):
try:
if len(line) > 1:
data_dict = json.loads(line)
if data_dict["error"] != 0:
stat_data = str(line)
stat_data = str(line) + str(type) + str(id)
ret = send_email(stat_data)
if ret:
logging.info('like query统计邮件发送成功')
......@@ -449,7 +452,7 @@ def reply_answer(cookies_get, id, content):
data=post_dict)
logging.info("response.text:%s" % response.text)
get_error(response.text)
get_error(response.text, "reply", id)
return response.text
except:
......@@ -459,7 +462,7 @@ def reply_answer(cookies_get, id, content):
def set_reply_to_redis():
try:
redis_url = "redis://:Gengmei123@172.21.52.16:6379/15"
redis_url = "redis://:gHtmeIqBhVHtH8MG@172.17.40.140:6379/1"
redis_client = redis.StrictRedis.from_url(redis_url)
return redis_client
......@@ -480,7 +483,7 @@ def reply2(cookies_get, id, content, replied_id):
data=post_dict)
logging.info("response.text:%s" % response.text)
get_error(response.text)
get_error(response.text, "reply2", id)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
index_contrast/pictorial.py 0 → 100644
View file @ 5686973a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
import os.path
import re
import json
from elasticsearch import Elasticsearch
import elasticsearch.helpers
from django.conf import settings
ES_INFO_LIST = [
{
"host": "172.17.32.22",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object):
cli_obj = None
cli_info_list = ES_INFO_LIST
index_prefix = ES_INDEX_PREFIX
@classmethod
def get_cli(cls, cli_info=None):
try:
init_args = {
'sniff_on_start': False,
'sniff_on_connection_fail': False,
}
es_cli_info = cli_info if cli_info else cls.cli_info_list
cls.cli_obj = Elasticsearch(hosts=es_cli_info, **init_args)
return cls.cli_obj
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def get_official_index_name(cls, sub_index_name, index_flag=None):
"""
:remark:get official es index name
:param sub_index_name:
:param index_flag:
:return:
"""
try:
index_flag = None
assert (index_flag in [None, "read", "write"])
official_index_name = cls.index_prefix + "-" + sub_index_name
if index_flag:
official_index_name += "-" + index_flag
return official_index_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def __load_mapping(cls, doc_type):
try:
mapping_file_path = os.path.join(
os.path.dirname(__file__),
'..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
mapping = ''
with open(mapping_file_path, 'r') as f:
for line in f:
# 去掉注释
mapping += re.sub(r'//.*$', '', line)
mapping = json.loads(mapping)
return mapping
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def create_index(cls, es_cli, sub_index_name):
"""
:remark: create es index,alias index
:param sub_index_name:
:return:
"""
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name)
index_exist = es_cli.indices.exists(official_index_name)
if not index_exist:
es_cli.indices.create(official_index_name)
read_alias_name = cls.get_official_index_name(sub_index_name, "read")
es_cli.indices.put_alias(official_index_name, read_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
es_cli.indices.put_alias(official_index_name, write_alias_name)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc", force_sync=False):
"""
:remark: put index mapping
:param es_cli:
:param sub_index_name:
:param mapping_type:
:return:
"""
try:
assert (es_cli is not None)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
index_exist = es_cli.indices.exists(write_alias_name)
if not index_exist and not force_sync:
return False
mapping_dict = cls.__load_mapping(sub_index_name)
es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_indices_template(cls, es_cli, template_file_name, template_name):
"""
:remark put index template
:param es_cli:
:param template_file_name:
:param template_name:
:return:
"""
try:
assert (es_cli is not None)
mapping_dict = cls.__load_mapping(template_file_name)
es_cli.indices.put_template(name=template_name, body=mapping_dict)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
return False
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
bulk_actions = []
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False,
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-pictorial-test-190717904":
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
if not batch_search:
if not routing:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size)
else:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size, routing=routing)
if is_suggest_request:
return res
else:
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
res = es_cli.msearch(body=query_body, index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def if_es_node_load_high(cls, es_cli):
try:
assert (es_cli is not None)
high_num = 0
es_nodes_list = list()
es_nodes_ori_info = es_cli.cat.nodes()
es_nodes_info_list = es_nodes_ori_info.split("\n")
for item in es_nodes_info_list:
try:
item_list = item.split(" ")
if len(item_list) == 11:
cpu_load = item_list[4]
elif len(item_list) == 10:
cpu_load = item_list[3]
else:
continue
int_cpu_load = int(cpu_load)
if int_cpu_load > 60:
high_num += 1
es_nodes_list.append(int_cpu_load)
except:
logging.error("catch exception,item:%s,err_msg:%s" % (str(item), traceback.format_exc()))
return True
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"range": {"update_time": {"gte": "2019-05-09T00:00:00+00:00"}}}
]
}
}
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, 0, 1000)
result_dict_ids = list()
for old_item in result_dict["hits"]:
old_source = old_item["_source"]
old_id = old_source["id"]
# 获取新index
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"id": old_id}},
{"term": {"is_online": True}}
]
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-pictorial-test-190717904", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
print(new_id)
print("-----id-----")
if old_source["is_online"] != new_source["is_online"]:
print(old_source["is_online"])
print(new_source["is_online"])
print("-----is_online-----")
if old_source["is_deleted"] != new_source["is_deleted"]:
print(old_source["is_deleted"])
print(new_source["is_deleted"])
print("-----is_deleted-----")
if old_source["is_recommend"] != new_source["is_recommend"]:
print(old_source["is_recommend"])
print(new_source["is_recommend"])
print("-----is_recommend-----")
if old_source["name"] != new_source["name"]:
print(old_source["name"])
print(new_source["name"])
print("-----name-----")
if old_source["description"] != new_source["description"]:
print(old_source["description"])
print(new_source["description"])
print("-----description-----")
if old_source["topic_num"] != new_source["topic_num"]:
print(old_source["topic_num"])
print(new_source["topic_num"])
print("-----topic_num-----")
if old_source["creator_id"] != new_source["creator_id"]:
print(old_source["creator_id"])
print(new_source["creator_id"])
print("-----creator_id-----")
if old_source["icon"] != new_source["icon"]:
print(old_source["icon"])
print(new_source["icon"])
print("-----icon-----")
if old_source["high_quality_topic_num"] != new_source["high_quality_topic_num"]:
print(old_source["high_quality_topic_num"])
print(new_source["high_quality_topic_num"])
print("-----high_quality_topic_num-----")
if old_source["create_time"] != new_source["create_time"]:
print(old_source["create_time"])
print(new_source["create_time"])
print("-----create_time-----")
if old_source["update_time"] != new_source["update_time"]:
print(old_source["update_time"])
print(new_source["update_time"])
print("-----update_time-----")
if old_source["tag_id"] != new_source["tag_id"]:
print(old_source["tag_id"])
print(new_source["tag_id"])
print("-----tag_id-----")
if old_source["tag_name"] != new_source["tag_name"]:
print(old_source["tag_name"])
print(new_source["tag_name"])
print("-----tag_name-----")
if old_source["topic_id_list"] != new_source["topic_id_list"]:
print(old_source["topic_id_list"])
print(new_source["topic_id_list"])
print("-----topic_id_list-----")
if old_source["effective"] != new_source["effective"]:
print(old_source["effective"])
print(new_source["effective"])
print("-----effective-----")
if old_source["offline_score"] != new_source["offline_score"]:
print(old_source["offline_score"])
print(new_source["offline_score"])
print("-----offline_score-----")
if old_source["is_default"] != new_source["is_default"]:
print(old_source["is_default"])
print(new_source["is_default"])
print("-----is_default-----")
if old_source["is_cover"] != new_source["is_cover"]:
print(old_source["is_cover"])
print(new_source["is_cover"])
print("-----is_cover-----")
if old_source["topic_vote_number"] != new_source["topic_vote_number"]:
print(old_source["topic_vote_number"])
print(new_source["topic_vote_number"])
print("-----topic_vote_number-----")
if old_source["activity_join"] != new_source["activity_join"]:
print(old_source["activity_join"])
print(new_source["activity_join"])
print("-----activity_join-----")
index_contrast/product.py 0 → 100644
View file @ 5686973a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
import os.path
import re
import json
from elasticsearch import Elasticsearch
import elasticsearch.helpers
from django.conf import settings
ES_INFO_LIST = [
{
"host": "172.17.32.22",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object):
cli_obj = None
cli_info_list = ES_INFO_LIST
index_prefix = ES_INDEX_PREFIX
@classmethod
def get_cli(cls, cli_info=None):
try:
init_args = {
'sniff_on_start': False,
'sniff_on_connection_fail': False,
}
es_cli_info = cli_info if cli_info else cls.cli_info_list
cls.cli_obj = Elasticsearch(hosts=es_cli_info, **init_args)
return cls.cli_obj
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def get_official_index_name(cls, sub_index_name, index_flag=None):
"""
:remark:get official es index name
:param sub_index_name:
:param index_flag:
:return:
"""
try:
index_flag = None
assert (index_flag in [None, "read", "write"])
official_index_name = cls.index_prefix + "-" + sub_index_name
if index_flag:
official_index_name += "-" + index_flag
return official_index_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def __load_mapping(cls, doc_type):
try:
mapping_file_path = os.path.join(
os.path.dirname(__file__),
'..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
mapping = ''
with open(mapping_file_path, 'r') as f:
for line in f:
# 去掉注释
mapping += re.sub(r'//.*$', '', line)
mapping = json.loads(mapping)
return mapping
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def create_index(cls, es_cli, sub_index_name):
"""
:remark: create es index,alias index
:param sub_index_name:
:return:
"""
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name)
index_exist = es_cli.indices.exists(official_index_name)
if not index_exist:
es_cli.indices.create(official_index_name)
read_alias_name = cls.get_official_index_name(sub_index_name, "read")
es_cli.indices.put_alias(official_index_name, read_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
es_cli.indices.put_alias(official_index_name, write_alias_name)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc", force_sync=False):
"""
:remark: put index mapping
:param es_cli:
:param sub_index_name:
:param mapping_type:
:return:
"""
try:
assert (es_cli is not None)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
index_exist = es_cli.indices.exists(write_alias_name)
if not index_exist and not force_sync:
return False
mapping_dict = cls.__load_mapping(sub_index_name)
es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_indices_template(cls, es_cli, template_file_name, template_name):
"""
:remark put index template
:param es_cli:
:param template_file_name:
:param template_name:
:return:
"""
try:
assert (es_cli is not None)
mapping_dict = cls.__load_mapping(template_file_name)
es_cli.indices.put_template(name=template_name, body=mapping_dict)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
return False
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
bulk_actions = []
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False,
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-product-test-190731001":
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
if not batch_search:
if not routing:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size)
else:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size, routing=routing)
if is_suggest_request:
return res
else:
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
res = es_cli.msearch(body=query_body, index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def if_es_node_load_high(cls, es_cli):
try:
assert (es_cli is not None)
high_num = 0
es_nodes_list = list()
es_nodes_ori_info = es_cli.cat.nodes()
es_nodes_info_list = es_nodes_ori_info.split("\n")
for item in es_nodes_info_list:
try:
item_list = item.split(" ")
if len(item_list) == 11:
cpu_load = item_list[4]
elif len(item_list) == 10:
cpu_load = item_list[3]
else:
continue
int_cpu_load = int(cpu_load)
if int_cpu_load > 60:
high_num += 1
es_nodes_list.append(int_cpu_load)
except:
logging.error("catch exception,item:%s,err_msg:%s" % (str(item), traceback.format_exc()))
return True
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
]
}
}
result_dict = ESPerform.get_search_results(es_cli_obj, "product", q, 0, 1000)
result_dict_ids = list()
for old_item in result_dict["hits"]:
old_source = old_item["_source"]
old_id = old_source["id"]
# 获取新index
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"id": old_id}}
]
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-product-test-190731001", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
print(new_id)
print("-----id-----")
if old_source["is_deleted"] != new_source["is_deleted"]:
print(old_source["is_deleted"])
print(new_source["is_deleted"])
print("-----is_deleted-----")
if old_source["is_online"] != new_source["is_online"]:
print(old_source["is_online"])
print(new_source["is_online"])
print("-----is_online-----")
if old_source["create_time"] != new_source["create_time"]:
print(old_source["create_time"])
print(new_source["create_time"])
print("-----create_time-----")
if old_source["update_time"] != new_source["update_time"]:
print(old_source["update_time"])
print(new_source["update_time"])
print("-----update_time-----")
if old_source["create_time_val"] != new_source["create_time_val"]:
print(old_source["create_time_val"])
print(new_source["create_time_val"])
print("-----create_time_val-----")
if old_source["update_time_val"] != new_source["update_time_val"]:
print(old_source["update_time_val"])
print(new_source["update_time_val"])
print("-----update_time_val-----")
if old_source["price"] != new_source["price"]:
print(old_source["price"])
print(new_source["price"])
print("-----price-----")
if old_source["cn_name_sort"] != new_source["cn_name_sort"]:
print(old_source["cn_name_sort"])
print(new_source["cn_name_sort"])
print("-----cn_name_sort-----")
if old_source["cn_name_pre"] != new_source["cn_name_pre"]:
print(old_source["cn_name_pre"])
print(new_source["cn_name_pre"])
print("-----cn_name_pre-----")
if old_source["en_name_pre"] != new_source["en_name_pre"]:
print(old_source["en_name_pre"])
print(new_source["en_name_pre"])
print("-----en_name_pre-----")
if old_source["alias"] != new_source["alias"]:
print(old_source["alias"])
print(new_source["alias"])
print("-----alias-----")
if old_source["cn_name"] != new_source["cn_name"]:
print(old_source["cn_name"])
print(new_source["cn_name"])
print("-----cn_name-----")
if old_source["alias_pre"] != new_source["alias_pre"]:
print(old_source["alias_pre"])
print(new_source["alias_pre"])
print("-----alias_pre-----")
if old_source["en_name"] != new_source["en_name"]:
print(old_source["en_name"])
print(new_source["en_name"])
print("-----en_name-----")
if old_source["description"] != new_source["description"]:
print(old_source["description"])
print(new_source["description"])
print("-----description-----")
if old_source["have_image"] != new_source["have_image"]:
print(old_source["have_image"])
print(new_source["have_image"])
print("-----have_image-----")
if old_source["comment_nums"] != new_source["comment_nums"]:
print(old_source["comment_nums"])
print(new_source["comment_nums"])
print("-----comment_nums-----")
if old_source["brand_cn_name"] != new_source["brand_cn_name"]:
print(old_source["brand_cn_name"])
print(new_source["brand_cn_name"])
print("-----brand_cn_name-----")
if old_source["brand_en_name"] != new_source["brand_en_name"]:
print(old_source["brand_en_name"])
print(new_source["brand_en_name"])
print("-----brand_en_name-----")
if old_source["brand_alias"] != new_source["brand_alias"]:
print(old_source["brand_alias"])
print(new_source["brand_alias"])
print("-----brand_alias-----")
if old_source["brand_cn_name_pre"] != new_source["brand_cn_name_pre"]:
print(old_source["brand_cn_name_pre"])
print(new_source["brand_cn_name_pre"])
print("-----brand_cn_name_pre-----")
if old_source["brand_en_name_pre"] != new_source["brand_en_name_pre"]:
print(old_source["brand_en_name_pre"])
print(new_source["brand_en_name_pre"])
print("-----brand_en_name_pre-----")
if old_source["category_cn_name"] != new_source["category_cn_name"]:
print(old_source["category_cn_name"])
print(new_source["category_cn_name"])
print("-----category_cn_name-----")
if old_source["category_cn_name_pre"] != new_source["category_cn_name_pre"]:
print(old_source["category_cn_name_pre"])
print(new_source["category_cn_name_pre"])
print("-----category_cn_name_pre-----")
if old_source["effect_cn_name"] != new_source["effect_cn_name"]:
print(old_source["effect_cn_name"])
print(new_source["effect_cn_name"])
print("-----effect_cn_name-----")
if old_source["effect_cn_name_pre"] != new_source["effect_cn_name_pre"]:
print(old_source["effect_cn_name_pre"])
print(new_source["effect_cn_name_pre"])
print("-----effect_cn_name_pre-----")
index_contrast/tag.py 0 → 100644
View file @ 5686973a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
import os.path
import re
import json
from elasticsearch import Elasticsearch
import elasticsearch.helpers
ES_INFO_LIST = [
{
"host": "172.17.32.22",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object):
cli_obj = None
cli_info_list = ES_INFO_LIST
index_prefix = ES_INDEX_PREFIX
@classmethod
def get_cli(cls, cli_info=None):
try:
init_args = {
'sniff_on_start': False,
'sniff_on_connection_fail': False,
}
es_cli_info = cli_info if cli_info else cls.cli_info_list
cls.cli_obj = Elasticsearch(hosts=es_cli_info, **init_args)
return cls.cli_obj
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def get_official_index_name(cls, sub_index_name, index_flag=None):
"""
:remark:get official es index name
:param sub_index_name:
:param index_flag:
:return:
"""
try:
index_flag = None
assert (index_flag in [None, "read", "write"])
official_index_name = cls.index_prefix + "-" + sub_index_name
if index_flag:
official_index_name += "-" + index_flag
return official_index_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def __load_mapping(cls, doc_type):
try:
mapping_file_path = os.path.join(
os.path.dirname(__file__),
'..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
mapping = ''
with open(mapping_file_path, 'r') as f:
for line in f:
# 去掉注释
mapping += re.sub(r'//.*$', '', line)
mapping = json.loads(mapping)
return mapping
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def create_index(cls, es_cli, sub_index_name):
"""
:remark: create es index,alias index
:param sub_index_name:
:return:
"""
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name)
index_exist = es_cli.indices.exists(official_index_name)
if not index_exist:
es_cli.indices.create(official_index_name)
read_alias_name = cls.get_official_index_name(sub_index_name, "read")
es_cli.indices.put_alias(official_index_name, read_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
es_cli.indices.put_alias(official_index_name, write_alias_name)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc", force_sync=False):
"""
:remark: put index mapping
:param es_cli:
:param sub_index_name:
:param mapping_type:
:return:
"""
try:
assert (es_cli is not None)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
index_exist = es_cli.indices.exists(write_alias_name)
if not index_exist and not force_sync:
return False
mapping_dict = cls.__load_mapping(sub_index_name)
es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_indices_template(cls, es_cli, template_file_name, template_name):
"""
:remark put index template
:param es_cli:
:param template_file_name:
:param template_name:
:return:
"""
try:
assert (es_cli is not None)
mapping_dict = cls.__load_mapping(template_file_name)
es_cli.indices.put_template(name=template_name, body=mapping_dict)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
return False
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
bulk_actions = []
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False,
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-prod-190722001":
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
if not batch_search:
if not routing:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size)
else:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size, routing=routing)
if is_suggest_request:
return res
else:
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
res = es_cli.msearch(body=query_body, index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
# official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def if_es_node_load_high(cls, es_cli):
try:
assert (es_cli is not None)
high_num = 0
es_nodes_list = list()
es_nodes_ori_info = es_cli.cat.nodes()
es_nodes_info_list = es_nodes_ori_info.split("\n")
for item in es_nodes_info_list:
try:
item_list = item.split(" ")
if len(item_list) == 11:
cpu_load = item_list[4]
elif len(item_list) == 10:
cpu_load = item_list[3]
else:
continue
int_cpu_load = int(cpu_load)
if int_cpu_load > 60:
high_num += 1
es_nodes_list.append(int_cpu_load)
except:
logging.error("catch exception,item:%s,err_msg:%s" % (str(item), traceback.format_exc()))
return True
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
]
}
}
result_dict = ESPerform.get_search_results(es_cli_obj, "tag", q, 0, 10000)
result_dict_ids = list()
for old_item in result_dict["hits"]:
old_source = old_item["_source"]
old_id = old_source["id"]
# 获取新index
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"id": old_id}}
]
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-tag-prod-190722001", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
print(new_id)
print("-----id-----")
if old_source["name"] != new_source["name"]:
print(old_source["name"])
print(new_source["name"])
print("-----name-----")
if old_source["tag_type"] != new_source["tag_type"]:
print(old_source["tag_type"])
print(new_source["tag_type"])
print("-----tag_type-----")
if old_source["collection"] != new_source["collection"]:
print(old_source["collection"])
print(new_source["collection"])
print("-----collection-----")
if old_source["is_ai"] != new_source["is_ai"]:
print(old_source["is_ai"])
print(new_source["is_ai"])
print("-----is_ai-----")
if old_source["is_own"] != new_source["is_own"]:
print(old_source["is_own"])
print(new_source["is_own"])
print("-----is_own-----")
if old_source["is_online"] != new_source["is_online"]:
print(old_source["is_online"])
print(new_source["is_online"])
print("-----is_online-----")
if old_source["suggest"] != new_source["suggest"]:
print(old_source["suggest"])
print(new_source["suggest"])
print("-----suggest-----")
for key, values in old_source["suggest"].items():
if key == "input":
for i in values:
if i not in values:
print(i)
else:
is_online = values["is_online"]
is_deleted = values["is_deleted"]
if is_online != new_source["suggest"]["contexts"]["is_online"]:
print(is_online)
print("----is_online--------")
if is_deleted != new_source["suggest"]["contexts"]["is_deleted"]:
print(is_deleted)
print("-------is_deleted---------")
if old_source["is_deleted"] != new_source["is_deleted"]:
print(old_source["is_deleted"])
print(new_source["is_deleted"])
print("-----is_deleted-----")
if old_source["near_new_topic_num"] != new_source["near_new_topic_num"]:
print(old_source["near_new_topic_num"])
print(new_source["near_new_topic_num"])
print("-----near_new_topic_num-----")
index_contrast/topic.py 0 → 100644
View file @ 5686973a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
import os.path
import re
import json
from elasticsearch import Elasticsearch
import elasticsearch.helpers
from django.conf import settings
ES_INFO_LIST = [
{
"host": "172.17.32.22",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object):
cli_obj = None
cli_info_list = ES_INFO_LIST
index_prefix = ES_INDEX_PREFIX
@classmethod
def get_cli(cls, cli_info=None):
try:
init_args = {
'sniff_on_start': False,
'sniff_on_connection_fail': False,
}
es_cli_info = cli_info if cli_info else cls.cli_info_list
cls.cli_obj = Elasticsearch(hosts=es_cli_info, **init_args)
return cls.cli_obj
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def get_official_index_name(cls, sub_index_name, index_flag=None):
"""
:remark:get official es index name
:param sub_index_name:
:param index_flag:
:return:
"""
try:
assert (index_flag in [None, "read", "write"])
official_index_name = cls.index_prefix + "-" + sub_index_name
if index_flag:
official_index_name += "-" + index_flag
return official_index_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def __load_mapping(cls, doc_type):
try:
mapping_file_path = os.path.join(
os.path.dirname(__file__),
'..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
mapping = ''
with open(mapping_file_path, 'r') as f:
for line in f:
# 去掉注释
mapping += re.sub(r'//.*$', '', line)
mapping = json.loads(mapping)
return mapping
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def create_index(cls, es_cli, sub_index_name):
"""
:remark: create es index,alias index
:param sub_index_name:
:return:
"""
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name)
index_exist = es_cli.indices.exists(official_index_name)
if not index_exist:
es_cli.indices.create(official_index_name)
read_alias_name = cls.get_official_index_name(sub_index_name, "read")
es_cli.indices.put_alias(official_index_name, read_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
es_cli.indices.put_alias(official_index_name, write_alias_name)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc", force_sync=False):
"""
:remark: put index mapping
:param es_cli:
:param sub_index_name:
:param mapping_type:
:return:
"""
try:
assert (es_cli is not None)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
index_exist = es_cli.indices.exists(write_alias_name)
if not index_exist and not force_sync:
return False
mapping_dict = cls.__load_mapping(sub_index_name)
es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_indices_template(cls, es_cli, template_file_name, template_name):
"""
:remark put index template
:param es_cli:
:param template_file_name:
:param template_name:
:return:
"""
try:
assert (es_cli is not None)
mapping_dict = cls.__load_mapping(template_file_name)
es_cli.indices.put_template(name=template_name, body=mapping_dict)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
return False
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
bulk_actions = []
if sub_index_name == "topic" or \
sub_index_name == "topic-star-routing" or \
sub_index_name == "topic-high-star":
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
'routing': data["content_level"]
})
else:
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False,
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-topic-high-star-prod-190801001":
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
if not batch_search:
if not routing:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size)
else:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size, routing=routing)
if is_suggest_request:
return res
else:
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
res = es_cli.msearch(body=query_body, index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def if_es_node_load_high(cls, es_cli):
try:
assert (es_cli is not None)
high_num = 0
es_nodes_list = list()
es_nodes_ori_info = es_cli.cat.nodes()
es_nodes_info_list = es_nodes_ori_info.split("\n")
for item in es_nodes_info_list:
try:
item_list = item.split(" ")
if len(item_list) == 11:
cpu_load = item_list[4]
elif len(item_list) == 10:
cpu_load = item_list[3]
else:
continue
int_cpu_load = int(cpu_load)
if int_cpu_load > 60:
high_num += 1
es_nodes_list.append(int_cpu_load)
except:
logging.error("catch exception,item:%s,err_msg:%s" % (str(item), traceback.format_exc()))
return True
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
@classmethod
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=size, routing="4,5,6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("topic_id_list:%s" % str(topic_id_list))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
content_level = [6, 5, 4, 3, 2, 1]
for lev in content_level:
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"content_level": lev}},
{"range": {"update_time": {"gte": "2019-05-09T00:00:00+00:00"}}}
]
}
}
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, 0, 100)
result_dict_ids = list()
for old_item in result_dict["hits"]:
old_source = old_item["_source"]
old_id = old_source["id"]
# 获取新index
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"term": {
"id": old_id,
"is_online": True
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-topic-high-star-prod-190801001", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
print(new_id)
print("-----id-----")
if old_source["is_shadow"] != new_source["is_shadow"]:
print(old_source["is_shadow"])
print(new_source["is_shadow"])
print("-----is_shadow-----")
if old_source["content_level"] != new_source["content_level"]:
print(old_source["content_level"])
print(new_source["content_level"])
print("-----content_level-----")
if old_source["is_online"] != new_source["is_online"]:
print(old_source["is_online"])
print(new_source["is_online"])
print("-----is_online-----")
if old_source["is_deleted"] != new_source["is_deleted"]:
print(old_source["is_deleted"])
print(new_source["is_deleted"])
print("-----is_deleted-----")
if old_source["is_recommend"] != new_source["is_recommend"]:
print(old_source["is_recommend"])
print(new_source["is_recommend"])
print("-----is_recommend-----")
if old_source["is_complaint"] != new_source["is_complaint"]:
print(old_source["is_complaint"])
print(new_source["is_complaint"])
print("-----is_complaint-----")
if old_source["is_excellent"] != new_source["is_excellent"]:
print(old_source["is_excellent"])
print(new_source["is_excellent"])
print("-----is_excellent-----")
if old_source["is_operation_home_recommend"] != new_source["is_operation_home_recommend"]:
print(old_source["is_operation_home_recommend"])
print(new_source["is_operation_home_recommend"])
print("-----is_operation_home_recommend-----")
if old_source["vote_num"] != new_source["vote_num"]:
print(old_source["vote_num"])
print(new_source["vote_num"])
print("-----vote_num-----")
if old_source["reply_num"] != new_source["reply_num"]:
print(old_source["reply_num"])
print(new_source["reply_num"])
print("-----reply_num-----")
if old_source["user_id"] != new_source["user_id"]:
print(old_source["user_id"])
print(new_source["user_id"])
print("-----user_id-----")
if old_source["group_id"] != new_source["group_id"]:
print(old_source["group_id"])
print(new_source["group_id"])
print("-----group_id-----")
if old_source["share_num"] != new_source["share_num"]:
print(old_source["share_num"])
print(new_source["share_num"])
print("-----share_num-----")
if old_source["offline_score"] != new_source["offline_score"]:
print(old_source["offline_score"])
print(new_source["offline_score"])
print("-----offline_score-----")
if old_source["manual_score"] != new_source["manual_score"]:
print(old_source["manual_score"])
print(new_source["manual_score"])
print("-----manual_score-----")
if old_source["has_image"] != new_source["has_image"]:
print(old_source["has_image"])
print(new_source["has_image"])
print("-----has_image-----")
if old_source["has_video"] != new_source["has_video"]:
print(old_source["has_video"])
print(new_source["has_video"])
print("-----has_video-----")
if old_source["language_type"] != new_source["language_type"]:
print(old_source["language_type"])
print(new_source["language_type"])
print("-----language_type-----")
if old_source["virtual_content_level"] != new_source["virtual_content_level"]:
print(old_source["virtual_content_level"])
print(new_source["virtual_content_level"])
print("-----virtual_content_level-----")
if old_source["like_num_crawl"] != new_source["like_num_crawl"]:
print(old_source["like_num_crawl"])
print(new_source["like_num_crawl"])
print("-----like_num_crawl-----")
if old_source["comment_num_crawl"] != new_source["comment_num_crawl"]:
print(old_source["comment_num_crawl"])
print(new_source["comment_num_crawl"])
print("-----comment_num_crawl-----")
if old_source["platform"] != new_source["platform"]:
print(old_source["platform"])
print(new_source["platform"])
print("-----platform-----")
if old_source["platform_id"] != new_source["platform_id"]:
print(old_source["platform_id"])
print(new_source["platform_id"])
print("-----platform_id-----")
#
if old_source["drop_score"] != new_source["drop_score"]:
print(old_source["drop_score"])
print(new_source["drop_score"])
print("-----drop_score-----")
if old_source["sort_score"] != new_source["sort_score"]:
print(old_source["sort_score"])
print(new_source["sort_score"])
print("-----sort_score-----")
if old_source["create_time_val"] != new_source["create_time_val"]:
print(old_source["create_time_val"])
print(new_source["create_time_val"])
print("-----create_time_val-----")
#
if old_source["update_time_val"] != new_source["update_time_val"]:
print(old_source["update_time_val"])
print(new_source["update_time_val"])
print("-----update_time_val-----")
if old_source["total_vote_num"] != new_source["total_vote_num"]:
print(old_source["total_vote_num"])
print(new_source["total_vote_num"])
print("-----total_vote_num-----")
if old_source["pictorial_id"] != new_source["pictorial_id"]:
print(old_source["pictorial_id"])
print(new_source["pictorial_id"])
print("-----pictorial_id-----")
if old_source["pick_id_list"] != new_source["pick_id_list"]:
print(old_source["pick_id_list"])
print(new_source["pick_id_list"])
print("-----pick_id_list-----")
if old_source["tag_list"] != new_source["tag_list"]:
print(old_source["tag_list"])
print(new_source["tag_list"])
print("-----tag_list-----")
if old_source["edit_tag_list"] != new_source["edit_tag_list"]:
print(old_source["edit_tag_list"])
print(new_source["edit_tag_list"])
print("-----edit_tag_list-----")
if old_source["tag_name_list"] != new_source["tag_name_list"]:
print(old_source["tag_name_list"])
print(new_source["tag_name_list"])
print("-----tag_name_list-----")
if old_source["name"] != new_source["name"]:
print(old_source["name"])
print(new_source["name"])
print("-----name-----")
if old_source["description"] != new_source["description"]:
print(old_source["description"])
print(new_source["description"])
print("-----description-----")
if old_source["user_nick_name"] != new_source["user_nick_name"]:
print(old_source["user_nick_name"])
print(new_source["user_nick_name"])
print("-----user_nick_name-----")
if old_source["user_nick_name_pre"] != new_source["user_nick_name_pre"]:
print(old_source["user_nick_name_pre"])
print(new_source["user_nick_name_pre"])
print("-----user_nick_name_pre-----")
index_contrast/user.py 0 → 100644
View file @ 5686973a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
import os.path
import re
import json
from elasticsearch import Elasticsearch
import elasticsearch.helpers
from django.conf import settings
ES_INFO_LIST = [
{
"host": "172.17.32.22",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object):
cli_obj = None
cli_info_list = ES_INFO_LIST
index_prefix = ES_INDEX_PREFIX
@classmethod
def get_cli(cls, cli_info=None):
try:
init_args = {
'sniff_on_start': False,
'sniff_on_connection_fail': False,
}
es_cli_info = cli_info if cli_info else cls.cli_info_list
cls.cli_obj = Elasticsearch(hosts=es_cli_info, **init_args)
return cls.cli_obj
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def get_official_index_name(cls, sub_index_name, index_flag=None):
"""
:remark:get official es index name
:param sub_index_name:
:param index_flag:
:return:
"""
try:
index_flag = None
assert (index_flag in [None, "read", "write"])
official_index_name = cls.index_prefix + "-" + sub_index_name
if index_flag:
official_index_name += "-" + index_flag
return official_index_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def __load_mapping(cls, doc_type):
try:
mapping_file_path = os.path.join(
os.path.dirname(__file__),
'..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
mapping = ''
with open(mapping_file_path, 'r') as f:
for line in f:
# 去掉注释
mapping += re.sub(r'//.*$', '', line)
mapping = json.loads(mapping)
return mapping
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def create_index(cls, es_cli, sub_index_name):
"""
:remark: create es index,alias index
:param sub_index_name:
:return:
"""
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name)
index_exist = es_cli.indices.exists(official_index_name)
if not index_exist:
es_cli.indices.create(official_index_name)
read_alias_name = cls.get_official_index_name(sub_index_name, "read")
es_cli.indices.put_alias(official_index_name, read_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
es_cli.indices.put_alias(official_index_name, write_alias_name)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc", force_sync=False):
"""
:remark: put index mapping
:param es_cli:
:param sub_index_name:
:param mapping_type:
:return:
"""
try:
assert (es_cli is not None)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
index_exist = es_cli.indices.exists(write_alias_name)
if not index_exist and not force_sync:
return False
mapping_dict = cls.__load_mapping(sub_index_name)
es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_indices_template(cls, es_cli, template_file_name, template_name):
"""
:remark put index template
:param es_cli:
:param template_file_name:
:param template_name:
:return:
"""
try:
assert (es_cli is not None)
mapping_dict = cls.__load_mapping(template_file_name)
es_cli.indices.put_template(name=template_name, body=mapping_dict)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
return False
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
bulk_actions = []
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False,
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-user-prod-190722001":
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
if not batch_search:
if not routing:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size)
else:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size, routing=routing)
if is_suggest_request:
return res
else:
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
res = es_cli.msearch(body=query_body, index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def if_es_node_load_high(cls, es_cli):
try:
assert (es_cli is not None)
high_num = 0
es_nodes_list = list()
es_nodes_ori_info = es_cli.cat.nodes()
es_nodes_info_list = es_nodes_ori_info.split("\n")
for item in es_nodes_info_list:
try:
item_list = item.split(" ")
if len(item_list) == 11:
cpu_load = item_list[4]
elif len(item_list) == 10:
cpu_load = item_list[3]
else:
continue
int_cpu_load = int(cpu_load)
if int_cpu_load > 60:
high_num += 1
es_nodes_list.append(int_cpu_load)
except:
logging.error("catch exception,item:%s,err_msg:%s" % (str(item), traceback.format_exc()))
return True
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"range": {"update_time": {"gte": "2019-05-09T00:00:00+00:00"}}}
]
}
}
result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, 0, 1000)
result_dict_ids = list()
for old_item in result_dict["hits"]:
old_source = old_item["_source"]
old_id = old_source["id"]
# 获取新index
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"id": old_id}}
]
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-user-prod-190722001", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
print(new_id)
print("-----id-----")
if old_source["user_id"] != new_source["user_id"]:
print(old_source["user_id"])
print(new_source["user_id"])
print("-----user_id-----")
if old_source["nick_name"] != new_source["nick_name"]:
print(old_source["nick_name"])
print(new_source["nick_name"])
print("-----nick_name-----")
if old_source["nick_pre"] != new_source["nick_pre"]:
print(old_source["nick_pre"])
print(new_source["nick_pre"])
print("-----nick_pre-----")
if old_source["nick_name_pre"] != new_source["nick_name_pre"]:
print(old_source["nick_name_pre"])
print(new_source["nick_name_pre"])
print("-----nick_name_pre-----")
if old_source["profile_pic"] != new_source["profile_pic"]:
print(old_source["profile_pic"])
print(new_source["profile_pic"])
print("-----profile_pic-----")
if old_source["gender"] != new_source["gender"]:
print(old_source["gender"])
print(new_source["gender"])
print("-----gender-----")
if old_source["is_deleted"] != new_source["is_deleted"]:
print(old_source["is_deleted"])
print(new_source["is_deleted"])
print("-----is_deleted-----")
if old_source["is_online"] != new_source["is_online"]:
print(old_source["is_online"])
print(new_source["is_online"])
print("-----is_online-----")
for i in old_source["tag_list"]:
if i not in new_source["tag_list"]:
print(i)
print(old_source["tag_list"])
print(new_source["tag_list"])
print("-----tag_list-----")
for i in old_source["useful_tag_list"]:
if i not in new_source["useful_tag_list"]:
print(i)
print(old_source["useful_tag_list"])
print(new_source["useful_tag_list"])
print("-----useful_tag_list-----")
if old_source["city_id"] != new_source["city_id"]:
print(old_source["city_id"])
print(new_source["city_id"])
print("-----city_id-----")
if old_source["country_id"] != new_source["country_id"]:
print(old_source["country_id"])
print(new_source["country_id"])
print("-----country_id-----")
if old_source["is_recommend"] != new_source["is_recommend"]:
print(old_source["is_recommend"])
print(new_source["is_recommend"])
print("-----is_recommend-----")
if old_source["is_shadow"] != new_source["is_shadow"]:
print(old_source["is_shadow"])
print(new_source["is_shadow"])
print("-----is_shadow-----")
if old_source["latest_topic_time_val"] != new_source["latest_topic_time_val"]:
print(old_source["latest_topic_time_val"])
print(new_source["latest_topic_time_val"])
print("-----latest_topic_time_val-----")
for item in old_source["attention_user_id_list"]:
if item not in new_source["attention_user_id_list"]:
print(item)
print(old_source["attention_user_id_list"])
print(new_source["attention_user_id_list"])
print("-----attention_user_id_list-----")
if old_source["pick_user_id_list"] != new_source["pick_user_id_list"]:
print(old_source["pick_user_id_list"])
print(new_source["pick_user_id_list"])
print("-----pick_user_id_list-----")
for item in old_source["same_pictorial_user_id_list"]:
if item not in new_source["same_pictorial_user_id_list"]:
print(item)
print(old_source["same_pictorial_user_id_list"])
print(new_source["same_pictorial_user_id_list"])
print("-----same_pictorial_user_id_list-----")
for item in old_source["attention_pictorial_id_list"]:
if item not in new_source["attention_pictorial_id_list"]:
print(old_source["attention_pictorial_id_list"])
print(new_source["attention_pictorial_id_list"])
print("-----attention_pictorial_id_list-----")
if old_source["create_time"] != new_source["create_time"]:
print(old_source["create_time"])
print(new_source["create_time"])
print("-----create_time-----")
if old_source["update_time"] != new_source["update_time"]:
print(old_source["update_time"])
print(new_source["update_time"])
print("-----update_time-----")
if old_source["create_time_val"] != new_source["create_time_val"]:
print(old_source["create_time_val"])
print(new_source["create_time_val"])
print("-----create_time_val-----")
if old_source["update_time_val"] != new_source["update_time_val"]:
print(old_source["update_time_val"])
print(new_source["update_time_val"])
print("-----update_time_val-----")
if old_source["count_topic"] != new_source["count_topic"]:
print(old_source["count_topic"])
print(new_source["count_topic"])
print("-----count_topic-----")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment