Skip to content

C
cybertron
Commit 44b068ac authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parent a4575de5
Hide whitespace changes
Inline Side-by-side
Showing with 508 additions and 47 deletions
.idea/workspace.xml
View file @ 44b068ac
......@@ -2,8 +2,7 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="47bd7678-47c9-4681-b4c9-928e6d933349" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/auto_click_51.py" beforeDir="false" afterPath="$PROJECT_DIR$/auto_click_51.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/auto_request.py" beforeDir="false" afterPath="$PROJECT_DIR$/auto_request.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/index_contrast/product.py" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
......@@ -15,26 +14,26 @@
<session id="-334786743">
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="py" value="93" />
<entry key="py" value="158" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="Python" value="93" />
<entry key="Python" value="158" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="batch_user_gen" value="2" />
<entry key="guanshui" value="4" />
<entry key="py" value="76" />
<entry key="py" value="78" />
<entry key="txt" value="8" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="PLAIN_TEXT" value="14" />
<entry key="Python" value="76" />
<entry key="Python" value="78" />
</counts>
</usages-collector>
<usages-collector id="statistics.lifecycle.project">
......@@ -50,37 +49,26 @@
</session>
</component>
<component name="FileEditorManager">
<splitter split-orientation="horizontal" split-proportion="0.68088466">
<split-first>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/auto_click_51.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="540">
<caret line="36" selection-start-line="36" selection-end-line="37" />
<folding>
<element signature="e#0#11#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</split-first>
<split-second>
<leaf>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/auto_click1.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="18" column="35" lean-forward="true" selection-start-line="17" selection-start-column="12" selection-end-line="18" selection-end-column="35" />
</state>
</provider>
</entry>
</file>
</leaf>
</split-second>
</splitter>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="101">
<caret line="197" column="69" selection-start-line="197" selection-start-column="12" selection-end-line="197" selection-end-column="69" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/index_contrast/product.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-439">
<caret line="373" column="39" lean-forward="true" selection-start-line="373" selection-start-column="39" selection-end-line="373" selection-end-column="39" />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FindInProjectRecents">
<findStrings>
......@@ -101,6 +89,7 @@
<option value="$PROJECT_DIR$/index_contrast/topic.py" />
<option value="$PROJECT_DIR$/auto_request.py" />
<option value="$PROJECT_DIR$/auto_click_51.py" />
<option value="$PROJECT_DIR$/index_contrast/product.py" />
</list>
</option>
</component>
......@@ -122,6 +111,11 @@
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="cybertron" type="b2602c69:ProjectViewProjectNode" />
<item name="cybertron" type="462c0819:PsiDirectoryNode" />
<item name="index_contrast" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
......@@ -132,6 +126,11 @@
<component name="PropertiesComponent">
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/index_contrast" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
......@@ -158,11 +157,12 @@
<servers />
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="1280" height="727" extended-state="6" />
<frame x="-323" y="-1080" width="1920" height="1080" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Structure" order="0" sideWeight="0.21188119" side_tool="true" weight="0.19112115" />
<window_info id="Favorites" order="1" side_tool="true" />
<window_info active="true" content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.23034154" />
<window_info active="true" content_ui="combo" id="Project" order="2" sideWeight="0.7872763" visible="true" weight="0.21169037" />
<window_info id="DB Browser" order="3" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.3284974" />
......@@ -172,7 +172,7 @@
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Terminal" order="8" visible="true" weight="0.0" />
<window_info anchor="bottom" id="Terminal" order="8" visible="true" weight="0.06321243" />
<window_info anchor="bottom" id="Event Log" order="9" side_tool="true" />
<window_info anchor="bottom" id="Python Console" order="10" />
<window_info anchor="bottom" id="DB Execution Console" order="11" />
......@@ -291,13 +291,6 @@
<entry file="file://$PROJECT_DIR$/answer_reply3.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="101">
<caret line="197" column="69" selection-start-line="197" selection-start-column="12" selection-end-line="197" selection-end-column="69" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="262">
......@@ -512,5 +505,19 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/user.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="101">
<caret line="197" column="69" selection-start-line="197" selection-start-column="12" selection-end-line="197" selection-end-column="69" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/index_contrast/product.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-439">
<caret line="373" column="39" lean-forward="true" selection-start-line="373" selection-start-column="39" selection-end-line="373" selection-end-column="39" />
</state>
</provider>
</entry>
</component>
</project>
\ No newline at end of file
index_contrast/product.py 0 → 100644
View file @ 44b068ac
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
import os.path
import re
import json
from elasticsearch import Elasticsearch
import elasticsearch.helpers
from django.conf import settings
ES_INFO_LIST = [
{
"host": "172.17.32.22",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
class ESPerform(object):
cli_obj = None
cli_info_list = ES_INFO_LIST
index_prefix = ES_INDEX_PREFIX
@classmethod
def get_cli(cls, cli_info=None):
try:
init_args = {
'sniff_on_start': False,
'sniff_on_connection_fail': False,
}
es_cli_info = cli_info if cli_info else cls.cli_info_list
cls.cli_obj = Elasticsearch(hosts=es_cli_info, **init_args)
return cls.cli_obj
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def get_official_index_name(cls, sub_index_name, index_flag=None):
"""
:remark:get official es index name
:param sub_index_name:
:param index_flag:
:return:
"""
try:
index_flag = None
assert (index_flag in [None, "read", "write"])
official_index_name = cls.index_prefix + "-" + sub_index_name
if index_flag:
official_index_name += "-" + index_flag
return official_index_name
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def __load_mapping(cls, doc_type):
try:
mapping_file_path = os.path.join(
os.path.dirname(__file__),
'..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
mapping = ''
with open(mapping_file_path, 'r') as f:
for line in f:
# 去掉注释
mapping += re.sub(r'//.*$', '', line)
mapping = json.loads(mapping)
return mapping
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def create_index(cls, es_cli, sub_index_name):
"""
:remark: create es index,alias index
:param sub_index_name:
:return:
"""
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name)
index_exist = es_cli.indices.exists(official_index_name)
if not index_exist:
es_cli.indices.create(official_index_name)
read_alias_name = cls.get_official_index_name(sub_index_name, "read")
es_cli.indices.put_alias(official_index_name, read_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
es_cli.indices.put_alias(official_index_name, write_alias_name)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc", force_sync=False):
"""
:remark: put index mapping
:param es_cli:
:param sub_index_name:
:param mapping_type:
:return:
"""
try:
assert (es_cli is not None)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
index_exist = es_cli.indices.exists(write_alias_name)
if not index_exist and not force_sync:
return False
mapping_dict = cls.__load_mapping(sub_index_name)
es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def put_indices_template(cls, es_cli, template_file_name, template_name):
"""
:remark put index template
:param es_cli:
:param template_file_name:
:param template_name:
:return:
"""
try:
assert (es_cli is not None)
mapping_dict = cls.__load_mapping(template_file_name)
es_cli.indices.put_template(name=template_name, body=mapping_dict)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-tag-test-190711901":
official_index_name = cls.get_official_index_name(sub_index_name, "write")
index_exists = es_cli.indices.exists(official_index_name)
logging.info("get index_exists:%s" % index_exists)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
return False
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
bulk_actions = []
for data in data_list:
if data:
bulk_actions.append({
'_op_type': 'index',
'_index': official_index_name,
'_type': doc_type,
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False,
batch_search=False, routing=None):
try:
assert (es_cli is not None)
official_index_name = sub_index_name
if sub_index_name != "mv-alpha-product-test-190731001":
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
if not batch_search:
if not routing:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size)
else:
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset,
size=size, routing=routing)
if is_suggest_request:
return res
else:
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
res = es_cli.msearch(body=query_body, index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(sub_index_name)
if not index_exists:
logging.error("index:%s is not existing,get_search_results error!" % sub_index_name)
return None
res = es_cli.indices.analyze(index=sub_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def if_es_node_load_high(cls, es_cli):
try:
assert (es_cli is not None)
high_num = 0
es_nodes_list = list()
es_nodes_ori_info = es_cli.cat.nodes()
es_nodes_info_list = es_nodes_ori_info.split("\n")
for item in es_nodes_info_list:
try:
item_list = item.split(" ")
if len(item_list) == 11:
cpu_load = item_list[4]
elif len(item_list) == 10:
cpu_load = item_list[3]
else:
continue
int_cpu_load = int(cpu_load)
if int_cpu_load > 60:
high_num += 1
es_nodes_list.append(int_cpu_load)
except:
logging.error("catch exception,item:%s,err_msg:%s" % (str(item), traceback.format_exc()))
return True
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return True
# 先获取一部分数据
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
]
}
}
result_dict = ESPerform.get_search_results(es_cli_obj, "product", q, 0, 1000)
result_dict_ids = list()
for old_item in result_dict["hits"]:
old_source = old_item["_source"]
old_id = old_source["id"]
# 获取新index
es_cli_obj = ESPerform.get_cli()
q = {}
q["query"] = {
"bool": {
"must": [
{"term": {"id": old_id}}
]
}
}
result_dict_test = ESPerform.get_search_results(es_cli_obj, "mv-alpha-product-test-190731001", q, 0, 1)
for new_item in result_dict_test["hits"]:
new_source = new_item["_source"]
new_id = new_source["id"]
print(new_id)
print("-----id-----")
if old_source["is_deleted"] != new_source["is_deleted"]:
print(old_source["is_deleted"])
print(new_source["is_deleted"])
print("-----is_deleted-----")
if old_source["is_online"] != new_source["is_online"]:
print(old_source["is_online"])
print(new_source["is_online"])
print("-----is_online-----")
if old_source["create_time"] != new_source["create_time"]:
print(old_source["create_time"])
print(new_source["create_time"])
print("-----create_time-----")
if old_source["update_time"] != new_source["update_time"]:
print(old_source["update_time"])
print(new_source["update_time"])
print("-----update_time-----")
if old_source["create_time_val"] != new_source["create_time_val"]:
print(old_source["create_time_val"])
print(new_source["create_time_val"])
print("-----create_time_val-----")
if old_source["update_time_val"] != new_source["update_time_val"]:
print(old_source["update_time_val"])
print(new_source["update_time_val"])
print("-----update_time_val-----")
if old_source["price"] != new_source["price"]:
print(old_source["price"])
print(new_source["price"])
print("-----price-----")
if old_source["cn_name_sort"] != new_source["cn_name_sort"]:
print(old_source["cn_name_sort"])
print(new_source["cn_name_sort"])
print("-----cn_name_sort-----")
if old_source["cn_name_pre"] != new_source["cn_name_pre"]:
print(old_source["cn_name_pre"])
print(new_source["cn_name_pre"])
print("-----cn_name_pre-----")
if old_source["en_name_pre"] != new_source["en_name_pre"]:
print(old_source["en_name_pre"])
print(new_source["en_name_pre"])
print("-----en_name_pre-----")
if old_source["alias"] != new_source["alias"]:
print(old_source["alias"])
print(new_source["alias"])
print("-----alias-----")
if old_source["cn_name"] != new_source["cn_name"]:
print(old_source["cn_name"])
print(new_source["cn_name"])
print("-----cn_name-----")
if old_source["alias_pre"] != new_source["alias_pre"]:
print(old_source["alias_pre"])
print(new_source["alias_pre"])
print("-----alias_pre-----")
if old_source["en_name"] != new_source["en_name"]:
print(old_source["en_name"])
print(new_source["en_name"])
print("-----en_name-----")
if old_source["description"] != new_source["description"]:
print(old_source["description"])
print(new_source["description"])
print("-----description-----")
if old_source["have_image"] != new_source["have_image"]:
print(old_source["have_image"])
print(new_source["have_image"])
print("-----have_image-----")
if old_source["comment_nums"] != new_source["comment_nums"]:
print(old_source["comment_nums"])
print(new_source["comment_nums"])
print("-----comment_nums-----")
if old_source["brand_cn_name"] != new_source["brand_cn_name"]:
print(old_source["brand_cn_name"])
print(new_source["brand_cn_name"])
print("-----brand_cn_name-----")
if old_source["brand_en_name"] != new_source["brand_en_name"]:
print(old_source["brand_en_name"])
print(new_source["brand_en_name"])
print("-----brand_en_name-----")
if old_source["brand_alias"] != new_source["brand_alias"]:
print(old_source["brand_alias"])
print(new_source["brand_alias"])
print("-----brand_alias-----")
if old_source["brand_cn_name_pre"] != new_source["brand_cn_name_pre"]:
print(old_source["brand_cn_name_pre"])
print(new_source["brand_cn_name_pre"])
print("-----brand_cn_name_pre-----")
if old_source["brand_en_name_pre"] != new_source["brand_en_name_pre"]:
print(old_source["brand_en_name_pre"])
print(new_source["brand_en_name_pre"])
print("-----brand_en_name_pre-----")
if old_source["category_cn_name"] != new_source["category_cn_name"]:
print(old_source["category_cn_name"])
print(new_source["category_cn_name"])
print("-----category_cn_name-----")
if old_source["category_cn_name_pre"] != new_source["category_cn_name_pre"]:
print(old_source["category_cn_name_pre"])
print(new_source["category_cn_name_pre"])
print("-----category_cn_name_pre-----")
if old_source["effect_cn_name"] != new_source["effect_cn_name"]:
print(old_source["effect_cn_name"])
print(new_source["effect_cn_name"])
print("-----effect_cn_name-----")
if old_source["effect_cn_name_pre"] != new_source["effect_cn_name_pre"]:
print(old_source["effect_cn_name_pre"])
print(new_source["effect_cn_name_pre"])
print("-----effect_cn_name_pre-----")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment