1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 22 09:22:16 2017
@author: hanye
"""
from __future__ import print_function
from pandora import api
from pandora.models import *
from pandora.utils import *
import json
import time
import datetime
def func_find_total_hits_from_qiniu(fetch_time_start_ts_i, fetch_time_end_ts_i):
endpoint = 'https://logdb.qiniu.com'
url_new='/v5/logdbkibana/msearch'
query_body_in_threads={
"query": {
"bool": {
"must": [
{"range": {"fetch_time": {"from": fetch_time_start_ts_i,
"to": fetch_time_end_ts_i,
"include_lower": "true",
"include_upper": "false"}}}
]
}
},
"size": 0,
"from": 0,
}
json_body_query='{"index":["csmvoide"]}\n'+json.dumps(query_body_in_threads)
# client claim should be near request clause, it seems qiniu has a relatively short
# keep-alive time
client = api.Client(endpoint, 'UdtK_JT7yhln0-yA0-a2I96s497c_rwl-jC7Fikz', 'mRjiaujBTd_P7TxvE__25Ryx62qFjWH9cBzHNC6y')
retry_counter=0
while retry_counter<100:
try:
query_resp=client._do_request('POST', url_new, json_body_query)
break
except:
retry_counter+=1
print('got exception when _do_request, sleep for 10 seconds to retry %d' % retry_counter)
time.sleep(10)
if retry_counter==100:
print('Failed to establish connection after %d retries, return -1, %s' % (retry_counter, datetime.datetime.now()))
return -1
re_text=query_resp.response.text
re_dict=json.loads(re_text, encoding='utf-8')
data_total=re_dict['responses'][0]['hits']['total']
return data_total