Commit 54cd81e8 authored by 郭羽's avatar 郭羽

hive test

parent 48a1d60c
import redis
from pyhive import hive
def getRedisConn():
pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
conn = redis.Redis(connection_pool=pool)
# conn = redis.Redis(host="172.16.50.145", port=6379, password="XfkMCCdWDIU%ls$h",db=0)
# conn = redis.Redis(host="172.18.51.10", port=6379,db=0) #test
return conn
\ No newline at end of file
return conn
def getHiveConn():
# uri = "thrift://bj-gm-prod-cos-datacenter001:9083,thrift://bj-gm-prod-cos-datacenter002:9083"
# uri = "thrift://bj-gm-prod-cos-datacenter001:9083,thrift://bj-gm-prod-cos-datacenter002:9083"
conn = hive.connect(host="bj-gm-prod-cos-datacenter001",port=9083)
return conn
if __name__ == '__main__':
sql = """
SELECT DISTINCT t1.partition_date, t1.cl_id device_id, t1.card_id,t1.time_stamp,t1.page_stay
FROM
(
select partition_date,cl_id,business_id as card_id,time_stamp,page_stay
from online.bl_hdfs_maidian_updates
where action = 'page_view'
AND partition_date>='{startDay}' and partition_date<='{endDay}'
AND page_name='welfare_detail'
-- AND page_stay>=1
AND cl_id is not null
AND cl_id != ''
AND business_id is not null
AND business_id != ''
group by partition_date,cl_id,business_id,time_stamp,page_stay
) AS t1
join
( --渠道,新老
SELECT distinct device_id
FROM online.ml_device_day_active_status
where partition_date>='{startDay}' and partition_date<'{endDay}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
) t2
on t1.cl_id = t2.device_id
LEFT JOIN
( --去除黑名单
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_abnormal_device = 'true'
)t3
on t3.device_id=t2.device_id
WHERE t3.device_id is null
""".format(startDay="20210708", endDay="20210709")
conn = getHiveConn()
cur = conn.cursor()
cur.execute(sql)
datas = cur.fetchall()
print(len(datas))
for d in datas[0:10]:
print(d)
# REDIS_URL = 'redis://:@172.18.51.10:6379'
# redis_client = redis.StrictRedis.from_url(REDIS_URL, decode_responses=True)
# # redisKey = "service_recommend_read_cache_869982038583034"
# redisKey = "strategy:icon:click:869982038583034"
#
# d = {"1":1,"2":2}
# redis_client.hmset(redisKey,d)
# m = redis_client.hgetall(redisKey)
# pipline = redis_client.pipeline()
# for k,v in m.items():
# # print(k,v)
# pipline.hdel(redisKey,k)
# pipline.execute()
# pipline.close()
#
# m = redis_client.hgetall(redisKey)
# redis_client.hincrby(redisKey,"3",5)
# m = redis_client.hgetall(redisKey)
# redis_client.expire(redisKey,60*60*24*7)
# for k, v in m.items():
# print(k, v)
pass
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment