hive test

utils/connUtils.py

 import redis
+from pyhive import hive
 
 def getRedisConn():
     pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
@@ -6,3 +7,90 @@ def getRedisConn():
     # conn = redis.Redis(host="172.16.50.145", port=6379, password="XfkMCCdWDIU%ls$h",db=0)
     # conn = redis.Redis(host="172.18.51.10", port=6379,db=0) #test
     return conn
+
+def getHiveConn():
+    # uri = "thrift://bj-gm-prod-cos-datacenter001:9083,thrift://bj-gm-prod-cos-datacenter002:9083"
+    # uri = "thrift://bj-gm-prod-cos-datacenter001:9083,thrift://bj-gm-prod-cos-datacenter002:9083"
+
+    conn = hive.connect(host="bj-gm-prod-cos-datacenter001",port=9083)
+    return conn
+
+
+if __name__ == '__main__':
+    sql = """
+        SELECT DISTINCT t1.partition_date, t1.cl_id device_id, t1.card_id,t1.time_stamp,t1.page_stay 
+          FROM
+            (
+                select partition_date,cl_id,business_id as card_id,time_stamp,page_stay
+                from online.bl_hdfs_maidian_updates
+                where action = 'page_view'
+                AND partition_date>='{startDay}' and partition_date<='{endDay}'
+                AND page_name='welfare_detail'
+                -- AND page_stay>=1
+                AND cl_id is not null
+                AND cl_id != ''
+                AND business_id is not null
+                AND business_id != ''
+                group by partition_date,cl_id,business_id,time_stamp,page_stay
+            ) AS t1
+            join
+            (	--渠道，新老
+                SELECT distinct device_id
+                FROM online.ml_device_day_active_status
+                where partition_date>='{startDay}' and partition_date<'{endDay}'
+                AND active_type in ('1','2','4')
+                and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+                ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+                ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+                ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+                ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+                ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+                ,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
+                AND first_channel_source_type not like 'promotion\_jf\_%'
+            ) t2
+            on t1.cl_id = t2.device_id
+
+            LEFT JOIN
+            (	--去除黑名单
+                select distinct device_id
+                from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
+                where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
+                AND is_abnormal_device = 'true'
+            )t3 
+            on t3.device_id=t2.device_id
+            WHERE t3.device_id is null
+             """.format(startDay="20210708", endDay="20210709")
+
+    conn = getHiveConn()
+    cur = conn.cursor()
+    cur.execute(sql)
+    datas = cur.fetchall()
+    print(len(datas))
+    for d in datas[0:10]:
+        print(d)
+
+    # REDIS_URL = 'redis://:@172.18.51.10:6379'
+    # redis_client = redis.StrictRedis.from_url(REDIS_URL, decode_responses=True)
+    # # redisKey = "service_recommend_read_cache_869982038583034"
+    # redisKey = "strategy:icon:click:869982038583034"
+    #
+    # d = {"1":1,"2":2}
+    # redis_client.hmset(redisKey,d)
+
+    # m = redis_client.hgetall(redisKey)
+    # pipline = redis_client.pipeline()
+    # for k,v in m.items():
+    #     # print(k,v)
+    #     pipline.hdel(redisKey,k)
+    # pipline.execute()
+    # pipline.close()
+    #
+    # m = redis_client.hgetall(redisKey)
+    # redis_client.hincrby(redisKey,"3",5)
+    # m = redis_client.hgetall(redisKey)
+    # redis_client.expire(redisKey,60*60*24*7)
+    # for k, v in m.items():
+    #     print(k, v)
+
+
+    pass
\ No newline at end of file