hive test

utils/connUtils.py

@@ -8,67 +8,8 @@ def getRedisConn():
     # conn = redis.Redis(host="172.18.51.10", port=6379,db=0) #test
     return conn
 
-def getHiveConn():
-    # uri = "thrift://bj-gm-prod-cos-datacenter001:9083,thrift://bj-gm-prod-cos-datacenter002:9083"
-    # uri = "thrift://bj-gm-prod-cos-datacenter001:9083,thrift://bj-gm-prod-cos-datacenter002:9083"
-
-    conn = hive.connect(host="bj-gm-prod-cos-datacenter001",port=9083)
-    return conn
-
 
 if __name__ == '__main__':
-    sql = """
-        SELECT DISTINCT t1.partition_date, t1.cl_id device_id, t1.card_id,t1.time_stamp,t1.page_stay 
-          FROM
-            (
-                select partition_date,cl_id,business_id as card_id,time_stamp,page_stay
-                from online.bl_hdfs_maidian_updates
-                where action = 'page_view'
-                AND partition_date>='{startDay}' and partition_date<='{endDay}'
-                AND page_name='welfare_detail'
-                -- AND page_stay>=1
-                AND cl_id is not null
-                AND cl_id != ''
-                AND business_id is not null
-                AND business_id != ''
-                group by partition_date,cl_id,business_id,time_stamp,page_stay
-            ) AS t1
-            join
-            (	--渠道，新老
-                SELECT distinct device_id
-                FROM online.ml_device_day_active_status
-                where partition_date>='{startDay}' and partition_date<'{endDay}'
-                AND active_type in ('1','2','4')
-                and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
-                ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
-                ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
-                ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
-                ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
-                ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
-                ,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
-                AND first_channel_source_type not like 'promotion\_jf\_%'
-            ) t2
-            on t1.cl_id = t2.device_id
-
-            LEFT JOIN
-            (	--去除黑名单
-                select distinct device_id
-                from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
-                where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
-                AND is_abnormal_device = 'true'
-            )t3 
-            on t3.device_id=t2.device_id
-            WHERE t3.device_id is null
-             """.format(startDay="20210708", endDay="20210709")
-
-    conn = getHiveConn()
-    cur = conn.cursor()
-    cur.execute(sql)
-    datas = cur.fetchall()
-    print(len(datas))
-    for d in datas[0:10]:
-        print(d)
-
     # REDIS_URL = 'redis://:@172.18.51.10:6379'
     # redis_client = redis.StrictRedis.from_url(REDIS_URL, decode_responses=True)
     # # redisKey = "service_recommend_read_cache_869982038583034"