Commit b22b9fb7 authored by litaolemo's avatar litaolemo

update

parent fd53a1cc
...@@ -71,51 +71,48 @@ for t in range(0, task_days): ...@@ -71,51 +71,48 @@ for t in range(0, task_days):
today_str = now.strftime("%Y%m%d") today_str = now.strftime("%Y%m%d")
yesterday_str = (now + datetime.timedelta(days=-1)).strftime("%Y%m%d") yesterday_str = (now + datetime.timedelta(days=-1)).strftime("%Y%m%d")
one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d") one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d")
sql_distinct_device_id = r""" sql_distinct_device_id = """SELECT partition_date,device_id FROM
SELECT partition_date,device_id (
FROM SELECT user_id,partition_date,
( if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
SELECT user_id,partition_date, FROM online.ml_user_updates
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id WHERE partition_date>='{yesterday_str}' AND partition_date<'{today_str}'
FROM online.ml_user_updates )t1
WHERE partition_date>='{yesterday_str}' AND partition_date<'{today_str}' JOIN
)t1 (
JOIN SELECT distinct user_id
( FROM online.tl_hdfs_doctor_view
SELECT distinct user_id WHERE partition_date = '{yesterday_str}'
FROM online.tl_hdfs_doctor_view
WHERE partition_date = '{yesterday_str}' UNION ALL
SELECT user_id
UNION ALL FROM ml.ml_c_ct_ui_user_dimen_d
SELECT user_id WHERE partition_day = '{yesterday_str}'
FROM ml.ml_c_ct_ui_user_dimen_d AND (is_puppet = 'true' or is_classifyuser = 'true')
WHERE partition_day = '{yesterday_str}'
AND (is_puppet = 'true' or is_classifyuser = 'true') UNION ALL
select distinct user_id
UNION ALL from dim.dim_device_user_staff
select distinct user_id UNION ALL
from dim.dim_device_user_staff
UNION ALL SELECT distinct t1.user_id
FROM
SELECT distinct t1.user_id (
FROM SELECT user_id, v.device_id as device_id
( FROM online.ml_user_history_detail
SELECT user_id, v.device_id as device_id LATERAL VIEW EXPLODE(device_history_list) v AS device_id
FROM online.ml_user_history_detail WHERE partition_date = '{yesterday_str}'
LATERAL VIEW EXPLODE(device_history_list) v AS device_id )t1
WHERE partition_date = '{yesterday_str}' JOIN
)t1 (
JOIN SELECT device_id
( FROM online.ml_device_history_detail
SELECT device_id WHERE partition_date = '{yesterday_str}'
FROM online.ml_device_history_detail AND is_login_doctor = '1'
WHERE partition_date = '{yesterday_str}' )t2
AND is_login_doctor = '1' ON t1.device_id = t2.device_id
)t2 )t2
ON t1.device_id = t2.device_id on t1.user_id=t2.user_id group by partition_date,device_id
)t2
on t1.user_id=t2.user_id
group by partition_date,device_id
""".format(today_str=today_str, yesterday_str=yesterday_str, ) """.format(today_str=today_str, yesterday_str=yesterday_str, )
distinct_device_id_df = spark.sql(sql_distinct_device_id) distinct_device_id_df = spark.sql(sql_distinct_device_id)
distinct_device_id_df.show(1, False) distinct_device_id_df.show(1, False)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment