Commit 6ee9106b authored by litaolemo's avatar litaolemo

update

parent 4a4424f5
...@@ -71,10 +71,10 @@ for t in range(0, task_days): ...@@ -71,10 +71,10 @@ for t in range(0, task_days):
today_str = now.strftime("%Y%m%d") today_str = now.strftime("%Y%m%d")
yesterday_str = (now + datetime.timedelta(days=-1)).strftime("%Y%m%d") yesterday_str = (now + datetime.timedelta(days=-1)).strftime("%Y%m%d")
one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d") one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d")
sql_distinct_device_id = """SELECT partition_date,device_id FROM sql_distinct_device_id = r"""SELECT partition_date,device_id FROM
( (
SELECT user_id,partition_date, SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id if(size(device_list) > 0, device_list [0], '') AS device_id
FROM online.ml_user_updates FROM online.ml_user_updates
WHERE partition_date>='{yesterday_str}' AND partition_date<'{today_str}' WHERE partition_date>='{yesterday_str}' AND partition_date<'{today_str}'
)t1 )t1
...@@ -113,10 +113,10 @@ for t in range(0, task_days): ...@@ -113,10 +113,10 @@ for t in range(0, task_days):
ON t1.device_id = t2.device_id ON t1.device_id = t2.device_id
)t3 )t3
on t1.user_id=t3.user_id group by partition_date,device_id on t1.user_id=t3.user_id group by partition_date,device_id
""".format(today_str=today_str, yesterday_str=yesterday_str ) """.format(today_str=today_str, yesterday_str=yesterday_str)
print(sql_distinct_device_id) print(sql_distinct_device_id)
distinct_device_id_df = spark.sql(sql_distinct_device_id) distinct_device_id_df = spark.sql(sql_distinct_device_id)
distinct_device_id_df.show(1) # distinct_device_id_df.show(20)
sql_res = distinct_device_id_df.collect() sql_res = distinct_device_id_df.collect()
for res in sql_res: for res in sql_res:
print(res) print(res)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment