1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
# -*- coding:UTF-8 -*-
# @Time : 2020/8/20 17:20
# @File : start_task.py
# @email : litao@igengmei.com
# @author : litao
import subprocess
import datetime
# from pyhive import hive
conent_detail_page_grayscale_ctr_str = '/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/conent_detail_page_grayscale_ctr.py\n'
recommend_strategy_d_str = '/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/recommend_strategy_d.py\n'
hive_host = "bj-gm-prod-cos-datacenter003"
hive_port = 10000
hive_username = "strategy"
hive_database = "online"
# conn = hive.Connection(host=hive_host, port=hive_port, username=hive_username, database=hive_database)
# cursor = conn.cursor()
table_1 = "pm.tl_pm_recommend_strategy_d"
# def if_write_data(table):
# now = datetime.datetime.now() - datetime.timedelta(days=1)
# date_str = now.strftime("%Y%m%d")
# sql_str = "select count(*) from {0} where day_id = {1}".format(table,date_str)
# res = cursor.fetchall(sql_str)
# print(res)
# if res:
# return True
# else:
# return False
def func_cal_task(task_str):
sub_str = "source /srv/envs/esmm/bin/activate\n"
print(sub_str)
sub_str += task_str
subprocess.run(sub_str, shell=True)
if __name__ == "__main__":
# func_cal_task()
# print(if_write_data(table_1))
# cursor.close()
# conn.close()
func_cal_task(recommend_strategy_d_str)
func_cal_task(conent_detail_page_grayscale_ctr_str)