update

task/start_task.py

+# -*- coding:UTF-8 -*-
+# @Time  : 2020/8/20 17:20
+# @File  : start_task.py
+# @email : litao@igengmei.com
+# @author : litao
+
+import subprocess
+import datetime
+from pyhive import hive
+
+conent_detail_page_grayscale_ctr_str = '/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/conent_detail_page_grayscale_ctr.py\n'
+recommend_strategy_d_str = '/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/recommend_strategy_d.py\n'
+
+
+hive_host = "bj-gm-prod-cos-datacenter003"
+hive_port = 10000
+hive_username = "strategy"
+hive_database = "online"
+
+conn = hive.Connection(host=hive_host, port=hive_port, username=hive_username, database=hive_database)
+cursor = conn.cursor()
+
+table_1 = "pm.tl_pm_recommend_strategy_d"
+
+
+def if_write_data(table):
+    now = datetime.datetime.now() - datetime.timedelta(days=1)
+    date_str = now.strftime("%Y%m%d")
+    sql_str = "select count(*) from {0} where day_id = {1}".format(table,date_str)
+    res = cursor.fetchall(sql_str)
+    print(res)
+    if res:
+        return True
+    else:
+        return False
+
+
+
+def func_cal_task(task_str):
+    sub_str = "source /srv/envs/esmm/bin/activate\n"
+    print(sub_str)
+    sub_str += task_str
+    subprocess.run(sub_str, shell=True)
+
+
+
+if __name__ == "__main__":
+    # func_cal_task()
+    print(if_write_data(table_1))
+    cursor.close()
+    conn.close()
+