update

task/start_task.py

@@ -6,7 +6,7 @@
 
 import subprocess
 import datetime
-from pyhive import hive
+# from pyhive import hive
 
 conent_detail_page_grayscale_ctr_str = '/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/conent_detail_page_grayscale_ctr.py\n'
 recommend_strategy_d_str = '/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.executorEnv.LD_LIBRARY_PATH="/opt/java/jdk1.8.0_181/jre/lib/amd64/server:/opt/cloudera/parcels/CDH-5.16.1-1.cdh5.16.1.p0.3/lib64" --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/recommend_strategy_d.py\n'
@@ -17,22 +17,22 @@ hive_port = 10000
 hive_username = "strategy"
 hive_database = "online"
 
-conn = hive.Connection(host=hive_host, port=hive_port, username=hive_username, database=hive_database)
-cursor = conn.cursor()
+# conn = hive.Connection(host=hive_host, port=hive_port, username=hive_username, database=hive_database)
+# cursor = conn.cursor()
 
 table_1 = "pm.tl_pm_recommend_strategy_d"
 
 
-def if_write_data(table):
-    now = datetime.datetime.now() - datetime.timedelta(days=1)
-    date_str = now.strftime("%Y%m%d")
-    sql_str = "select count(*) from {0} where day_id = {1}".format(table,date_str)
-    res = cursor.fetchall(sql_str)
-    print(res)
-    if res:
-        return True
-    else:
-        return False
+# def if_write_data(table):
+#     now = datetime.datetime.now() - datetime.timedelta(days=1)
+#     date_str = now.strftime("%Y%m%d")
+#     sql_str = "select count(*) from {0} where day_id = {1}".format(table,date_str)
+#     res = cursor.fetchall(sql_str)
+#     print(res)
+#     if res:
+#         return True
+#     else:
+#         return False
 
 
 
@@ -46,7 +46,8 @@ def func_cal_task(task_str):
 
 if __name__ == "__main__":
     # func_cal_task()
-    print(if_write_data(table_1))
-    cursor.close()
-    conn.close()
-
+    # print(if_write_data(table_1))
+    # cursor.close()
+    # conn.close()
+    func_cal_task(recommend_strategy_d_str)
+    func_cal_task(conent_detail_page_grayscale_ctr_str)
\ No newline at end of file