diff --git a/README.md b/README.md index 78c2afa5366c1f8eaff8a9a54ee5cfcdfe1429d9..e76ef37dc6658f5a239797e339d64402416000ea 100644 --- a/README.md +++ b/README.md @@ -10,4 +10,8 @@ metabase æ•°æ®å¼€å‘相关代ç /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/recommend_strategy_d.py /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 2 --num-executors 100 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_strategy_d.py /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 2g --executor-cores 4 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/core_indicators_monitoring.py -/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 2g --executor-cores 4 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/advertisement_strategy_d.py +/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 4 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/advertisement_strategy_d.py +/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_meigou_ctr.py +/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_diary_ctr.py +/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_tractate_ctr.py +/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_answer_ctr.py diff --git a/task.sh b/task.sh index b083902c9038aae37d628fbb98b02919984ad720..6ed448b14bccb067ef65a9dce4fdb9f5883a3d6b 100644 --- a/task.sh +++ b/task.sh @@ -2,5 +2,6 @@ source /srv/envs/esmm/bin/activate /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/conent_detail_page_grayscale_ctr.py /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/recommend_strategy_d.py /opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_strategy_d.py -/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 2g --executor-cores 4 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/core_indicators_monitoring.py +/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 4 --num-executors 120 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/core_indicators_monitoring.py +/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 4 --num-executors 120 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_meigou_ctr.py diff --git a/task/search_answer_ctr.py b/task/search_answer_ctr.py index 8658af5a485016e6f0af33641bcc61f2c9729060..24353fd7a2d25b4a271591fa299635e70a120ca6 100644 --- a/task/search_answer_ctr.py +++ b/task/search_answer_ctr.py @@ -204,11 +204,15 @@ and (dev.device_id is null or dev.device_id='') print(res) if res.ACTIVE_TYPE: if res.ACTIVE_TYPE in ('1', '2'): - res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE + if res.CLICK_NUM: + res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE else: - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE + if res.CLICK_NUM: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE for active_type in res_dict: for device_os_type in res_dict[active_type]: diff --git a/task/search_diary_ctr.py b/task/search_diary_ctr.py index c85f0f0aa1a5233278a9c289734bcbdb2916f7a8..60644da394be1016dc561c0bbc95ca431a22c5fe 100644 --- a/task/search_diary_ctr.py +++ b/task/search_diary_ctr.py @@ -204,11 +204,15 @@ and (dev.device_id is null or dev.device_id='') print(res) if res.ACTIVE_TYPE: if res.ACTIVE_TYPE in ('1', '2'): - res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE + if res.CLICK_NUM: + res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE else: - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE + if res.CLICK_NUM: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE for active_type in res_dict: for device_os_type in res_dict[active_type]: diff --git a/task/search_meigou_ctr.py b/task/search_meigou_ctr.py index 8a511b2904b483e52190f7e1555efef311a0e455..ee438f894df44be0400e385dcd7101ef33b653ff 100644 --- a/task/search_meigou_ctr.py +++ b/task/search_meigou_ctr.py @@ -205,11 +205,15 @@ and (dev.device_id is null or dev.device_id='') print(res) if res.ACTIVE_TYPE: if res.ACTIVE_TYPE in ('1', '2'): - res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE + if res.CLICK_NUM: + res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE else: - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE + if res.CLICK_NUM: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE for active_type in res_dict: for device_os_type in res_dict[active_type]: diff --git a/task/search_tractate_ctr.py b/task/search_tractate_ctr.py index d3a114f1aca90673660e5421cdce2d60f10fe7ce..e08fa760ea3d93e05d855849a470a1caa88cdd35 100644 --- a/task/search_tractate_ctr.py +++ b/task/search_tractate_ctr.py @@ -204,12 +204,15 @@ and (dev.device_id is null or dev.device_id='') print(res) if res.ACTIVE_TYPE: if res.ACTIVE_TYPE in ('1', '2'): - res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE + if res.CLICK_NUM: + res_dict["新增"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["新增"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE else: - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM - res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE - + if res.CLICK_NUM: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["click_num"] += res.CLICK_NUM + if res.EXPOSURE: + res_dict["è€æ´»"][res.DEVICE_OS_TYPE]["exposure"] += res.EXPOSURE for active_type in res_dict: for device_os_type in res_dict[active_type]: partition_date = yesterday_str