Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
M
meta_base_code
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
黎涛
meta_base_code
Commits
1e44ccba
Commit
1e44ccba
authored
Sep 07, 2020
by
litaolemo
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update
parent
02703186
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
7 additions
and
8 deletions
+7
-8
task.sh
task.sh
+1
-1
core_indicators_monitoring.py
task/core_indicators_monitoring.py
+1
-1
search_meigou_ctr.py
task/search_meigou_ctr.py
+4
-5
search_strategy_d.py
task/search_strategy_d.py
+1
-1
No files found.
task.sh
View file @
1e44ccba
source
/srv/envs/esmm/bin/activate
/opt/spark/bin/spark-submit
--master
yarn
--deploy-mode
client
--queue
root.strategy
--driver-memory
16g
--executor-memory
1g
--executor-cores
1
--num-executors
70
--conf
spark.default.parallelism
=
100
--conf
spark.storage.memoryFraction
=
0.5
--conf
spark.shuffle.memoryFraction
=
0.3
--conf
spark.locality.wait
=
0
--jars
/srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/conent_detail_page_grayscale_ctr.py
/opt/spark/bin/spark-submit
--master
yarn
--deploy-mode
client
--queue
root.strategy
--driver-memory
16g
--executor-memory
1g
--executor-cores
1
--num-executors
70
--conf
spark.default.parallelism
=
100
--conf
spark.storage.memoryFraction
=
0.5
--conf
spark.shuffle.memoryFraction
=
0.3
--conf
spark.locality.wait
=
0
--jars
/srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/recommend_strategy_d.py
#
/opt/spark/bin/spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_strategy_d.py
/opt/spark/bin/spark-submit
--master
yarn
--deploy-mode
client
--queue
root.strategy
--driver-memory
16g
--executor-memory
1g
--executor-cores
1
--num-executors
70
--conf
spark.default.parallelism
=
100
--conf
spark.storage.memoryFraction
=
0.5
--conf
spark.shuffle.memoryFraction
=
0.3
--conf
spark.locality.wait
=
0
--jars
/srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/search_strategy_d.py
/opt/spark/bin/spark-submit
--master
yarn
--deploy-mode
client
--queue
root.strategy
--driver-memory
16g
--executor-memory
2g
--executor-cores
4
--num-executors
70
--conf
spark.default.parallelism
=
100
--conf
spark.storage.memoryFraction
=
0.5
--conf
spark.shuffle.memoryFraction
=
0.3
--conf
spark.locality.wait
=
0
--jars
/srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/meta_base_code/task/core_indicators_monitoring.py
task/core_indicators_monitoring.py
View file @
1e44ccba
...
...
@@ -60,7 +60,7 @@ spark.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJso
spark
.
sql
(
"CREATE TEMPORARY FUNCTION arrayMerge AS 'com.gmei.hive.common.udf.UDFArryMerge'"
)
task_list
=
[]
task_days
=
5
task_days
=
10
for
t
in
range
(
0
,
task_days
):
day_num
=
0
-
t
...
...
task/search_meigou_ctr.py
View file @
1e44ccba
...
...
@@ -165,17 +165,16 @@ for t in range(0, task_days):
print
(
"-------------------------------"
)
sql_spam_pv_device_id
=
"""
SELECT T.DEVICE_ID, --设备ID
select T.DEVICE_ID,T.CARD_ID,if(NVL(C.EXPOSURE,0) <> 0 ,cast((NVL(T.CLICK_NUM,0)/NVL(C.EXPOSURE,0)) as decimal(18,5)) , 0) as search_ctr from
(SELECT T.DEVICE_ID, --设备ID
T.CARD_ID, --卡片ID
SUM(T.CLICK_NUM) AS CLICK_NUM, --点击次数
C.EXPOSURE as EXPOSURE --曝光次数
if(NVL(C.EXPOSURE,0) <> 0 ,cast((NVL(T.CLICK_NUM,0)/NVL(C.EXPOSURE,0)) as decimal(18,5)) , 0) as search_ctr
SUM(T.CLICK_NUM) AS CLICK_NUM --点击次数
FROM ML.ML_C_ET_CK_CLICK_DIMEN_D T
WHERE T.PARTITION_DAY = '{partition_day}'
AND T.PAGE_CODE = 'search_result_welfare'
AND T.ACTION IN ('goto_welfare_detail','search_result_welfare_click_item')
GROUP BY T.DEVICE_ID,
T.CARD_ID
T.CARD_ID
) T
left join
(SELECT T.DEVICE_ID as DEVICE_ID, --设备ID
...
...
task/search_strategy_d.py
View file @
1e44ccba
...
...
@@ -72,7 +72,7 @@ spark.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJso
spark
.
sql
(
"CREATE TEMPORARY FUNCTION arrayMerge AS 'com.gmei.hive.common.udf.UDFArryMerge'"
)
task_list
=
[]
task_days
=
90
task_days
=
5
for
t
in
range
(
0
,
task_days
):
day_num
=
0
-
t
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment