Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
B
bi-report
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
data
bi-report
Commits
ac8773ea
Commit
ac8773ea
authored
Apr 26, 2020
by
赵建伟
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update codes
parent
ccd02635
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
21 additions
and
0 deletions
+21
-0
hive2csv.sh
lib/shell/hive2csv.sh
+4
-0
hive2hive.sh
lib/shell/hive2hive.sh
+4
-0
sparksql2csv.sh
lib/shell/sparksql2csv.sh
+2
-0
sparksql2email.sh
lib/shell/sparksql2email.sh
+8
-0
waitsuccess.sh
lib/shell/waitsuccess.sh
+3
-0
No files found.
lib/shell/hive2csv.sh
View file @
ac8773ea
...
@@ -15,6 +15,8 @@ timestamp=`date +"%Y%m%d%H%M%S"`
...
@@ -15,6 +15,8 @@ timestamp=`date +"%Y%m%d%H%M%S"`
tmp_file_name
=
"/data/bi-report/tmp/tmp_"
$timestamp
".sql"
tmp_file_name
=
"/data/bi-report/tmp/tmp_"
$timestamp
".sql"
result_file_name
=
"/data/bi-report/result/"
$1
"_"
$partition_date
".csv"
result_file_name
=
"/data/bi-report/result/"
$1
"_"
$partition_date
".csv"
[
$?
-eq
0
]
||
exit
1
#输入的账期为空,则取T+1默认账期
#输入的账期为空,则取T+1默认账期
if
[
-z
${
V_DATE
}
]
;
then
if
[
-z
${
V_DATE
}
]
;
then
V_DATE
=
`
date
--date
=
"-1 day"
+%Y-%m-%d
`
V_DATE
=
`
date
--date
=
"-1 day"
+%Y-%m-%d
`
...
@@ -53,6 +55,8 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
...
@@ -53,6 +55,8 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
$tmp_file_name
[
$?
-eq
0
]
||
exit
1
# 新标签与新标签对应集合类型 的对应关系
# 新标签与新标签对应集合类型 的对应关系
/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter004:10000
-n
data
--outputformat
=
csv2
--showHeader
=
true
-e
"
`
cat
$tmp_file_name
`
"
>
$result_file_name
/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter004:10000
-n
data
--outputformat
=
csv2
--showHeader
=
true
-e
"
`
cat
$tmp_file_name
`
"
>
$result_file_name
rm
-rf
$tmp_file_name
rm
-rf
$tmp_file_name
...
...
lib/shell/hive2hive.sh
View file @
ac8773ea
...
@@ -13,6 +13,8 @@ V_DATE=$2
...
@@ -13,6 +13,8 @@ V_DATE=$2
timestamp
=
`
date
+
"%Y%m%d%H%M%S"
`
timestamp
=
`
date
+
"%Y%m%d%H%M%S"
`
tmp_file_name
=
"/data/bi-report/tmp/tmp_"
$timestamp
".sql"
tmp_file_name
=
"/data/bi-report/tmp/tmp_"
$timestamp
".sql"
[
$?
-eq
0
]
||
exit
1
#输入的账期为空,则取T+1默认账期
#输入的账期为空,则取T+1默认账期
if
[
-z
${
V_DATE
}
]
;
then
if
[
-z
${
V_DATE
}
]
;
then
V_DATE
=
`
date
--date
=
"-1 day"
+%Y-%m-%d
`
V_DATE
=
`
date
--date
=
"-1 day"
+%Y-%m-%d
`
...
@@ -51,6 +53,8 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
...
@@ -51,6 +53,8 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
$tmp_file_name
[
$?
-eq
0
]
||
exit
1
# 新标签与新标签对应集合类型 的对应关系
# 新标签与新标签对应集合类型 的对应关系
/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter004:10000
-n
data
-e
"
`
cat
$tmp_file_name
`
"
/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter004:10000
-n
data
-e
"
`
cat
$tmp_file_name
`
"
rm
-rf
$tmp_file_name
rm
-rf
$tmp_file_name
...
...
lib/shell/sparksql2csv.sh
View file @
ac8773ea
...
@@ -50,6 +50,8 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" tmp.sql
...
@@ -50,6 +50,8 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" tmp.sql
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
tmp.sql
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
tmp.sql
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
tmp.sql
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
tmp.sql
[
$?
-eq
0
]
||
exit
1
# 新标签与新标签对应集合类型 的对应关系
# 新标签与新标签对应集合类型 的对应关系
/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online
-n
data
--outputformat
=
csv2
--showHeader
=
true
-e
"
`
cat
tmp.sql
`
"
>
$2
/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online
-n
data
--outputformat
=
csv2
--showHeader
=
true
-e
"
`
cat
tmp.sql
`
"
>
$2
rm
-rf
tmp.sql
rm
-rf
tmp.sql
...
...
lib/shell/sparksql2email.sh
View file @
ac8773ea
...
@@ -15,6 +15,8 @@ timestamp=`date +"%Y%m%d%H%M%S"`
...
@@ -15,6 +15,8 @@ timestamp=`date +"%Y%m%d%H%M%S"`
tmp_file_name
=
"/data/bi-report/tmp/tmp_"
$timestamp
".sql"
tmp_file_name
=
"/data/bi-report/tmp/tmp_"
$timestamp
".sql"
result_file_name
=
"/data/bi-report/result/"
$1
"_"
$partition_date
".csv"
result_file_name
=
"/data/bi-report/result/"
$1
"_"
$partition_date
".csv"
[
$?
-eq
0
]
||
exit
1
#输入的账期为空,则取T+1默认账期
#输入的账期为空,则取T+1默认账期
if
[
-z
${
V_DATE
}
]
;
then
if
[
-z
${
V_DATE
}
]
;
then
V_DATE
=
`
date
--date
=
"-1 day"
+%Y-%m-%d
`
V_DATE
=
`
date
--date
=
"-1 day"
+%Y-%m-%d
`
...
@@ -53,11 +55,17 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
...
@@ -53,11 +55,17 @@ sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_1DAY_M_D_AGO/
$V_1DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
$tmp_file_name
sed
-i
"s/
\$
V_7DAY_M_D_AGO/
$V_7DAY_M_D_AGO
/g"
$tmp_file_name
[
$?
-eq
0
]
||
exit
1
# 新标签与新标签对应集合类型 的对应关系
# 新标签与新标签对应集合类型 的对应关系
/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online
-n
data
--outputformat
=
csv2
--showHeader
=
true
-e
"
`
cat
$tmp_file_name
`
"
>
$result_file_name
/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline
-u
jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online
-n
data
--outputformat
=
csv2
--showHeader
=
true
-e
"
`
cat
$tmp_file_name
`
"
>
$result_file_name
[
$?
-eq
0
]
||
exit
1
rm
-rf
$tmp_file_name
rm
-rf
$tmp_file_name
[
$?
-eq
0
]
||
exit
1
to
=
$2
to
=
$2
cc
=
$3
cc
=
$3
echo
"to: "
$to
echo
"to: "
$to
...
...
lib/shell/waitsuccess.sh
View file @
ac8773ea
...
@@ -17,6 +17,8 @@ export DB=$2
...
@@ -17,6 +17,8 @@ export DB=$2
export
TABLE
=
$3
export
TABLE
=
$3
export
PARTITIONS
=
$yesterday_date
export
PARTITIONS
=
$yesterday_date
[
$?
-eq
0
]
||
exit
1
/bin/bash /srv/apps/data-ready-checker/client/bin/checker.sh
/bin/bash /srv/apps/data-ready-checker/client/bin/checker.sh
exit
$?
exit
$?
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment