Commit ef1aedd3 authored by 赵建伟's avatar 赵建伟

refact codes

parent 9e124d90
No preview for this file type
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/sqls/bak/demo_spark_param/demo_spark_param.sql" dialect="SQL92" />
</component>
</project>
\ No newline at end of file
No preview for this file type
......@@ -2,7 +2,14 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/../../shell/hive.sh" afterDir="false" />
<change afterPath="$PROJECT_DIR$/../../shell/hive" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/en-cn.properties" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/en-cn.properties" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/create_daliy_content.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/create_daily_content_data.sql" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/daily_content_data_insert.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/daily_content_data.sql" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step2.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step2.job" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step3.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step3.job" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step4.job" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/report/daily_content_data.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/report/select_daily_content_data.sql" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -17,9 +24,30 @@
<executable />
</execution>
</component>
<component name="DatabaseView">
<option name="SHOW_INTERMEDIATE" value="true" />
<option name="GROUP_DATA_SOURCES" value="true" />
<option name="GROUP_SCHEMA" value="true" />
<option name="GROUP_CONTENTS" value="false" />
<option name="SORT_POSITIONED" value="false" />
<option name="SHOW_EMPTY_GROUPS" value="false" />
<option name="AUTO_SCROLL_FROM_SOURCE" value="false" />
<option name="HIDDEN_KINDS">
<set />
</option>
<expand />
<select />
</component>
<component name="FavoritesManager">
<favorites_list name="bi-report-service" />
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Class" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/../../.." />
</component>
......@@ -45,7 +73,7 @@
<property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunManager" selected="JUnit.ExcelServiceTest.testGenExcel">
<component name="RunManager" selected="Application.Test">
<configuration name="ExcelServiceImpl" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.gmei.data.gateway.server.service.impl.ExcelServiceImpl" />
<module name="service-server" />
......@@ -59,6 +87,19 @@
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration name="Test" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.gmei.data.gateway.server.Test" />
<module name="service-server" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.gmei.data.gateway.server.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration default="true" type="ClojureREPL" factoryName="Local" activateToolWindowBeforeRun="false">
<setting name="replType" value="NREPL" />
<setting name="execution" value="LEININGEN" />
......@@ -125,6 +166,7 @@
</configuration>
<recent_temporary>
<list>
<item itemvalue="Application.Test" />
<item itemvalue="JUnit.ExcelServiceTest.testGenExcel" />
<item itemvalue="Application.ExcelServiceImpl" />
<item itemvalue="JUnit.EmailServiceTest.testSendAttachmentsMail" />
......@@ -146,6 +188,7 @@
<workItem from="1591668349091" duration="1455000" />
<workItem from="1592214013610" duration="2379000" />
<workItem from="1592568026557" duration="772000" />
<workItem from="1593835574199" duration="1924000" />
</task>
<servers />
</component>
......@@ -153,25 +196,29 @@
<option name="version" value="1" />
</component>
<component name="WindowStateProjectService">
<state width="1878" height="313" key="GridCell.Tab.0.bottom" timestamp="1592276447004">
<state x="773" y="303" key="#com.intellij.ide.util.MemberChooser" timestamp="1593835959317">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state x="773" y="303" key="#com.intellij.ide.util.MemberChooser/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593835959317" />
<state width="1878" height="313" key="GridCell.Tab.0.bottom" timestamp="1593836935231">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447004" />
<state width="1878" height="313" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935231" />
<state width="1878" height="243" key="GridCell.Tab.0.bottom/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520783" />
<state width="1878" height="313" key="GridCell.Tab.0.center" timestamp="1592276447002">
<state width="1878" height="313" key="GridCell.Tab.0.center" timestamp="1593836935230">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447002" />
<state width="1878" height="313" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935230" />
<state width="1878" height="243" key="GridCell.Tab.0.center/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
<state width="1878" height="313" key="GridCell.Tab.0.left" timestamp="1592276447001">
<state width="1878" height="313" key="GridCell.Tab.0.left" timestamp="1593836935229">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447001" />
<state width="1878" height="313" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935229" />
<state width="1878" height="243" key="GridCell.Tab.0.left/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
<state width="1878" height="313" key="GridCell.Tab.0.right" timestamp="1592276447003">
<state width="1878" height="313" key="GridCell.Tab.0.right" timestamp="1593836935230">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447003" />
<state width="1878" height="313" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935230" />
<state width="1878" height="243" key="GridCell.Tab.0.right/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520782" />
<state width="1878" height="386" key="GridCell.Tab.1.bottom" timestamp="1591671347527">
<screen x="0" y="22" width="1920" height="1129" />
......
......@@ -34,4 +34,10 @@ public class BiReportEndpoint {
public int sendEmail(@PathVariable("to") String to,@PathVariable("cc") String cc,@PathVariable("name") String name){
return biReportService.sendEmail(to,cc,name);
}
@ResponseBody
@RequestMapping(value = "/email/{name}/{to}/{cc}",method = RequestMethod.GET)
public int email(@PathVariable("name") String name,@PathVariable("to") String to,@PathVariable("cc") String cc){
return biReportService.email(name,to,cc);
}
}
......@@ -2,7 +2,8 @@ package com.gmei.data.gateway.server.service;
public interface BiReportService {
int execSql(String name);
int execSqlByHive(String name);
int genExcel(String name);
int execSqlByHive(String name);
int email(String name, String to, String cc);
int sendEmail(String to,String cc,String name);
}
......@@ -4,5 +4,6 @@ import com.gmei.data.gateway.server.dto.ExcelGenDto;
public interface SparksqlService {
int executeSql(String name);
ExcelGenDto getResult(String name);
ExcelGenDto getResultDto(String name);
}
......@@ -34,6 +34,21 @@ public class BiReportServiceImpl implements BiReportService {
return hivesqlService.executeSql(name);
}
@Override
public int email(String name, String to, String cc) {
int rs = Constants.SUCCESS;
ExcelGenDto resultDto = sparksqlService.getResult(name);
if(null == resultDto){
rs = Constants.FAIL;
}else{
int genExcelRs = excelService.genExcel(resultDto,name);
if(Constants.SUCCESS == genExcelRs){
rs = emailService.sendAttachmentsMail(to,cc,name);
}
}
return rs;
}
@Override
public int genExcel(String name){
ExcelGenDto resultDto = sparksqlService.getResultDto(name);
......
......@@ -29,10 +29,14 @@ public class SparksqlServiceImpl implements SparksqlService {
@Value("${sql.input.path}")
private String sqlPath;
@Value("${sql.base.path}")
private String sqlBasePath;
@Value("${datasource.sparkSql.jdbcUrl}")
private String sparkSqlJdbcUrl;
@Value("${proper.file.path}")
private String properFilePath;
@Value("${proper.base.path}")
private String properBasePath;
private static final Logger logger = LoggerFactory.getLogger(SparksqlServiceImpl.class);
@Override
......@@ -72,8 +76,8 @@ public class SparksqlServiceImpl implements SparksqlService {
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
Statement statement = conn.createStatement();
statement.executeUpdate("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar");
statement.executeUpdate("CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade'");
//statement.executeUpdate("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar");
//statement.executeUpdate("CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade'");
int updateCount = statement.executeUpdate(hql);
if(updateCount > 0){
logger.info("*** Execute By Sparksql success, hql: {}",hql);
......@@ -89,6 +93,85 @@ public class SparksqlServiceImpl implements SparksqlService {
return Constants.SUCCESS;
}
@Override
public ExcelGenDto getResult(String name) {
List<String> sheetNameList = new ArrayList<String>();
List<List<String>> headersList = new ArrayList<List<String>>();
List<List<List>> rowsList = new ArrayList<List<List>>();
File sqlDir = new File(sqlBasePath + "pm/" + name + "/report");
File[] files = sqlDir.listFiles();
for(File file : files){
String fileName = file.getName();
String nameStr =fileName.replace(".sql","");
String subject = "";
String value = ProperUtils.getValue(properBasePath + "pm/" + name + "/en-cn.properties" , nameStr);
if(null != value){
subject = value;
}else{
subject = nameStr;
}
sheetNameList.add(subject);
List<String> headers = new ArrayList<String>();
List<List> rows = new ArrayList<List>();
BufferedReader reader = null;
StringBuffer sbf = new StringBuffer();
try {
reader = new BufferedReader(new FileReader(file));
String tempStr;
while ((tempStr = reader.readLine()) != null) {
sbf.append("\n").append(tempStr);
}
reader.close();
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage());
continue;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
e1.printStackTrace();
logger.error(e1.getMessage());
continue;
}
}
}
String hql = sbf.toString();
hql = hql.replaceAll("${partition_date}", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("${partition_day}", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("$partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("$partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll(";","");
Connection conn = null;
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
Statement statement = conn.createStatement();
ResultSet rs = statement.executeQuery(hql);
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
for(int i=1;i<= columnCount;i++){
String columnName = metaData.getColumnName(i);
headers.add(columnName);
}
while(rs.next()){
List<String> rowVals = new ArrayList<String>();
for(int i=1;i<= columnCount;i++){
rowVals.add(rs.getString(i));
}
rows.add(rowVals);
}
}catch (Exception e){
e.printStackTrace();
logger.error(e.getMessage());
return null;
}
headersList.add(headers);
rowsList.add(rows);
}
return new ExcelGenDto(sheetNameList,headersList,rowsList);
}
@Override
public ExcelGenDto getResultDto(String name){
List<String> sheetNameList = new ArrayList<String>();
......
......@@ -27,6 +27,8 @@ mail:
sql:
input:
path: /home/bi/bi-report/sqls/
base:
path: /home/bi/bi-report/
#---excel config---
excel:
......@@ -40,4 +42,6 @@ logging:
proper:
file:
path: /home/bi/bi-report/conf/en-cn.properties
#path: /Users/apple/git-repo/bi-report/conf/en-cn.properties
\ No newline at end of file
#path: /Users/apple/git-repo/bi-report/conf/en-cn.properties
base:
path: /home/bi/bi-report/
package com.gmei.data.gateway.server;
import com.gmei.data.gateway.server.utils.DateUtils;
/**
* @ClassName Test
* @Author apple
* @Date 2020/7/4
* @Version V1.0
**/
public class Test {
public static void main(String[] args) {
// String str = "select a from bb;";
// System.out.println(str.replace(";",""));
String str01 = "select a from bb where aa = ${partition_date};";
String str02 = "select a from bb where aa = ${partition_day};";
String str03 = "select a from bb where aa = $partition_date;";
String str04 = "select a from bb where aa = $partition_day;";
System.out.println(str01.replace("${partition_date}",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
System.out.println(str02.replace("${partition_day}",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
System.out.println(str03.replace("$partition_date",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
System.out.println(str04.replace("$partition_day",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
}
}
server:
port: 8553
spring:
#---email config---
mail:
host: smtp.exmail.qq.com
username: youjiantongji@wanmeizhensuo.com
password: mq37tgk8nGi6eg1Hgq388oro3itvdmPl
default-encoding: UTF-8
datasource:
sparkSql:
jdbcUrl: jdbc:hive2://bj-gm-prod-cos-datacenter003:10010
hiveSql:
jdbcUrl: jdbc:hive2://bj-gm-prod-cos-datacenter006:10000
#---email send config---
mail:
fromMail:
addr: youjiantongji@wanmeizhensuo.com
content: BI报表见附件,请查收。
toMail:
bcc: zhaojianwei@igengmei.com
#---sql path---
sql:
input:
path: /home/bi/bi-report/sqls/
base:
path: /home/bi/bi-report/
#---excel config---
excel:
output:
path: /data/bi-report/result/
#---log config---
logging:
config: classpath:logback.xml
proper:
file:
path: /home/bi/bi-report/conf/en-cn.properties
#path: /Users/apple/git-repo/bi-report/conf/en-cn.properties
\ No newline at end of file
__ .__
_______ ____ ______ ____________/ |_ ______ ______________ _|__| ____ ____
\_ __ \_/ __ \\____ \ / _ \_ __ \ __\ ______ / ___// __ \_ __ \ \/ / |/ ___\/ __ \
| | \/\ ___/| |_> > <_> ) | \/| | /_____/ \___ \\ ___/| | \/\ /| \ \__\ ___/
|__| \___ > __/ \____/|__| |__| /____ >\___ >__| \_/ |__|\___ >___ >
\/|__| \/ \/ \/ \/
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="true" scanPeriod="1 seconds">
<contextName>logback</contextName>
<property name="log.path" value="/data/bi-report/logs/bi-report-service.log"/>
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.path}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${log.path}.%d{yyyy-MM-dd}.zip</fileNamePattern>
</rollingPolicy>
<encoder>
<pattern>%date %level [%thread] %logger{36} [%file : %line] %msg%n
</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</configuration>
\ No newline at end of file
#!/bin/bash
# 定义变量方便修改
hive="/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter005:10000 -n data"
#至少输入一个参数 第一个参数为需求码(必选项),第二个参数为分区日期(可选)
if [ $# -lt 1 ]; then
echo "Param project_name lost, please check again!"
exit 0
fi
#判断sql脚本是否存在
project_name=$1
sql_path="/home/bi/bi-report/pm/$project_name/etl/$project_name.sql"
if [ ! -f $sql_path ]; then
echo "Param project_name error, please check again!"
exit `$sql_path`
fi
# 如果是输入的日期按照取输入日期;如果没输入日期取当前时间的前一天
if [ -n "$2" ] ;then
partition_date=$2
else
partition_date=`date --date="-1 day" +%Y%m%d`
fi
#参数替换
tmp_sql=`cat ${sql_path}`
sql=${tmp_sql//'${partition_day}'/${partition_date}}
echo -e "${sql}"
#执行sql
$hive -e "$sql"
\ No newline at end of file
daily_content_data=内容日报-新
\ No newline at end of file
select_daily_content_data=内容日报-新
\ No newline at end of file
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
command=curl -X GET http://localhost:8553/api/report/execSql?name=daily_content_data_insert
\ No newline at end of file
command=/home/bi/bi-report/lib/shell/hive daily_content_data
\ No newline at end of file
#step3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_content_d
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_content_data/zhaojianwei@igengmei.com/jianweizhao@yeah.net
\ No newline at end of file
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,malinxi@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaojianwei@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com/daily_content_data
\ No newline at end of file
......@@ -86,4 +86,4 @@ SELECT
,per_article_pv AS `专栏PV/专栏UV`
,article_stay AS `专栏单设备时长(m)`
FROM pm.tl_pm_content_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','');
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment