Skip to content

B
bi-report
Commit 28c56446 authored by 赵建伟's avatar 赵建伟
Browse files
Options

refact codes

parent 5cd859a7
Hide whitespace changes
Inline Side-by-side
Showing with 52 additions and 209 deletions
.DS_Store
View file @ 28c56446
No preview for this file type
lib/java/bi-report-service/.idea/workspace.xml
View file @ 28c56446
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/daily_recommend_strategy.zip" beforeDir="false" />
</list>
<list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
......@@ -44,6 +42,13 @@
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/../../.." />
</component>
<component name="MavenImportPreferences">
<option name="importingSettings">
<MavenImportingSettings>
<option name="importAutomatically" value="true" />
</MavenImportingSettings>
</option>
</component>
<component name="ProjectCodeStyleSettingsMigration">
<option name="version" value="1" />
</component>
......@@ -65,7 +70,17 @@
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/service-client/shs" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/service-client/shs" />
</key>
</component>
<component name="RunAnythingCache">
<option name="myCommands">
<command value="mvn clean install -DskipTests" />
</option>
</component>
<component name="RunManager" selected="Application.Test">
<configuration name="ExcelServiceImpl" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
......@@ -183,6 +198,7 @@
<workItem from="1592214013610" duration="2379000" />
<workItem from="1592568026557" duration="772000" />
<workItem from="1593835574199" duration="4438000" />
<workItem from="1594091429524" duration="3648000" />
</task>
<servers />
</component>
......@@ -197,25 +213,25 @@
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state x="773" y="303" key="#com.intellij.ide.util.MemberChooser/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593835959317" />
<state width="1878" height="313" key="GridCell.Tab.0.bottom" timestamp="1593836935231">
<state width="1878" height="313" key="GridCell.Tab.0.bottom" timestamp="1594109306575">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935231" />
<state width="1878" height="313" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306575" />
<state width="1878" height="243" key="GridCell.Tab.0.bottom/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520783" />
<state width="1878" height="313" key="GridCell.Tab.0.center" timestamp="1593836935230">
<state width="1878" height="313" key="GridCell.Tab.0.center" timestamp="1594109306573">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935230" />
<state width="1878" height="313" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306573" />
<state width="1878" height="243" key="GridCell.Tab.0.center/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
<state width="1878" height="313" key="GridCell.Tab.0.left" timestamp="1593836935229">
<state width="1878" height="313" key="GridCell.Tab.0.left" timestamp="1594109306572">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935229" />
<state width="1878" height="313" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306572" />
<state width="1878" height="243" key="GridCell.Tab.0.left/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
<state width="1878" height="313" key="GridCell.Tab.0.right" timestamp="1593836935230">
<state width="1878" height="313" key="GridCell.Tab.0.right" timestamp="1594109306574">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935230" />
<state width="1878" height="313" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306574" />
<state width="1878" height="243" key="GridCell.Tab.0.right/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520782" />
<state width="1878" height="386" key="GridCell.Tab.1.bottom" timestamp="1591671347527">
<screen x="0" y="22" width="1920" height="1129" />
......@@ -233,5 +249,9 @@
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="386" key="GridCell.Tab.1.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1591671347526" />
<state x="625" y="273" width="670" height="675" key="run.anything.popup" timestamp="1594109288006">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state x="625" y="273" width="670" height="675" key="run.anything.popup/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109288006" />
</component>
</project>
\ No newline at end of file
lib/java/bi-report-service/pom.xml
View file @ 28c56446
......@@ -3,17 +3,14 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<groupId>com.gmei.data</groupId>
<artifactId>bi-report-service</artifactId>
<version>1.0.0-SNAPSHOT</version>
<name>bi-report-service</name>
<description>Demo project for Spring Boot</description>
<modules>
<module>service-server</module>
</modules>
<!--定义属性值-->
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
......@@ -21,7 +18,6 @@
<java.version>1.8</java.version>
<spring.boot.version>2.0.0.RELEASE</spring.boot.version>
</properties>
<!--Maven远程仓库设置 非必要，可以删除-->
<repositories>
<repository>
......@@ -47,7 +43,6 @@
</snapshots>
</pluginRepository>
</pluginRepositories>
<!-- 依赖声明 -->
<dependencyManagement>
<dependencies>
......@@ -61,4 +56,4 @@
</dependency>
</dependencies>
</dependencyManagement>
</project>
</project>
\ No newline at end of file
lib/java/bi-report-service/service-client/shs/email.sh 0 → 100755
View file @ 28c56446
#!/usr/bin/env bash
curl -X GET http://localhost:8553/api/report/email/demo/zhaojianwei@igengmei.com/jianweizhao@yeah.net
\ No newline at end of file
lib/java/bi-report-service/service-client/shs/start-finance.sh deleted 100644 → 0
View file @ 5cd859a7
#!/bin/bash
exit `curl -X GET http://172.16.44.8:8970/api/oozie/task/finance`
\ No newline at end of file
lib/java/bi-report-service/service-client/shs/start-oozie-finance.sh deleted 100644 → 0
View file @ 5cd859a7
#!/bin/bash
function replace_date(){
#old date
old_start_date=`cat job.properties | grep start_date | awk -F"=" '{print $2}'`
old_end_date=`cat job.properties | grep end_date | awk -F"=" '{print $2}'`
#new date
new_start_date=`date +"%Y-%m-%d"`"T00:00+0800"
new_end_date=`date +"%Y-%m-%d"`"T12:00+0800"
#replace
sed -i "s/$old_start_date/$new_start_date/g" job.properties
sed -i "s/$old_end_date/$new_end_date/g" job.properties
}
function start_job(){
export OOZIE_URL=http://bj-gm-prod-cos-datacenter005:11000/oozie
/opt/oozie-4.1.0-cdh5.16.1/bin/oozie job -config job.properties -run
#echo "/opt/oozie-4.1.0-cdh5.16.1/bin/oozie job -config job.properties -run"
}
cd /opt/warehouse/workflow/tl_finance_financeaccount_view/
replace_date
start_job
cd ../tl_finance_financeexpand_view/
replace_date
start_job
cd ../tl_finance_financerevenue_view/
replace_date
start_job
\ No newline at end of file
lib/java/bi-report-service/service-server/pom.xml
View file @ 28c56446
......@@ -7,16 +7,13 @@
<groupId>com.gmei.data</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>report-service-server</artifactId>
<properties>
<java.version>1.8</java.version>
<scala.version>2.11</scala.version>
<spark.version>2.4.5</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/endpoint/BiReportEndpoint.java
View file @ 28c56446
......@@ -17,18 +17,18 @@ public class BiReportEndpoint {
return biReportService.execSql(name);
}
@ResponseBody
@RequestMapping(value = "/execSqlByHive")
public int execSqlByHive(@RequestParam String name){
return biReportService.execSql(name);
}
@ResponseBody
@RequestMapping(value = "/genExcel")
public int genExcel(@RequestParam String name){
return biReportService.genExcel(name);
}
@ResponseBody
@RequestMapping(value = "/execSqlByHive")
public int execSqlByHive(@RequestParam String name){
return biReportService.execSql(name);
}
@ResponseBody
@RequestMapping(value = "/sendEmail/{to}/{cc}/{name}",method = RequestMethod.GET)
public int sendEmail(@PathVariable("to") String to,@PathVariable("cc") String cc,@PathVariable("name") String name){
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/EmailServiceImpl.java
View file @ 28c56446
......@@ -16,7 +16,6 @@ import org.springframework.stereotype.Component;
import javax.mail.internet.MimeMessage;
import java.io.File;
@Component
public class EmailServiceImpl implements EmailService {
@Value("${mail.fromMail.addr}")
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/ExcelServiceImpl.java
View file @ 28c56446
......@@ -64,13 +64,13 @@ public class ExcelServiceImpl implements ExcelService {
rs = Constants.FAIL;
logger.error("Error!");
}
// String subject = "";
// String value = ProperUtils.getValue(properFilePath, name);
// if(null != value){
// subject = value;
// }else{
// subject = name;
// }
//String subject = "";
//String value = ProperUtils.getValue(properFilePath, name);
//if(null != value){
// subject = value;
//}else{
// subject = name;
//}
FileOutputStream out = new FileOutputStream(filePath + name + "-" + DateUtils.getYesterdayDateSimpleStr() + ".xlsx");
wb.write(out);
out.close();
......@@ -172,9 +172,7 @@ public class ExcelServiceImpl implements ExcelService {
HSSFCellStyle style = workbook.createCellStyle();
style.setAlignment(HSSFCellStyle.ALIGN_CENTER);
style.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER);
HSSFSheet sheet = workbook.createSheet("sheet");
HSSFRow row0 = sheet.createRow(0);
HSSFCell cell_00 = row0.createCell(0);
cell_00.setCellStyle(style);
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/HivesqlServiceImpl.java
View file @ 28c56446
......@@ -62,7 +62,6 @@ public class HivesqlServiceImpl implements HivesqlService {
String hql = sbf.toString();
hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
Connection conn = null;
try{
conn = DriverManager.getConnection(hiveSqlJdbcUrl,"bi","");
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/SparksqlServiceImpl.java
View file @ 28c56446
......@@ -71,7 +71,6 @@ public class SparksqlServiceImpl implements SparksqlService {
String hql = sbf.toString();
hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
Connection conn = null;
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
......@@ -177,13 +176,11 @@ public class SparksqlServiceImpl implements SparksqlService {
List<String> sheetNameList = new ArrayList<String>();
List<List<String>> headersList = new ArrayList<List<String>>();
List<List<List>> rowsList = new ArrayList<List<List>>();
File sqlDir = new File(sqlPath + name);
File[] files = sqlDir.listFiles();
for(File file : files){
String fileName = file.getName();
String nameStr =fileName.replace(".sql","");
String subject = "";
String value = ProperUtils.getValue(properFilePath, nameStr);
if(null != value){
......@@ -192,10 +189,8 @@ public class SparksqlServiceImpl implements SparksqlService {
subject = nameStr;
}
sheetNameList.add(subject);
List<String> headers = new ArrayList<String>();
List<List> rows = new ArrayList<List>();
BufferedReader reader = null;
StringBuffer sbf = new StringBuffer();
try {
......@@ -223,7 +218,6 @@ public class SparksqlServiceImpl implements SparksqlService {
String hql = sbf.toString();
hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
Connection conn = null;
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/utils/DateUtils.java
View file @ 28c56446
......@@ -8,7 +8,6 @@ import java.util.Date;
* @author apple
*/
public class DateUtils {
private static final String DATE_FORMATE_YMDHMS = "yyyy-MM-dd HH:mm:ss";
private static final String DATE_FORMATE_YMD_SIMPLE = "yyyyMMdd";
private static final String DATE_FORMATE_YMD = "yyyy-MM-dd";
......@@ -29,14 +28,6 @@ public class DateUtils {
return new SimpleDateFormat(DATE_FORMATE_YMD).format(new Date());
}
/**
* 获取当前时间字符串
* @return
*/
// public static String getCurrentDateSimpleStr() {
// return new SimpleDateFormat(DATE_FORMATE_YMD_SIMPLE).format(new Date());
// }
/**
* 获取昨天的日期字符串
* @return
......
lib/java/bi-report-service/service-server/src/main/resources/logback.xml
View file @ 28c56446
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="true" scanPeriod="1 seconds">
<contextName>logback</contextName>
<property name="log.path" value="/data/bi-report/logs/bi-report-service.log"/>
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.path}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
......@@ -20,10 +17,8 @@
</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</configuration>
\ No newline at end of file
lib/java/bi-report-service/service-server/src/test/html/demo.html deleted 100644 → 0
View file @ 5cd859a7
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<h1 style="text-align:center;">北极星监控预警信息</h1>
<h2 style="color:red;">北极星监控预警</h2>
<h3>北极星监控预警</h3>
<h4>北极星监控预警</h4>
<h5>北极星监控预警</h5>
<h6>北极星监控预警</h6>
\ No newline at end of file
lib/java/bi-report-service/service-server/src/test/java/com/gmei/data/gateway/server/Test.java deleted 100644 → 0
View file @ 5cd859a7
package com.gmei.data.gateway.server;
import com.gmei.data.gateway.server.utils.DateUtils;
/**
* @ClassName Test
* @Author apple
* @Date 2020/7/4
* @Version V1.0
**/
public class Test {
public static void main(String[] args) {
// String str = "select a from bb;";
// System.out.println(str.replace(";",""));
String str01 = "select a from bb where aa = ${partition_date};";
String str02 = "select a from bb where aa = ${partition_day};";
String str03 = "select a from bb where aa = $partition_date;";
String str04 = "select a from bb where aa = $partition_day;";
System.out.println(str01.replace("${partition_date}",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
System.out.println(str02.replace("${partition_day}",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
System.out.println(str03.replace("$partition_date",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
System.out.println(str04.replace("$partition_day",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
}
}
lib/java/bi-report-service/service-server/target/classes/application.yml deleted 100644 → 0
View file @ 5cd859a7
server:
port: 8553
spring:
#---email config---
mail:
host: smtp.exmail.qq.com
username: youjiantongji@wanmeizhensuo.com
password: mq37tgk8nGi6eg1Hgq388oro3itvdmPl
default-encoding: UTF-8
datasource:
sparkSql:
jdbcUrl: jdbc:hive2://bj-gm-prod-cos-datacenter003:10010
hiveSql:
jdbcUrl: jdbc:hive2://bj-gm-prod-cos-datacenter006:10000
#---email send config---
mail:
fromMail:
addr: youjiantongji@wanmeizhensuo.com
content: BI报表见附件，请查收。
toMail:
bcc: zhaojianwei@igengmei.com
#---sql path---
sql:
input:
path: /home/bi/bi-report/sqls/
base:
path: /home/bi/bi-report/
#---excel config---
excel:
output:
path: /data/bi-report/result/
#---log config---
logging:
config: classpath:logback.xml
proper:
file:
path: /home/bi/bi-report/conf/en-cn.properties
#path: /Users/apple/git-repo/bi-report/conf/en-cn.properties
\ No newline at end of file
lib/java/bi-report-service/service-server/target/classes/banner.txt deleted 100644 → 0
View file @ 5cd859a7
__ .__
_______ ____ ______ ____________/ |_ ______ ______________ _|__| ____ ____
\_ __ \_/ __ \\____ \ / _ \_ __ \ __\ ______ / ___// __ \_ __ \ \/ / |/ ___\/ __ \
| | \/\ ___/| |_> > <_> ) | \/| | /_____/ \___ \\ ___/| | \/\ /| \ \__\ ___/
|__| \___ > __/ \____/|__| |__| /____ >\___ >__| \_/ |__|\___ >___ >
\/|__| \/ \/ \/ \/
lib/java/bi-report-service/service-server/target/classes/logback.xml deleted 100644 → 0
View file @ 5cd859a7
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="true" scanPeriod="1 seconds">
<contextName>logback</contextName>
<property name="log.path" value="/data/bi-report/logs/bi-report-service.log"/>
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.path}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${log.path}.%d{yyyy-MM-dd}.zip</fileNamePattern>
</rollingPolicy>
<encoder>
<pattern>%date %level [%thread] %logger{36} [%file : %line] %msg%n
</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</configuration>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment