refact codes

lib/java/bi-report-service/.idea/workspace.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/daily_recommend_strategy.zip" beforeDir="false" />
-    </list>
+    <list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="" />
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
@@ -44,6 +42,13 @@
   <component name="Git.Settings">
     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/../../.." />
   </component>
+  <component name="MavenImportPreferences">
+    <option name="importingSettings">
+      <MavenImportingSettings>
+        <option name="importAutomatically" value="true" />
+      </MavenImportingSettings>
+    </option>
+  </component>
   <component name="ProjectCodeStyleSettingsMigration">
     <option name="version" value="1" />
   </component>
@@ -65,7 +70,17 @@
     <property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
     <property name="WebServerToolWindowFactoryState" value="false" />
     <property name="aspect.path.notification.shown" value="true" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$/service-client/shs" />
+  </component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$/service-client/shs" />
+    </key>
+  </component>
+  <component name="RunAnythingCache">
+    <option name="myCommands">
+      <command value="mvn clean install -DskipTests" />
+    </option>
   </component>
   <component name="RunManager" selected="Application.Test">
     <configuration name="ExcelServiceImpl" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
@@ -183,6 +198,7 @@
       <workItem from="1592214013610" duration="2379000" />
       <workItem from="1592568026557" duration="772000" />
       <workItem from="1593835574199" duration="4438000" />
+      <workItem from="1594091429524" duration="3648000" />
     </task>
     <servers />
   </component>
@@ -197,25 +213,25 @@
       <screen x="0" y="22" width="1920" height="1129" />
     </state>
     <state x="773" y="303" key="#com.intellij.ide.util.MemberChooser/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593835959317" />
-    <state width="1878" height="313" key="GridCell.Tab.0.bottom" timestamp="1593836935231">
+    <state width="1878" height="313" key="GridCell.Tab.0.bottom" timestamp="1594109306575">
       <screen x="0" y="22" width="1920" height="1129" />
     </state>
-    <state width="1878" height="313" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935231" />
+    <state width="1878" height="313" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306575" />
     <state width="1878" height="243" key="GridCell.Tab.0.bottom/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520783" />
-    <state width="1878" height="313" key="GridCell.Tab.0.center" timestamp="1593836935230">
+    <state width="1878" height="313" key="GridCell.Tab.0.center" timestamp="1594109306573">
       <screen x="0" y="22" width="1920" height="1129" />
     </state>
-    <state width="1878" height="313" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935230" />
+    <state width="1878" height="313" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306573" />
     <state width="1878" height="243" key="GridCell.Tab.0.center/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
-    <state width="1878" height="313" key="GridCell.Tab.0.left" timestamp="1593836935229">
+    <state width="1878" height="313" key="GridCell.Tab.0.left" timestamp="1594109306572">
       <screen x="0" y="22" width="1920" height="1129" />
     </state>
-    <state width="1878" height="313" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935229" />
+    <state width="1878" height="313" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306572" />
     <state width="1878" height="243" key="GridCell.Tab.0.left/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
-    <state width="1878" height="313" key="GridCell.Tab.0.right" timestamp="1593836935230">
+    <state width="1878" height="313" key="GridCell.Tab.0.right" timestamp="1594109306574">
       <screen x="0" y="22" width="1920" height="1129" />
     </state>
-    <state width="1878" height="313" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593836935230" />
+    <state width="1878" height="313" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109306574" />
     <state width="1878" height="243" key="GridCell.Tab.0.right/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520782" />
     <state width="1878" height="386" key="GridCell.Tab.1.bottom" timestamp="1591671347527">
       <screen x="0" y="22" width="1920" height="1129" />
@@ -233,5 +249,9 @@
       <screen x="0" y="22" width="1920" height="1129" />
     </state>
     <state width="1878" height="386" key="GridCell.Tab.1.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1591671347526" />
+    <state x="625" y="273" width="670" height="675" key="run.anything.popup" timestamp="1594109288006">
+      <screen x="0" y="22" width="1920" height="1129" />
+    </state>
+    <state x="625" y="273" width="670" height="675" key="run.anything.popup/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594109288006" />
   </component>
 </project>
\ No newline at end of file

lib/java/bi-report-service/pom.xml

@@ -3,17 +3,14 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>
-
     <groupId>com.gmei.data</groupId>
     <artifactId>bi-report-service</artifactId>
     <version>1.0.0-SNAPSHOT</version>
     <name>bi-report-service</name>
     <description>Demo project for Spring Boot</description>
-
     <modules>
         <module>service-server</module>
     </modules>
-
     <!--定义属性值-->
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -21,7 +18,6 @@
         <java.version>1.8</java.version>
         <spring.boot.version>2.0.0.RELEASE</spring.boot.version>
     </properties>
-
     <!--Maven远程仓库设置 非必要，可以删除-->
     <repositories>
         <repository>
@@ -47,7 +43,6 @@
             </snapshots>
         </pluginRepository>
     </pluginRepositories>
-
     <!-- 依赖声明 -->
     <dependencyManagement>
         <dependencies>

lib/java/bi-report-service/service-client/shs/email.sh

+#!/usr/bin/env bash
+
+curl -X GET http://localhost:8553/api/report/email/demo/zhaojianwei@igengmei.com/jianweizhao@yeah.net
\ No newline at end of file

lib/java/bi-report-service/service-client/shs/start-finance.sh

-#!/bin/bash
-
-exit `curl -X GET http://172.16.44.8:8970/api/oozie/task/finance`
\ No newline at end of file

lib/java/bi-report-service/service-client/shs/start-oozie-finance.sh

-#!/bin/bash
-
-function replace_date(){
-  #old date
-  old_start_date=`cat job.properties | grep start_date | awk -F"=" '{print $2}'`
-  old_end_date=`cat job.properties | grep end_date | awk -F"=" '{print $2}'`
-  #new date
-  new_start_date=`date +"%Y-%m-%d"`"T00:00+0800"
-  new_end_date=`date +"%Y-%m-%d"`"T12:00+0800"
-  #replace
-  sed -i "s/$old_start_date/$new_start_date/g" job.properties
-  sed -i "s/$old_end_date/$new_end_date/g" job.properties
-}
-
-function start_job(){
-  export OOZIE_URL=http://bj-gm-prod-cos-datacenter005:11000/oozie
-  /opt/oozie-4.1.0-cdh5.16.1/bin/oozie job -config job.properties -run
-  #echo "/opt/oozie-4.1.0-cdh5.16.1/bin/oozie job -config job.properties -run"
-}
-
-cd /opt/warehouse/workflow/tl_finance_financeaccount_view/
-replace_date
-start_job
-
-cd ../tl_finance_financeexpand_view/
-replace_date
-start_job
-
-cd ../tl_finance_financerevenue_view/
-replace_date
-start_job
\ No newline at end of file

lib/java/bi-report-service/service-server/pom.xml

@@ -7,16 +7,13 @@
         <groupId>com.gmei.data</groupId>
         <version>1.0.0-SNAPSHOT</version>
     </parent>
-
     <modelVersion>4.0.0</modelVersion>
     <artifactId>report-service-server</artifactId>
-
     <properties>
         <java.version>1.8</java.version>
         <scala.version>2.11</scala.version>
         <spark.version>2.4.5</spark.version>
     </properties>
-
     <dependencies>
         <dependency>
             <groupId>org.springframework.boot</groupId>

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/endpoint/BiReportEndpoint.java

@@ -17,18 +17,18 @@ public class BiReportEndpoint {
         return biReportService.execSql(name);
     }
 
-    @ResponseBody
-    @RequestMapping(value = "/execSqlByHive")
-    public int execSqlByHive(@RequestParam String name){
-        return biReportService.execSql(name);
-    }
-
     @ResponseBody
     @RequestMapping(value = "/genExcel")
     public int genExcel(@RequestParam String name){
         return biReportService.genExcel(name);
     }
 
+    @ResponseBody
+    @RequestMapping(value = "/execSqlByHive")
+    public int execSqlByHive(@RequestParam String name){
+        return biReportService.execSql(name);
+    }
+
     @ResponseBody
     @RequestMapping(value = "/sendEmail/{to}/{cc}/{name}",method = RequestMethod.GET)
     public int sendEmail(@PathVariable("to") String to,@PathVariable("cc") String cc,@PathVariable("name") String name){

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/EmailServiceImpl.java

@@ -16,7 +16,6 @@ import org.springframework.stereotype.Component;
 import javax.mail.internet.MimeMessage;
 import java.io.File;
 
-
 @Component
 public class EmailServiceImpl implements EmailService {
     @Value("${mail.fromMail.addr}")

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/ExcelServiceImpl.java

@@ -64,13 +64,13 @@ public class ExcelServiceImpl implements ExcelService {
                 rs = Constants.FAIL;
                 logger.error("Error!");
             }
-    //        String subject = "";
-    //        String value = ProperUtils.getValue(properFilePath, name);
-    //        if(null != value){
+            //String subject = "";
+            //String value = ProperUtils.getValue(properFilePath, name);
+            //if(null != value){
             //    subject = value;
-    //        }else{
+            //}else{
             //    subject = name;
-    //        }
+            //}
             FileOutputStream out = new FileOutputStream(filePath + name + "-" + DateUtils.getYesterdayDateSimpleStr() + ".xlsx");
             wb.write(out);
             out.close();
@@ -172,9 +172,7 @@ public class ExcelServiceImpl implements ExcelService {
       HSSFCellStyle style = workbook.createCellStyle();
       style.setAlignment(HSSFCellStyle.ALIGN_CENTER);
       style.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER);
-
       HSSFSheet sheet = workbook.createSheet("sheet");
-
       HSSFRow row0 = sheet.createRow(0);
       HSSFCell cell_00 = row0.createCell(0);
       cell_00.setCellStyle(style);

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/HivesqlServiceImpl.java

@@ -62,7 +62,6 @@ public class HivesqlServiceImpl implements HivesqlService {
             String hql = sbf.toString();
             hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
                     .replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
-
             Connection conn = null;
             try{
                 conn = DriverManager.getConnection(hiveSqlJdbcUrl,"bi","");

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/SparksqlServiceImpl.java

@@ -71,7 +71,6 @@ public class SparksqlServiceImpl implements SparksqlService {
             String hql = sbf.toString();
             hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
                     .replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
-
             Connection conn = null;
             try{
                 conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
@@ -177,13 +176,11 @@ public class SparksqlServiceImpl implements SparksqlService {
         List<String> sheetNameList = new ArrayList<String>();
         List<List<String>> headersList = new ArrayList<List<String>>();
         List<List<List>> rowsList = new ArrayList<List<List>>();
-
         File sqlDir = new File(sqlPath + name);
         File[] files = sqlDir.listFiles();
         for(File file : files){
             String fileName = file.getName();
             String nameStr =fileName.replace(".sql","");
-
             String subject = "";
             String value = ProperUtils.getValue(properFilePath, nameStr);
             if(null != value){
@@ -192,10 +189,8 @@ public class SparksqlServiceImpl implements SparksqlService {
                 subject = nameStr;
             }
             sheetNameList.add(subject);
-
             List<String> headers = new ArrayList<String>();
             List<List> rows = new ArrayList<List>();
-
             BufferedReader reader = null;
             StringBuffer sbf = new StringBuffer();
             try {
@@ -223,7 +218,6 @@ public class SparksqlServiceImpl implements SparksqlService {
             String hql = sbf.toString();
             hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
                     .replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
-
             Connection conn = null;
             try{
                 conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/utils/DateUtils.java

@@ -8,7 +8,6 @@ import java.util.Date;
  * @author apple
  */
 public class DateUtils {
-
 	private static final String DATE_FORMATE_YMDHMS = "yyyy-MM-dd HH:mm:ss";
 	private static final String DATE_FORMATE_YMD_SIMPLE = "yyyyMMdd";
 	private static final String DATE_FORMATE_YMD = "yyyy-MM-dd";
@@ -29,14 +28,6 @@ public class DateUtils {
 		return new SimpleDateFormat(DATE_FORMATE_YMD).format(new Date());
 	}
 
-	/**
-	 * 获取当前时间字符串
-	 * @return
-	 */
-//	public static String getCurrentDateSimpleStr() {
-//		return new SimpleDateFormat(DATE_FORMATE_YMD_SIMPLE).format(new Date());
-//	}
-
 	/**
 	 * 获取昨天的日期字符串
 	 * @return

lib/java/bi-report-service/service-server/src/main/resources/logback.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <configuration debug="false" scan="true" scanPeriod="1 seconds">
-
     <contextName>logback</contextName>
     <property name="log.path" value="/data/bi-report/logs/bi-report-service.log"/>
-
     <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
         </encoder>
     </appender>
-
     <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <file>${log.path}</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -20,10 +17,8 @@
             </pattern>
         </encoder>
     </appender>
-
     <root level="info">
         <appender-ref ref="console"/>
         <appender-ref ref="file"/>
     </root>
-
 </configuration>
\ No newline at end of file

lib/java/bi-report-service/service-server/src/test/html/demo.html

-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<h1 style="text-align:center;">北极星监控预警信息</h1>
-<h2 style="color:red;">北极星监控预警</h2>
-<h3>北极星监控预警</h3>
-<h4>北极星监控预警</h4>
-<h5>北极星监控预警</h5>
-<h6>北极星监控预警</h6>
\ No newline at end of file

lib/java/bi-report-service/service-server/src/test/java/com/gmei/data/gateway/server/Test.java

-package com.gmei.data.gateway.server;
-
-import com.gmei.data.gateway.server.utils.DateUtils;
-
-/**
- * @ClassName Test
- * @Author apple
- * @Date 2020/7/4
- * @Version V1.0
- **/
-public class Test {
-  public static void main(String[] args) {
-//    String str = "select a from bb;";
-//    System.out.println(str.replace(";",""));
-    String str01 = "select a from bb where aa =  ${partition_date};";
-    String str02 = "select a from bb where aa =  ${partition_day};";
-    String str03 = "select a from bb where aa =  $partition_date;";
-    String str04 = "select a from bb where aa =  $partition_day;";
-    System.out.println(str01.replace("${partition_date}",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
-    System.out.println(str02.replace("${partition_day}",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
-    System.out.println(str03.replace("$partition_date",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
-    System.out.println(str04.replace("$partition_day",String.format("'%s'", DateUtils.getYesterdayDateSimpleStr())).replace(";",""));
-  }
-}

lib/java/bi-report-service/service-server/target/classes/application.yml

-server:
-  port: 8553
-
-spring:
-  #---email config---
-  mail:
-    host: smtp.exmail.qq.com
-    username: youjiantongji@wanmeizhensuo.com
-    password: mq37tgk8nGi6eg1Hgq388oro3itvdmPl
-    default-encoding: UTF-8
-
-datasource:
-  sparkSql:
-    jdbcUrl: jdbc:hive2://bj-gm-prod-cos-datacenter003:10010
-  hiveSql:
-    jdbcUrl: jdbc:hive2://bj-gm-prod-cos-datacenter006:10000
-
-#---email send config---
-mail:
-  fromMail:
-    addr: youjiantongji@wanmeizhensuo.com
-    content: BI报表见附件，请查收。
-  toMail:
-    bcc: zhaojianwei@igengmei.com
-
-#---sql path---
-sql:
-  input:
-    path: /home/bi/bi-report/sqls/
-  base:
-    path: /home/bi/bi-report/
-
-#---excel config---
-excel:
-  output:
-    path: /data/bi-report/result/
-
-#---log config---
-logging:
-  config: classpath:logback.xml
-
-proper:
-  file:
-    path: /home/bi/bi-report/conf/en-cn.properties
-    #path: /Users/apple/git-repo/bi-report/conf/en-cn.properties
\ No newline at end of file

lib/java/bi-report-service/service-server/target/classes/banner.txt

-                                      __                                      .__
- _______   ____ ______   ____________/  |_            ______ ______________  _|__| ____  ____
- \_  __ \_/ __ \\____ \ /  _ \_  __ \   __\  ______  /  ___// __ \_  __ \  \/ /  |/ ___\/ __ \
-  |  | \/\  ___/|  |_> >  <_> )  | \/|  |   /_____/  \___ \\  ___/|  | \/\   /|  \  \__\  ___/
-  |__|    \___  >   __/ \____/|__|   |__|           /____  >\___  >__|    \_/ |__|\___  >___  >
-              \/|__|                                     \/     \/                    \/    \/

lib/java/bi-report-service/service-server/target/classes/logback.xml

-<?xml version="1.0" encoding="UTF-8"?>
-<configuration debug="false" scan="true" scanPeriod="1 seconds">
-
-    <contextName>logback</contextName>
-    <property name="log.path" value="/data/bi-report/logs/bi-report-service.log"/>
-
-    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>${log.path}</file>
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-            <fileNamePattern>${log.path}.%d{yyyy-MM-dd}.zip</fileNamePattern>
-        </rollingPolicy>
-        <encoder>
-            <pattern>%date %level [%thread] %logger{36} [%file : %line] %msg%n
-            </pattern>
-        </encoder>
-    </appender>
-
-    <root level="info">
-        <appender-ref ref="console"/>
-        <appender-ref ref="file"/>
-    </root>
-
-</configuration>
\ No newline at end of file