update codes

.idea/workspace.xml

@@ -58,7 +58,7 @@
     <property name="WebServerToolWindowFactoryState" value="false" />
     <property name="aspect.path.notification.shown" value="true" />
     <property name="ignore_missing_gitignore" value="true" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$/bin" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
     <property name="project.structure.last.edited" value="Global Libraries" />
     <property name="project.structure.proportion" value="0.0" />
     <property name="project.structure.side.proportion" value="0.2" />
@@ -127,6 +127,7 @@
       <workItem from="1588761853482" duration="3826000" />
       <workItem from="1588905392183" duration="4522000" />
       <workItem from="1589447155495" duration="19738000" />
+      <workItem from="1590723962134" duration="470000" />
     </task>
     <task id="LOCAL-00001" summary="add init codes.">
       <created>1587723565207</created>
@@ -407,10 +408,11 @@
     <state x="560" y="344" key="Vcs.Push.Dialog.v2/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@0.22.1920.1178" timestamp="1589536752839" />
     <state x="560" y="312" key="Vcs.Push.Dialog.v2/246.1080.1920.1151/0.22.1920.1058@0.22.1920.1058" timestamp="1588232541898" />
     <state x="806" y="1395" key="Vcs.Push.Dialog.v2/246.1080.1920.1151/0.22.1920.1058@246.1080.1920.1151" timestamp="1587900239771" />
-    <state x="625" y="409" key="git4idea.merge.GitPullDialog" timestamp="1589534639344">
-      <screen x="0" y="22" width="1920" height="1178" />
+    <state x="2377" y="1578" key="git4idea.merge.GitPullDialog" timestamp="1590723993384">
+      <screen x="1752" y="1200" width="1920" height="1151" />
     </state>
     <state x="625" y="409" key="git4idea.merge.GitPullDialog/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@0.22.1920.1178" timestamp="1589534639344" />
+    <state x="2377" y="1578" key="git4idea.merge.GitPullDialog/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590723993384" />
     <state x="578" y="284" key="search.everywhere.popup" timestamp="1590043229933">
       <screen x="0" y="22" width="1920" height="1178" />
     </state>

lib/java/bi-report-service/.idea/workspace.xml

@@ -216,7 +216,7 @@
       <workItem from="1588762492573" duration="403000" />
       <workItem from="1588837331043" duration="673000" />
       <workItem from="1589455301393" duration="17006000" />
-      <workItem from="1590398287572" duration="3149000" />
+      <workItem from="1590398287572" duration="8449000" />
     </task>
     <servers />
   </component>
@@ -286,32 +286,32 @@
     </state>
     <state x="553" y="153" key="CommitChangelistDialog2/246.1080.1920.1151/0.22.1920.1058@0.22.1920.1058" timestamp="1584956237641" />
     <state x="784" y="1223" key="CommitChangelistDialog2/246.1080.1920.1151/0.22.1920.1058@246.1080.1920.1151" timestamp="1588154132365" />
-    <state width="1878" height="243" key="GridCell.Tab.0.bottom" timestamp="1590402030839">
+    <state width="1878" height="243" key="GridCell.Tab.0.bottom" timestamp="1590487369269">
       <screen x="1752" y="1200" width="1920" height="1151" />
     </state>
     <state width="1878" height="243" key="GridCell.Tab.0.bottom/0.22.1920.1129@0.22.1920.1129" timestamp="1588761074648" />
-    <state width="1878" height="243" key="GridCell.Tab.0.bottom/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590402030839" />
+    <state width="1878" height="243" key="GridCell.Tab.0.bottom/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590487369269" />
     <state width="1878" height="321" key="GridCell.Tab.0.bottom/246.1080.1920.1151/0.22.1920.1058@0.22.1920.1058" timestamp="1584688137007" />
     <state width="1878" height="243" key="GridCell.Tab.0.bottom/246.1080.1920.1151/0.22.1920.1058@246.1080.1920.1151" timestamp="1588838004638" />
-    <state width="1878" height="243" key="GridCell.Tab.0.center" timestamp="1590402030838">
+    <state width="1878" height="243" key="GridCell.Tab.0.center" timestamp="1590487369267">
       <screen x="1752" y="1200" width="1920" height="1151" />
     </state>
     <state width="1878" height="243" key="GridCell.Tab.0.center/0.22.1920.1129@0.22.1920.1129" timestamp="1588761074647" />
-    <state width="1878" height="243" key="GridCell.Tab.0.center/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590402030838" />
+    <state width="1878" height="243" key="GridCell.Tab.0.center/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590487369267" />
     <state width="1878" height="321" key="GridCell.Tab.0.center/246.1080.1920.1151/0.22.1920.1058@0.22.1920.1058" timestamp="1584688137005" />
     <state width="1878" height="243" key="GridCell.Tab.0.center/246.1080.1920.1151/0.22.1920.1058@246.1080.1920.1151" timestamp="1588838004637" />
-    <state width="1878" height="243" key="GridCell.Tab.0.left" timestamp="1590402030837">
+    <state width="1878" height="243" key="GridCell.Tab.0.left" timestamp="1590487369265">
       <screen x="1752" y="1200" width="1920" height="1151" />
     </state>
     <state width="1878" height="243" key="GridCell.Tab.0.left/0.22.1920.1129@0.22.1920.1129" timestamp="1588761074647" />
-    <state width="1878" height="243" key="GridCell.Tab.0.left/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590402030837" />
+    <state width="1878" height="243" key="GridCell.Tab.0.left/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590487369265" />
     <state width="1878" height="321" key="GridCell.Tab.0.left/246.1080.1920.1151/0.22.1920.1058@0.22.1920.1058" timestamp="1584688137005" />
     <state width="1878" height="243" key="GridCell.Tab.0.left/246.1080.1920.1151/0.22.1920.1058@246.1080.1920.1151" timestamp="1588838004636" />
-    <state width="1878" height="243" key="GridCell.Tab.0.right" timestamp="1590402030838">
+    <state width="1878" height="243" key="GridCell.Tab.0.right" timestamp="1590487369268">
       <screen x="1752" y="1200" width="1920" height="1151" />
     </state>
     <state width="1878" height="243" key="GridCell.Tab.0.right/0.22.1920.1129@0.22.1920.1129" timestamp="1588761074648" />
-    <state width="1878" height="243" key="GridCell.Tab.0.right/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590402030838" />
+    <state width="1878" height="243" key="GridCell.Tab.0.right/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590487369268" />
     <state width="1878" height="321" key="GridCell.Tab.0.right/246.1080.1920.1151/0.22.1920.1058@0.22.1920.1058" timestamp="1584688137006" />
     <state width="1878" height="243" key="GridCell.Tab.0.right/246.1080.1920.1151/0.22.1920.1058@246.1080.1920.1151" timestamp="1588838004637" />
     <state width="1878" height="395" key="GridCell.Tab.1.bottom" timestamp="1590137977817">
@@ -338,11 +338,11 @@
       <screen x="0" y="22" width="1920" height="1058" />
     </state>
     <state x="560" y="291" key="Vcs.Push.Dialog.v2/246.1080.1920.1151/0.22.1920.1058@0.22.1920.1058" timestamp="1584949293518" />
-    <state x="2221" y="1390" width="670" height="675" key="run.anything.popup" timestamp="1589943863616">
+    <state x="2221" y="1390" width="670" height="675" key="run.anything.popup" timestamp="1590402203344">
       <screen x="1752" y="1200" width="1920" height="1151" />
     </state>
     <state x="469" y="208" width="670" height="663" key="run.anything.popup/0.22.1920.1129@0.22.1920.1129" timestamp="1588345465540" />
-    <state x="2221" y="1390" width="670" height="675" key="run.anything.popup/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1589943863616" />
+    <state x="2221" y="1390" width="670" height="675" key="run.anything.popup/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590402203344" />
     <state x="715" y="1270" width="670" height="675" key="run.anything.popup/246.1080.1920.1151/0.22.1920.1058@246.1080.1920.1151" timestamp="1588837938694" />
     <state x="2330" y="1458" width="764" height="676" key="search.everywhere.popup" timestamp="1589943432014">
       <screen x="1752" y="1200" width="1920" height="1151" />

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/endpoint/BiReportEndpoint.java

@@ -17,6 +17,12 @@ public class BiReportEndpoint {
         return biReportService.execSql(name);
     }
 
+    @ResponseBody
+    @RequestMapping(value = "/execSqlByHive")
+    public int execSqlByHive(@RequestParam String name){
+        return biReportService.execSql(name);
+    }
+
     @ResponseBody
     @RequestMapping(value = "/genExcel")
     public int genExcel(@RequestParam String name){

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/BiReportService.java

@@ -2,6 +2,7 @@ package com.gmei.data.gateway.server.service;
 
 public interface BiReportService {
     int execSql(String name);
+    int execSqlByHive(String name);
     int genExcel(String name);
     int sendEmail(String to,String cc,String name);
 }

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/HivesqlService.java

 package com.gmei.data.gateway.server.service;
 
-import com.gmei.data.gateway.server.dto.ExcelGenDto;
-
 public interface HivesqlService {
     int executeSql(String name);
 }

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/SparksqlService.java

@@ -3,5 +3,6 @@ package com.gmei.data.gateway.server.service;
 import com.gmei.data.gateway.server.dto.ExcelGenDto;
 
 public interface SparksqlService {
+    int executeSql(String name);
     ExcelGenDto getResultDto(String name);
 }

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/BiReportServiceImpl.java

@@ -26,6 +26,11 @@ public class BiReportServiceImpl implements BiReportService {
 
     @Override
     public int execSql(String name){
+        return sparksqlService.executeSql(name);
+    }
+
+    @Override
+    public int execSqlByHive(String name){
         return hivesqlService.executeSql(name);
     }
 

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/HivesqlServiceImpl.java

@@ -68,13 +68,13 @@ public class HivesqlServiceImpl implements HivesqlService {
                 statement.executeUpdate("set mapreduce.job.queuename=data");
                 int updateCount = statement.executeUpdate(hql);
                 if(updateCount > 0){
-                    logger.info("*** Execute success, hql: {}",hql);
+                    logger.info("*** Execute By Hive success, hql: {}",hql);
                 }else{
-                    logger.warn("*** Execute finish, hql: {}",hql);
+                    logger.warn("*** Execute By Hive finish, hql: {}",hql);
                 }
             }catch (Exception e){
                 e.printStackTrace();
-                logger.error("*** Execute failed, error message: {}, hql: {}",e.getMessage(),hql);
+                logger.error("*** Execute By Hive failed, error message: {}, hql: {}",e.getMessage(),hql);
                 return 1;
             }
         }

lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/SparksqlServiceImpl.java

@@ -33,6 +33,56 @@ public class SparksqlServiceImpl implements SparksqlService {
     private String properFilePath;
     private static final Logger logger = LoggerFactory.getLogger(SparksqlServiceImpl.class);
 
+    @Override
+    public int executeSql(String name){
+        File sqlDir = new File(sqlPath + name);
+        File[] files = sqlDir.listFiles();
+        for(File file : files){
+            BufferedReader reader = null;
+            StringBuffer sbf = new StringBuffer();
+            try {
+                reader = new BufferedReader(new FileReader(file));
+                String tempStr;
+                while ((tempStr = reader.readLine()) != null) {
+                    sbf.append("\t").append(tempStr);
+                }
+                reader.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+                logger.error(e.getMessage());
+                continue;
+            } finally {
+                if (reader != null) {
+                    try {
+                        reader.close();
+                    } catch (IOException e1) {
+                        e1.printStackTrace();
+                        logger.error(e1.getMessage());
+                        continue;
+                    }
+                }
+            }
+            String hql = sbf.toString();
+
+            Connection conn = null;
+            try{
+                conn = DriverManager.getConnection(sparkSqlJdbcUrl,"bi","");
+                Statement statement = conn.createStatement();
+                int updateCount = statement.executeUpdate(hql);
+                if(updateCount > 0){
+                    logger.info("*** Execute By Sparksql success, hql: {}",hql);
+                }else{
+                    logger.warn("*** Execute By Sparksql finish, hql: {}",hql);
+                }
+            }catch (Exception e){
+                e.printStackTrace();
+                logger.error("*** Execute By Sparksql failed, error message: {}, hql: {}",e.getMessage(),hql);
+                return 1;
+            }
+        }
+        return 0;
+    }
+
     @Override
     public ExcelGenDto getResultDto(String name){
         List<String> sheetNameList = new ArrayList<String>();