Skip to content

B
bi-report
Commit bc1725fe authored by 赵建伟's avatar 赵建伟
Browse files
Options

update codes

parent b78cda04
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 254 additions and 60 deletions
.DS_Store
View file @ bc1725fe
No preview for this file type
.idea/workspace.xml 0 → 100644
View file @ bc1725fe
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="d5f48aa6-efd5-46e7-8592-d3409cf68c59" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/lib/shell/hive.sh" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="CodeStyleSettingsInfer">
<option name="done" value="true" />
</component>
<component name="ComposerSettings">
<execution>
<executable />
</execution>
</component>
<component name="FavoritesManager">
<favorites_list name="bi-report" />
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="ProjectCodeStyleSettingsMigration">
<option name="version" value="1" />
</component>
<component name="ProjectId" id="1d1NRg7HlRiWDUuE2xo1ruXomlE" />
<component name="ProjectLevelVcsManager" settingsEditedManually="true">
<ConfirmationsSetting value="2" id="Add" />
</component>
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showExcludedFiles" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="ignore_missing_gitignore" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunManager">
<configuration default="true" type="ClojureREPL" factoryName="Local" activateToolWindowBeforeRun="false">
<setting name="replType" value="NREPL" />
<setting name="execution" value="LEININGEN" />
<setting name="jvmArgs" value="" />
<setting name="parameters" value="" />
<setting name="workingDir" value="" />
<setting name="profiles" value="" />
<setting name="aliases" value="" />
<setting name="fixLineNumbers" value="false" />
<method v="2" />
</configuration>
<configuration default="true" type="ClojureREPL" factoryName="Remote" activateToolWindowBeforeRun="false">
<setting name="host" value="" />
<setting name="port" value="0" />
<setting name="replType" value="SOCKET" />
<setting name="configType" value="SPECIFY" />
<setting name="fixLineNumbers" value="false" />
<method v="2" />
</configuration>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task id="LOCAL-00001" summary="add database name pm">
<created>1590736919279</created>
<option name="number" value="00001" />
<option name="presentableId" value="LOCAL-00001" />
<option name="project" value="LOCAL" />
<updated>1590736919279</updated>
</task>
<task active="true" id="Default" summary="Default task">
<changelist id="d5f48aa6-efd5-46e7-8592-d3409cf68c59" name="Default Changelist" comment="" />
<created>1591588632171</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1591588632171</updated>
<workItem from="1591588633866" duration="2409000" />
<workItem from="1591668557485" duration="1164000" />
<workItem from="1591757105976" duration="605000" />
<workItem from="1591843790575" duration="598000" />
<workItem from="1592568360803" duration="2738000" />
</task>
<option name="localTasksCounter" value="2" />
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="add database name pm" />
<option name="LAST_COMMIT_MESSAGE" value="add database name pm" />
</component>
<component name="WindowStateProjectService">
<state x="554" y="197" key="CommitChangelistDialog2" timestamp="1592797476112">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state x="554" y="197" key="CommitChangelistDialog2/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592797476112" />
<state x="2306" y="1378" key="CommitChangelistDialog2/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590736919071" />
<state x="2312" y="1515" key="Vcs.Push.Dialog.v2" timestamp="1590736921013">
<screen x="1752" y="1200" width="1920" height="1151" />
</state>
<state x="2312" y="1515" key="Vcs.Push.Dialog.v2/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590736921013" />
<state x="625" y="385" key="git4idea.merge.GitPullDialog" timestamp="1591588719752">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state x="625" y="385" key="git4idea.merge.GitPullDialog/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1591588719752" />
</component>
</project>
\ No newline at end of file
lib/java/bi-report-service/.idea/workspace.xml 0 → 100644
View file @ bc1725fe
This diff is collapsed.
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/common/Constants.java
View file @ bc1725fe
......@@ -5,7 +5,7 @@ package com.gmei.data.gateway.server.common;
*/
public class Constants {
//执行失败
public static Integer FAIL = 1;
public static Integer FAIL = 0;
//执行成功
public static Integer SUCCESS = 0;
public static Integer SUCCESS = 1;
}
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/ExcelServiceImpl.java
View file @ bc1725fe
......@@ -89,10 +89,12 @@ public class ExcelServiceImpl implements ExcelService {
XSSFCellStyle cellStyle=wb.createCellStyle();
cellStyle.setVerticalAlignment(VerticalAlignment.CENTER);
cellStyle.setAlignment(HorizontalAlignment.CENTER);
//cellStyle.setAlignment(HorizontalAlignment.LEFT);
cellStyle.setBorderBottom(BorderStyle.THIN);
cellStyle.setBorderLeft(BorderStyle.THIN);
cellStyle.setBorderRight(BorderStyle.THIN);
cellStyle.setBorderTop(BorderStyle.THIN);
cellStyle.setWrapText(true);//自动换行
for(int i =0;i<sheetList.size();i++){
XSSFSheet sheet = sheetList.get(i);
List<String> headers = headersList.get(i);
......@@ -101,7 +103,7 @@ public class ExcelServiceImpl implements ExcelService {
XSSFRow header = sheet.createRow(0);
//设置sheet头部列宽
for(int ii=0;ii<headers.size();ii++){
sheet.setColumnWidth(ii,20*256);
sheet.setColumnWidth(ii,12*256);
}
//给sheet头部赋值以及设置样式
Integer columIndex = 0;
......@@ -116,7 +118,7 @@ public class ExcelServiceImpl implements ExcelService {
XSSFRow hssfRow = sheet.createRow(rowIndex);
//设置单元格长度
for(int ii=0;ii<row.size();ii++){
sheet.setColumnWidth(ii,20*256);
sheet.setColumnWidth(ii,12*256);
}
//依次给单元格赋值
Integer j = 0;
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/SparksqlServiceImpl.java
View file @ bc1725fe
package com.gmei.data.gateway.server.service.impl;
import com.gmei.data.gateway.server.common.Constants;
import com.gmei.data.gateway.server.dto.ExcelGenDto;
import com.gmei.data.gateway.server.service.SparksqlService;
import com.gmei.data.gateway.server.utils.DateUtils;
......@@ -69,8 +70,10 @@ public class SparksqlServiceImpl implements SparksqlService {
Connection conn = null;
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"bi","");
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
Statement statement = conn.createStatement();
statement.executeUpdate("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar");
statement.executeUpdate("CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade'");
int updateCount = statement.executeUpdate(hql);
if(updateCount > 0){
logger.info("*** Execute By Sparksql success, hql: {}",hql);
......@@ -80,10 +83,10 @@ public class SparksqlServiceImpl implements SparksqlService {
}catch (Exception e){
e.printStackTrace();
logger.error("*** Execute By Sparksql failed, error message: {}, hql: {}",e.getMessage(),hql);
return 1;
return Constants.FAIL;
}
}
return 0;
return Constants.SUCCESS;
}
@Override
......
lib/java/bi-report-service/service-server/src/test/java/com/gmei/data/gateway/server/service/EmailServiceTest.java
View file @ bc1725fe
......@@ -19,14 +19,14 @@ public class EmailServiceTest {
@Autowired
private EmailService emailService;
@Test
public void testSendAttachmentsMail() {
emailService.sendAttachmentsMail("zhaojianwei@igengmei.com","jianweizhao@yeah.net","demo");
}
@Test
public void testSendFormattedMail() {
String htmlContent = "<h1>demo</h1>";
emailService.sendFormatedMail("zhaojianwei@igengmei.com","jianweizhao@yeah.net","demo",htmlContent);
}
// @Test
// public void testSendAttachmentsMail() {
// emailService.sendAttachmentsMail("zhaojianwei@igengmei.com","jianweizhao@yeah.net","demo");
// }
//
// @Test
// public void testSendFormattedMail() {
// String htmlContent = "<h1>demo</h1>";
// emailService.sendFormatedMail("zhaojianwei@igengmei.com","jianweizhao@yeah.net","demo",htmlContent);
// }
}
lib/java/bi-report-service/service-server/src/test/java/com/gmei/data/gateway/server/service/ExcelServiceTest.java
View file @ bc1725fe
......@@ -25,26 +25,38 @@ public class ExcelServiceTest {
@Test
public void testGenExcel() {
List<String> sheetNameList = new ArrayList<String>();
sheetNameList.add("sheet01");
List<List<String>> headersList = new ArrayList<List<String>>();
List<String> list = new ArrayList<String>();
list.add("aa");
list.add("bb");
list.add("cc");
headersList.add(list);
List<List<List>> rowsList = new ArrayList<List<List>>();
List<List> list01 = new ArrayList<List>();
List list02 = new ArrayList();
list02.add("1");
list02.add("2");
list02.add("3");
list01.add(list02);
rowsList.add(list01);
ExcelGenDto excelGenDto = new ExcelGenDto(sheetNameList,headersList,rowsList);
excelService.genExcel(excelGenDto,"报表测试");
// List<String> sheetNameList = new ArrayList<String>();
// sheetNameList.add("sheet01");
//
// List<List<String>> headersList = new ArrayList<List<String>>();
// List<String> list = new ArrayList<String>();
// list.add("日期");
// list.add("设备类型");
// list.add("活跃类型");
// list.add("是否AI渠\r\n道");
// list.add("DAU");
// list.add("内容详情\r\n页UV");
// list.add("内容详情\r\n页PV");
// list.add("内容UV/\r\nDAU");
// list.add("内容PV/\r\n内容UV");
// headersList.add(list);
//
// List<List<List>> rowsList = new ArrayList<List<List>>();
// List<List> list01 = new ArrayList<List>();
// List list02 = new ArrayList();
// list02.add("20200614");
// list02.add("android");
// list02.add("新增设备");
// list02.add("AI");
// list02.add("115");
// list02.add("11");
// list02.add("21");
// list02.add("0.0957");
// list02.add("1.9091");
// list01.add(list02);
// rowsList.add(list01);
//
// ExcelGenDto excelGenDto = new ExcelGenDto(sheetNameList,headersList,rowsList);
// excelService.genExcel(excelGenDto,"报表测试");
}
}
lib/java/bi-report-service/service-server/src/test/java/com/gmei/data/gateway/server/service/SparksqlServiceTest.java
View file @ bc1725fe
......@@ -22,26 +22,26 @@ public class SparksqlServiceTest {
@Autowired
private SparksqlService sparksqlService;
@Test
public void testGenExcel() {
ExcelGenDto resultDto = sparksqlService.getResultDto("/Users/apple/git-repo/bi-report/sqls/demo");
List<String> sheetNameList = resultDto.getSheetNameList();
for (String sheetName: sheetNameList) {
System.out.println("sheetName--"+sheetName);
}
List<List<String>> headersList = resultDto.getHeadersList();
for (List<String> list: headersList) {
for (String header : list) {
System.out.println(header);
}
}
List<List<List>> rowsList = resultDto.getRowsList();
for (List<List> list : rowsList) {
for (List rows : list) {
for (Object row: rows) {
System.out.println(row.toString());
}
}
}
}
// @Test
// public void testGenExcel() {
// ExcelGenDto resultDto = sparksqlService.getResultDto("/Users/apple/git-repo/bi-report/sqls/demo");
// List<String> sheetNameList = resultDto.getSheetNameList();
// for (String sheetName: sheetNameList) {
// System.out.println("sheetName--"+sheetName);
// }
// List<List<String>> headersList = resultDto.getHeadersList();
// for (List<String> list: headersList) {
// for (String header : list) {
// System.out.println(header);
// }
// }
// List<List<List>> rowsList = resultDto.getRowsList();
// for (List<List> list : rowsList) {
// for (List rows : list) {
// for (Object row: rows) {
// System.out.println(row.toString());
// }
// }
// }
// }
}
lib/shell/hive.sh 0 → 100755
View file @ bc1725fe
#!/bin/bash
# 定义变量方便修改
APP=gmall
hive=/opt/module/hive/bin/hive
# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n $1 ] ;then
log_date=$1
else
log_date=`date --date="$V_PARYMD -1 day" +%Y%m%d`
fi
sql="
set hive.exec.dynamic.partition.mode=nonstrict;
insert overwrite table "$APP".dwd_order_info partition(dt)
select * from "$APP".ods_order_info
where dt='$log_date' and id is not null;
insert overwrite table "$APP".dwd_order_detail partition(dt)
select * from "$APP".ods_order_detail
where dt='$log_date' and id is not null;
insert overwrite table "$APP".dwd_user_info partition(dt)
select * from "$APP".ods_user_info
where dt='$log_date' and id is not null;
insert overwrite table "$APP".dwd_payment_info partition(dt)
select * from "$APP".ods_payment_info
where dt='$log_date' and id is not null;
insert overwrite table "$APP".dwd_sku_info partition(dt)
select
sku.id,
sku.spu_id,
sku.price,
sku.sku_name,
sku.sku_desc,
sku.weight,
sku.tm_id,
sku.category3_id,
c2.id category2_id ,
c1.id category1_id,
c3.name category3_name,
c2.name category2_name,
c1.name category1_name,
sku.create_time,
sku.dt
from
"$APP".ods_sku_info sku
join "$APP".ods_base_category3 c3 on sku.category3_id=c3.id
join "$APP".ods_base_category2 c2 on c3.category2_id=c2.id
join "$APP".ods_base_category1 c1 on c2.category1_id=c1.id
where sku.dt='$log_date' and c2.dt='$log_date'
and c3.dt='$log_date' and c1.dt='$log_date'
and sku.id is not null;
"
$hive -e "$sql"
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment