Merge branch 'zhaojianwei' into 'master'

add monitor function codes

See merge request !4

src/main/java/com/gmei/data/dqmp/common/Constants.java

@@ -41,5 +41,7 @@ public class Constants {
 	public static final String REFER = "refer";
 	public static final String VOLATILITY = "volatility";
 	public static final String SPECIAL = "special";
+	// 日环比监控类型
+	public static final String MONITOR_TYPE = "day_chain";
 
 }

src/main/java/com/gmei/data/dqmp/domain/TblMonitorVolatilityRule.java

+package com.gmei.data.dqmp.domain;
+
+public class TblMonitorVolatilityRule {
+    private Integer id;
+
+    private String dbName;
+
+    private String tbName;
+
+    private String colName;
+
+    private String indicatorType;
+
+    private String monitorType;
+
+    private Double threshold;
+
+    private String checkTime;
+
+    private String comment;
+
+    public Integer getId() {
+        return id;
+    }
+
+    public void setId(Integer id) {
+        this.id = id;
+    }
+
+    public String getDbName() {
+        return dbName;
+    }
+
+    public void setDbName(String dbName) {
+        this.dbName = dbName == null ? null : dbName.trim();
+    }
+
+    public String getTbName() {
+        return tbName;
+    }
+
+    public void setTbName(String tbName) {
+        this.tbName = tbName == null ? null : tbName.trim();
+    }
+
+    public String getColName() {
+        return colName;
+    }
+
+    public void setColName(String colName) {
+        this.colName = colName == null ? null : colName.trim();
+    }
+
+    public String getIndicatorType() {
+        return indicatorType;
+    }
+
+    public void setIndicatorType(String indicatorType) {
+        this.indicatorType = indicatorType == null ? null : indicatorType.trim();
+    }
+
+    public String getMonitorType() {
+        return monitorType;
+    }
+
+    public void setMonitorType(String monitorType) {
+        this.monitorType = monitorType == null ? null : monitorType.trim();
+    }
+
+    public Double getThreshold() {
+        return threshold;
+    }
+
+    public void setThreshold(Double threshold) {
+        this.threshold = threshold;
+    }
+
+    public String getCheckTime() {
+        return checkTime;
+    }
+
+    public void setCheckTime(String checkTime) {
+        this.checkTime = checkTime == null ? null : checkTime.trim();
+    }
+
+    public String getComment() {
+        return comment;
+    }
+
+    public void setComment(String comment) {
+        this.comment = comment == null ? null : comment.trim();
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(getClass().getSimpleName());
+        sb.append(" [");
+        sb.append("Hash = ").append(hashCode());
+        sb.append(", id=").append(id);
+        sb.append(", dbName=").append(dbName);
+        sb.append(", tbName=").append(tbName);
+        sb.append(", colName=").append(colName);
+        sb.append(", indicatorType=").append(indicatorType);
+        sb.append(", monitorType=").append(monitorType);
+        sb.append(", threshold=").append(threshold);
+        sb.append(", checkTime=").append(checkTime);
+        sb.append(", comment=").append(comment);
+        sb.append("]");
+        return sb.toString();
+    }
+}
\ No newline at end of file

src/main/java/com/gmei/data/dqmp/mapper/TblMonitorVolatilityRuleMapper.java

+package com.gmei.data.dqmp.mapper;
+
+import com.gmei.data.dqmp.domain.TblMonitorVolatilityRule;
+import com.gmei.data.dqmp.domain.TblMonitorVolatilityRuleCriteria;
+import java.util.List;
+import org.apache.ibatis.annotations.Param;
+
+public interface TblMonitorVolatilityRuleMapper {
+    int countByExample(TblMonitorVolatilityRuleCriteria example);
+
+    int deleteByExample(TblMonitorVolatilityRuleCriteria example);
+
+    int deleteByPrimaryKey(Integer id);
+
+    int insert(TblMonitorVolatilityRule record);
+
+    int insertSelective(TblMonitorVolatilityRule record);
+
+    List<TblMonitorVolatilityRule> selectByExample(TblMonitorVolatilityRuleCriteria example);
+
+    TblMonitorVolatilityRule selectByPrimaryKey(Integer id);
+
+    int updateByExampleSelective(@Param("record") TblMonitorVolatilityRule record, @Param("example") TblMonitorVolatilityRuleCriteria example);
+
+    int updateByExample(@Param("record") TblMonitorVolatilityRule record, @Param("example") TblMonitorVolatilityRuleCriteria example);
+
+    int updateByPrimaryKeySelective(TblMonitorVolatilityRule record);
+
+    int updateByPrimaryKey(TblMonitorVolatilityRule record);
+}
\ No newline at end of file

src/main/java/com/gmei/data/dqmp/scheduler/MonitorScheduler.java

+package com.gmei.data.dqmp.scheduler;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.gmei.data.dqmp.service.WarningService;
+
+@Component
+public class MonitorScheduler {
+	
+	@Autowired
+	private WarningService warningService;
+	
+    @Scheduled(cron="*/60 * * * * ?")
+    private void uniqueProcess(){
+    	warningService.checkAndWarning();
+    }
+}

src/main/java/com/gmei/data/dqmp/service/WarningService.java

+package com.gmei.data.dqmp.service;
+
+public interface WarningService {
+
+	void checkAndWarning();
+
+}

src/main/java/com/gmei/data/dqmp/service/impl/CheckServiceImpl.java

@@ -176,7 +176,8 @@ public class CheckServiceImpl implements CheckService {
 	 * @return
 	 */
 	private TblResultCheckUnique getUniqueCheckResult(String sql) {
-		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()));
+		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()))
+				.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateStr()));
 		logger.info("Sql content : {}", sql);
 		TblResultCheckUnique tblResultCheckUnique = null;
 		if (StringUtils.isBlank(sql)) {
@@ -212,7 +213,8 @@ public class CheckServiceImpl implements CheckService {
 	 * @return
 	 */
 	private TblResultCheckUnblank getUnblankCheckResult(String sql) {
-		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()));
+		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()))
+				.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateStr()));
 		logger.info("Sql content : {}", sql);
 		TblResultCheckUnblank tblResultCheckUnblank = null;
 		if (StringUtils.isBlank(sql)) {
@@ -248,7 +250,8 @@ public class CheckServiceImpl implements CheckService {
 	 * @return
 	 */
 	private TblResultCheckRefer getReferCheckResult(String sql) {
-		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()));
+		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()))
+				.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateStr()));
 		logger.info("Sql content : {}", sql);
 		TblResultCheckRefer tblResultCheckRefer = null;
 		if (StringUtils.isBlank(sql)) {
@@ -280,8 +283,4 @@ public class CheckServiceImpl implements CheckService {
 		}
 		return tblResultCheckRefer;
 	}
-	
-	public static void main(String[] args) {
-		String sql = "";
-	}
 }

src/main/java/com/gmei/data/dqmp/service/impl/MonitorServiceImpl.java

@@ -139,7 +139,8 @@ public class MonitorServiceImpl implements MonitorService {
 	 * @return
 	 */
 	private TblResultMonitorVolatility getMonitorVolatilityResult(String sql, String indicatorType) {
-		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()));
+		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()))
+				.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateStr()));
 		logger.info("Sql content : {},indicatorType: {}", sql, indicatorType);
 		TblResultMonitorVolatility tblResultMonitorVolatility = null;
 		if (StringUtils.isBlank(sql) || StringUtils.isBlank(indicatorType)) {
@@ -176,6 +177,7 @@ public class MonitorServiceImpl implements MonitorService {
 	 */
 	private TblResultMonitorSpecial getMonitorSpecialResult(String sql,String startTime) {
 		sql = sql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateStr()))
+				 .replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateStr()))
 				 .replaceAll("#start_time", "'" + DateUtils.getZeroTimeStrsMap().get(startTime) + "'")
 				 .replaceAll("#end_time", "'" + DateUtils.getTodayZeroTimeStr() + "'");
 		logger.info("Sql content : {}", sql);

src/main/java/com/gmei/data/dqmp/service/impl/WarningServiceImpl.java

+package com.gmei.data.dqmp.service.impl;
+
+import java.util.Date;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.gmei.data.dqmp.common.Constants;
+import com.gmei.data.dqmp.domain.TblMonitorVolatilityRule;
+import com.gmei.data.dqmp.domain.TblMonitorVolatilityRuleCriteria;
+import com.gmei.data.dqmp.domain.TblResultMonitorVolatility;
+import com.gmei.data.dqmp.domain.TblResultMonitorVolatilityCriteria;
+import com.gmei.data.dqmp.domain.TblResultMonitorVolatilityCriteria.Criteria;
+import com.gmei.data.dqmp.mapper.TblMonitorVolatilityRuleMapper;
+import com.gmei.data.dqmp.mapper.TblResultMonitorVolatilityMapper;
+import com.gmei.data.dqmp.service.DingdingService;
+import com.gmei.data.dqmp.service.MailService;
+import com.gmei.data.dqmp.service.WarningService;
+import com.gmei.data.dqmp.utils.CrontabUtils;
+import com.gmei.data.dqmp.utils.DateUtils;
+
+@Service
+public class WarningServiceImpl implements WarningService {
+
+	@Autowired
+	private DingdingService dingdingService;
+	@Autowired
+	private MailService mailService;
+	@Autowired
+	private TblMonitorVolatilityRuleMapper tblMonitorVolatilityRuleMapper;
+	@Autowired
+	private TblResultMonitorVolatilityMapper tblResultMonitorVolatilityMapper;
+	private static final Logger logger = LoggerFactory.getLogger(WarningServiceImpl.class);
+
+	@Override
+	public void checkAndWarning() {
+		List<TblMonitorVolatilityRule> tblMonitorVolatilityRuleList = tblMonitorVolatilityRuleMapper
+				.selectByExample(new TblMonitorVolatilityRuleCriteria());
+		Date currentDate = new Date();
+		String lastOneDayDateStr = DateUtils.getLastNumDaysDateStr(1);
+		String lastTwoDayDateStr = DateUtils.getLastNumDaysDateStr(2);
+		for (TblMonitorVolatilityRule tblMonitorVolatilityRule : tblMonitorVolatilityRuleList) {
+			if (Constants.MONITOR_TYPE.equals(tblMonitorVolatilityRule.getMonitorType())) {
+				String checkTime = tblMonitorVolatilityRule.getCheckTime();
+				String dbName = tblMonitorVolatilityRule.getDbName();
+				String tbName = tblMonitorVolatilityRule.getTbName();
+				if (!CrontabUtils.isToRunByMinites(currentDate, checkTime)) {
+					continue;
+				}
+				TblResultMonitorVolatilityCriteria tblResultMonitorVolatilityCriteria = new TblResultMonitorVolatilityCriteria();
+				Criteria criteria = tblResultMonitorVolatilityCriteria.createCriteria();
+				criteria.andDbNameEqualTo(dbName).andTbNameEqualTo(tbName)
+						.andColNameEqualTo(tblMonitorVolatilityRule.getColName())
+						.andIndicatorTypeEqualTo(tblMonitorVolatilityRule.getIndicatorType())
+						.andPartitionDateBetween(lastTwoDayDateStr, lastOneDayDateStr);
+				tblResultMonitorVolatilityCriteria.setOrderByClause("partition_date");
+				List<TblResultMonitorVolatility> tblResultMonitorVolatilityList = tblResultMonitorVolatilityMapper
+						.selectByExample(tblResultMonitorVolatilityCriteria);
+				Long lastOneDayIndicatorValue = null, lastTwoDayIndicatorValue = null;
+				for (TblResultMonitorVolatility tblResultMonitorVolatility : tblResultMonitorVolatilityList) {
+					String partitionDate = tblResultMonitorVolatility.getPartitionDate();
+					if (lastOneDayDateStr.equals(partitionDate)) {
+						lastOneDayIndicatorValue = tblResultMonitorVolatility.getIndicatorValue();
+					} else if (lastTwoDayDateStr.equals(partitionDate)) {
+						lastTwoDayIndicatorValue = tblResultMonitorVolatility.getIndicatorValue();
+					} else {
+						logger.error("PartitionDate value is error: {}", partitionDate);
+						continue;
+					}
+				}
+				if (lastOneDayIndicatorValue == null || lastTwoDayIndicatorValue == null) {
+					logger.error("The lastOneDayIndicatorValue or lastTwoDayIndicatorValue or all values of table {} is empty!", tblMonitorVolatilityRule.getTbName());
+					return;
+				}
+				Double rs = Math.abs((lastOneDayIndicatorValue - lastTwoDayIndicatorValue)
+						/ Double.valueOf(lastTwoDayIndicatorValue));
+				Double threshold = tblMonitorVolatilityRule.getThreshold();
+				if (rs > threshold) {
+					String warningMsg = String.format("表%s.%s中数据量变化超过%s，疑似异常，请核实!", dbName, tbName,threshold * 100 + "%");
+					mailService.sendSimpleMail(warningMsg);
+					dingdingService.sendMsgToDingding(warningMsg);
+					logger.error("表{}.{}中数据量变化超过{}，疑似异常，请核实!", dbName, tbName,threshold * 100+"%");
+				}
+			}
+		}
+	}
+}

src/main/java/com/gmei/data/dqmp/utils/DateUtils.java

@@ -23,6 +23,17 @@ public class DateUtils {
     	return new SimpleDateFormat(DATE_FORMATE_YMD).format(new Date());
     }
 	
+	/**
+	 * 获取上N天时间字符串
+	 * @return
+	 */
+	public static String getLastNumDaysDateStr(int daysNum) {
+		Calendar cld = Calendar.getInstance();
+		cld.setTime(new Date());
+		cld.add(Calendar.DAY_OF_MONTH, -daysNum);
+		return new SimpleDateFormat(DATE_FORMATE_YMD).format(cld.getTime());
+	}
+	
 	/**
 	 * 获取当前时间字符串
 	 * @return

src/main/resources/generator.xml

@@ -54,6 +54,7 @@
         	<generatedKey column="id" sqlStatement="MySql" identity="true"/>
         </table> -->
         <!-- <table tableName="tbl_sys_param_info" domainObjectName="TblSysParamInfo"/> -->
-        <table tableName="tbl_threshold_exceed_all" domainObjectName="TblThresholdExceedAll"/>
+        <!-- <table tableName="tbl_threshold_exceed_all" domainObjectName="TblThresholdExceedAll"/> -->
+        <table tableName="tbl_monitor_volatility_rule" domainObjectName="TblMonitorVolatilityRule"/>
     </context>
 </generatorConfiguration>