Skip to content

C
ctr-estimate
Commit 0b5f4091 authored by 赵建伟's avatar 赵建伟
Browse files
Options

update codes

parent 334c5b10
Hide whitespace changes
Inline Side-by-side
Showing with 162 additions and 118 deletions
bin/start_tag.sh
View file @ 0b5f4091
...@@ -27,7 +27,7 @@ $JAR_DIR/ctr-estimate-1.0-SNAPSHOT.jar \ ...@@ -27,7 +27,7 @@ $JAR_DIR/ctr-estimate-1.0-SNAPSHOT.jar \
--retryInteral 3000 \ --retryInteral 3000 \
--checkpointPath 'hdfs://bj-gmei-hdfs/user/data/flink/ctr-estimate-tag/checkpoint' \ --checkpointPath 'hdfs://bj-gmei-hdfs/user/data/flink/ctr-estimate-tag/checkpoint' \
--parallelism 6 \ --parallelism 6 \
--startTime '2020-04-04 16:08:00' \ --startTime '2020-04-04 17:00:00' \
>> /data/log/ctr-estimate/ctr-estimate-tag.out 2>&1 & >> /data/log/ctr-estimate/ctr-estimate-tag.out 2>&1 &
tail -f /data/log/ctr-estimate/ctr-estimate-tag.out tail -f /data/log/ctr-estimate/ctr-estimate-tag.out
......
src/main/java/com/gmei/data/ctr/sink/CtrEstimateTagMysqlSink.java
View file @ 0b5f4091
...@@ -173,11 +173,14 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim ...@@ -173,11 +173,14 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
} }
String platProject = deviceCurrentEstimateTagPlat.getPlatProject(); String platProject = deviceCurrentEstimateTagPlat.getPlatProject();
if(StringUtils.isNotBlank(platProject)){ if(StringUtils.isNotBlank(platProject)){
if(platProjectObject.keySet().contains(platProject)){ String[] splits = platProject.split(",");
Long aLong = platProjectObject.getLong(platProject); for(String split : splits){
platProjectObject.put(platProject,aLong + 1); if(platProjectObject.keySet().contains(split)){
}else{ Long aLong = platProjectObject.getLong(split);
platProjectObject.put(platProject,1); platProjectObject.put(split,aLong + 1);
}else{
platProjectObject.put(split,1);
}
} }
} }
String platFirstDemands = deviceCurrentEstimateTagPlat.getPlatFirstDemands(); String platFirstDemands = deviceCurrentEstimateTagPlat.getPlatFirstDemands();
...@@ -247,13 +250,13 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim ...@@ -247,13 +250,13 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"plat_second_solutions = '%s'," + "plat_second_solutions = '%s'," +
"last_update_time = '%s'" "last_update_time = '%s'"
+ "where device_id = '%s' and partition_date = '%s'", + "where device_id = '%s' and partition_date = '%s'",
deviceCurrentEstimateTagPlat.getPlatProject(), platProjectObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatFirstDemands(), platFirstDemandsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatFirstPositions(), platFirstPositionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatFirstSolutions(), platFirstSolutionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatSecondDemands(), platSecondDemandsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatSecondPositions(), platSecondPositionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatFirstSolutions(), platSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getLastUpdateTime(), deviceCurrentEstimateTagPlat.getLastUpdateTime(),
deviceCurrentEstimateTagPlat.getDeviceId(), deviceCurrentEstimateTagPlat.getDeviceId(),
deviceCurrentEstimateTagPlat.getPartitionDate() deviceCurrentEstimateTagPlat.getPartitionDate()
...@@ -275,13 +278,13 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim ...@@ -275,13 +278,13 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"last_update_time" "last_update_time"
+ ") values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')", + ") values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')",
deviceCurrentEstimateTagPlat.getDeviceId(), deviceCurrentEstimateTagPlat.getDeviceId(),
deviceCurrentEstimateTagPlat.getPlatProject(), platProjectObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatFirstDemands(), platFirstDemandsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatFirstPositions(), platFirstPositionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatFirstSolutions(), platFirstSolutionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatSecondDemands(), platSecondDemandsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatSecondPositions(), platSecondPositionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPlatSecondSolutions(), platSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPartitionDate(), deviceCurrentEstimateTagPlat.getPartitionDate(),
deviceCurrentEstimateTagPlat.getLastUpdateTime() deviceCurrentEstimateTagPlat.getLastUpdateTime()
) )
...@@ -369,128 +372,170 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim ...@@ -369,128 +372,170 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
} }
String commodityProject = deviceCurrentEstimateTag.getCommodityProject(); String commodityProject = deviceCurrentEstimateTag.getCommodityProject();
if(StringUtils.isNotBlank(commodityProject)){ if(StringUtils.isNotBlank(commodityProject)){
if(commodityProjectObject.keySet().contains(commodityProject)){ String[] splits = commodityProject.split(",");
Long aLong = commodityProjectObject.getLong(commodityProject); for (String split : splits) {
commodityProjectObject.put(commodityProject,aLong + 1); if(commodityProjectObject.keySet().contains(split)){
}else{ Long aLong = commodityProjectObject.getLong(split);
commodityProjectObject.put(commodityProject,1); commodityProjectObject.put(split,aLong + 1);
}else{
commodityProjectObject.put(split,1);
}
} }
} }
String commodityFirstDemands = deviceCurrentEstimateTag.getCommodityFirstDemands(); String commodityFirstDemands = deviceCurrentEstimateTag.getCommodityFirstDemands();
if(StringUtils.isNotBlank(commodityFirstDemands)){ if(StringUtils.isNotBlank(commodityFirstDemands)){
if(commodityFirstDemandsObject.keySet().contains(commodityFirstDemands)){ String[] splits = commodityFirstDemands.split(",");
Long aLong = commodityFirstDemandsObject.getLong(commodityFirstDemands); for (String split : splits) {
commodityFirstDemandsObject.put(commodityFirstDemands,aLong + 1); if(commodityFirstDemandsObject.keySet().contains(split)){
}else{ Long aLong = commodityFirstDemandsObject.getLong(split);
commodityFirstDemandsObject.put(commodityFirstDemands,1); commodityFirstDemandsObject.put(split,aLong + 1);
}else{
commodityFirstDemandsObject.put(split,1);
}
} }
} }
String commodityFirstPositions = deviceCurrentEstimateTag.getCommodityFirstPositions(); String commodityFirstPositions = deviceCurrentEstimateTag.getCommodityFirstPositions();
if(StringUtils.isNotBlank(commodityFirstPositions)){ if(StringUtils.isNotBlank(commodityFirstPositions)){
if(commodityFirstPositionsObject.keySet().contains(commodityFirstPositions)){ String[] splits = commodityFirstPositions.split(",");
Long aLong = commodityFirstPositionsObject.getLong(commodityFirstPositions); for (String split : splits) {
commodityFirstPositionsObject.put(commodityFirstPositions,aLong + 1); if(commodityFirstPositionsObject.keySet().contains(split)){
}else{ Long aLong = commodityFirstPositionsObject.getLong(split);
commodityFirstPositionsObject.put(commodityFirstPositions,1); commodityFirstPositionsObject.put(split,aLong + 1);
}else{
commodityFirstPositionsObject.put(split,1);
}
} }
} }
String commodityFirstSolutions = deviceCurrentEstimateTag.getCommodityFirstSolutions(); String commodityFirstSolutions = deviceCurrentEstimateTag.getCommodityFirstSolutions();
if(StringUtils.isNotBlank(commodityFirstSolutions)){ if(StringUtils.isNotBlank(commodityFirstSolutions)){
if(commodityFirstSolutionsObject.keySet().contains(commodityFirstSolutions)){ String[] splits = commodityFirstSolutions.split(",");
Long aLong = commodityFirstSolutionsObject.getLong(commodityFirstSolutions); for (String split : splits) {
commodityFirstSolutionsObject.put(commodityFirstSolutions,aLong + 1); if(commodityFirstSolutionsObject.keySet().contains(split)){
}else{ Long aLong = commodityFirstSolutionsObject.getLong(split);
commodityFirstSolutionsObject.put(commodityFirstSolutions,1); commodityFirstSolutionsObject.put(split,aLong + 1);
}else{
commodityFirstSolutionsObject.put(split,1);
}
} }
} }
String commoditySecondDemands = deviceCurrentEstimateTag.getCommoditySecondDemands(); String commoditySecondDemands = deviceCurrentEstimateTag.getCommoditySecondDemands();
if(StringUtils.isNotBlank(commoditySecondDemands)){ if(StringUtils.isNotBlank(commoditySecondDemands)){
if(commoditySecondDemandsObject.keySet().contains(commoditySecondDemands)){ String[] splits = commoditySecondDemands.split(",");
Long aLong = commoditySecondDemandsObject.getLong(commoditySecondDemands); for (String split : splits) {
commoditySecondDemandsObject.put(commoditySecondDemands,aLong + 1); if(commoditySecondDemandsObject.keySet().contains(split)){
}else{ Long aLong = commoditySecondDemandsObject.getLong(split);
commodityFirstSolutionsObject.put(commoditySecondDemands,1); commoditySecondDemandsObject.put(split,aLong + 1);
}else{
commodityFirstSolutionsObject.put(split,1);
}
} }
} }
String commoditySecondPositions = deviceCurrentEstimateTag.getCommoditySecondPositions(); String commoditySecondPositions = deviceCurrentEstimateTag.getCommoditySecondPositions();
if(StringUtils.isNotBlank(commoditySecondPositions)){ if(StringUtils.isNotBlank(commoditySecondPositions)){
if(commoditySecondPositionsObject.keySet().contains(commoditySecondPositions)){ String[] splits = commoditySecondPositions.split(",");
Long aLong = commoditySecondPositionsObject.getLong(commoditySecondPositions); for (String split : splits) {
commoditySecondPositionsObject.put(commoditySecondPositions,aLong + 1); if(commoditySecondPositionsObject.keySet().contains(split)){
}else{ Long aLong = commoditySecondPositionsObject.getLong(split);
commoditySecondPositionsObject.put(commoditySecondPositions,1); commoditySecondPositionsObject.put(split,aLong + 1);
}else{
commoditySecondPositionsObject.put(split,1);
}
} }
} }
String commoditySecondSolutions = deviceCurrentEstimateTag.getCommoditySecondSolutions(); String commoditySecondSolutions = deviceCurrentEstimateTag.getCommoditySecondSolutions();
if(StringUtils.isNotBlank(commoditySecondSolutions)){ if(StringUtils.isNotBlank(commoditySecondSolutions)){
if(commoditySecondSolutionsObject.keySet().contains(commoditySecondSolutions)){ String[] splits = commoditySecondSolutions.split(",");
Long aLong = commoditySecondSolutionsObject.getLong(commoditySecondSolutions); for (String split : splits) {
commoditySecondSolutionsObject.put(commoditySecondSolutions,aLong + 1); if(commoditySecondSolutionsObject.keySet().contains(split)){
}else{ Long aLong = commoditySecondSolutionsObject.getLong(split);
commoditySecondSolutionsObject.put(commoditySecondSolutions,1); commoditySecondSolutionsObject.put(split,aLong + 1);
}else{
commoditySecondSolutionsObject.put(split,1);
}
} }
} }
String contentProject = deviceCurrentEstimateTag.getContentProject(); String contentProject = deviceCurrentEstimateTag.getContentProject();
if(StringUtils.isNotBlank(contentProject)){ if(StringUtils.isNotBlank(contentProject)){
if(contentProjectObject.keySet().contains(contentProject)){ String[] splits = contentProject.split(",");
Long aLong = contentProjectObject.getLong(contentProject); for (String split :splits ) {
contentProjectObject.put(contentProject,aLong + 1); if(contentProjectObject.keySet().contains(split)){
}else{ Long aLong = contentProjectObject.getLong(split);
contentProjectObject.put(contentProject,1); contentProjectObject.put(split,aLong + 1);
}else{
contentProjectObject.put(split,1);
}
} }
} }
String contentFirstDemands = deviceCurrentEstimateTag.getContentFirstDemands(); String contentFirstDemands = deviceCurrentEstimateTag.getContentFirstDemands();
if(StringUtils.isNotBlank(contentFirstDemands)){ if(StringUtils.isNotBlank(contentFirstDemands)){
if(contentFirstDemandsObject.keySet().contains(contentFirstDemands)){ String[] splits = contentFirstDemands.split(",");
Long aLong = contentFirstDemandsObject.getLong(contentFirstDemands); for (String split : splits) {
contentFirstDemandsObject.put(contentFirstDemands,aLong + 1); if(contentFirstDemandsObject.keySet().contains(split)){
}else{ Long aLong = contentFirstDemandsObject.getLong(split);
contentFirstDemandsObject.put(contentFirstDemands,1); contentFirstDemandsObject.put(split,aLong + 1);
}else{
contentFirstDemandsObject.put(split,1);
}
} }
} }
String contentFirstPositions = deviceCurrentEstimateTag.getContentFirstPositions(); String contentFirstPositions = deviceCurrentEstimateTag.getContentFirstPositions();
if(StringUtils.isNotBlank(contentFirstPositions)){ if(StringUtils.isNotBlank(contentFirstPositions)){
if(contentFirstPositionsObject.keySet().contains(contentFirstPositions)){ String[] splits = contentFirstPositions.split(",");
Long aLong = contentFirstPositionsObject.getLong(contentFirstPositions); for (String split : splits) {
contentFirstPositionsObject.put(contentFirstPositions,aLong + 1); if(contentFirstPositionsObject.keySet().contains(split)){
}else{ Long aLong = contentFirstPositionsObject.getLong(split);
contentFirstPositionsObject.put(contentFirstPositions,1); contentFirstPositionsObject.put(split,aLong + 1);
}else{
contentFirstPositionsObject.put(split,1);
}
} }
} }
String contentFirstSolutions = deviceCurrentEstimateTag.getContentFirstSolutions(); String contentFirstSolutions = deviceCurrentEstimateTag.getContentFirstSolutions();
if(StringUtils.isNotBlank(contentFirstSolutions)){ if(StringUtils.isNotBlank(contentFirstSolutions)){
if(contentFirstSolutionsObject.keySet().contains(contentFirstSolutions)){ String[] splits = contentFirstSolutions.split(",");
Long aLong = contentFirstSolutionsObject.getLong(contentFirstSolutions); for (String split : splits) {
contentFirstSolutionsObject.put(contentFirstSolutions,aLong + 1); if(contentFirstSolutionsObject.keySet().contains(split)){
}else{ Long aLong = contentFirstSolutionsObject.getLong(split);
contentFirstSolutionsObject.put(contentFirstSolutions,1); contentFirstSolutionsObject.put(split,aLong + 1);
}else{
contentFirstSolutionsObject.put(split,1);
}
} }
} }
String contentSecondDemands = deviceCurrentEstimateTag.getContentSecondDemands(); String contentSecondDemands = deviceCurrentEstimateTag.getContentSecondDemands();
if(StringUtils.isNotBlank(contentSecondDemands)){ if(StringUtils.isNotBlank(contentSecondDemands)){
if(contentSecondDemandsObject.keySet().contains(contentSecondDemands)){ String[] splits = contentSecondDemands.split(",");
Long aLong = contentSecondDemandsObject.getLong(contentSecondDemands); for (String split : splits) {
contentSecondDemandsObject.put(contentSecondDemands,aLong + 1); if(contentSecondDemandsObject.keySet().contains(split)){
}else{ Long aLong = contentSecondDemandsObject.getLong(split);
contentSecondDemandsObject.put(contentSecondDemands,1); contentSecondDemandsObject.put(split,aLong + 1);
}else{
contentSecondDemandsObject.put(split,1);
}
} }
} }
String contentSecondPositions = deviceCurrentEstimateTag.getContentSecondPositions(); String contentSecondPositions = deviceCurrentEstimateTag.getContentSecondPositions();
if(StringUtils.isNotBlank(contentSecondPositions)){ if(StringUtils.isNotBlank(contentSecondPositions)){
if(contentSecondPositionsObject.keySet().contains(contentSecondPositions)){ String[] splits = contentSecondPositions.split(",");
Long aLong = contentSecondPositionsObject.getLong(contentSecondPositions); for (String split : splits) {
contentSecondPositionsObject.put(contentSecondPositions,aLong + 1); if(contentSecondPositionsObject.keySet().contains(split)){
}else{ Long aLong = contentSecondPositionsObject.getLong(split);
contentSecondPositionsObject.put(contentSecondPositions,1); contentSecondPositionsObject.put(split,aLong + 1);
}else{
contentSecondPositionsObject.put(split,1);
}
} }
} }
String contentSecondSolutions = deviceCurrentEstimateTag.getContentSecondSolutions(); String contentSecondSolutions = deviceCurrentEstimateTag.getContentSecondSolutions();
if(StringUtils.isNotBlank(contentSecondSolutions)){ if(StringUtils.isNotBlank(contentSecondSolutions)){
if(contentSecondSolutionsObject.keySet().contains(contentSecondSolutions)){ String[] splits = contentSecondSolutions.split(",");
Long aLong = contentSecondSolutionsObject.getLong(contentSecondSolutions); for (String split : splits) {
contentSecondSolutionsObject.put(contentSecondSolutions,aLong + 1); if(contentSecondSolutionsObject.keySet().contains(split)){
}else{ Long aLong = contentSecondSolutionsObject.getLong(split);
contentSecondSolutionsObject.put(contentSecondSolutions,1); contentSecondSolutionsObject.put(split,aLong + 1);
}else{
contentSecondSolutionsObject.put(split,1);
}
} }
} }
if(isExist){ if(isExist){
...@@ -513,20 +558,20 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim ...@@ -513,20 +558,20 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"content_second_solutions = '%s'," + "content_second_solutions = '%s'," +
"last_update_time = '%s'" + "last_update_time = '%s'" +
"where device_id = '%s' and partition_date = '%s'", "where device_id = '%s' and partition_date = '%s'",
deviceCurrentEstimateTag.getCommodityProject(), commodityProjectObject.toJSONString(),
deviceCurrentEstimateTag.getCommodityFirstDemands(), commodityFirstDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getCommodityFirstPositions(), commodityFirstPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getCommodityFirstSolutions(), commodityFirstSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getCommoditySecondDemands(), commoditySecondDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getCommoditySecondPositions(), commoditySecondPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getCommoditySecondSolutions(), commoditySecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentProject(), contentProjectObject.toJSONString(),
deviceCurrentEstimateTag.getContentFirstDemands(), contentFirstDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getContentFirstPositions(), contentFirstPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentFirstSolutions(), contentFirstSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentSecondDemands(), contentSecondDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getContentSecondPositions(), contentSecondPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentSecondSolutions(), contentSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getLastUpdateTime(), deviceCurrentEstimateTag.getLastUpdateTime(),
deviceCurrentEstimateTag.getDeviceId(), deviceCurrentEstimateTag.getDeviceId(),
deviceCurrentEstimateTag.getPartitionDate() deviceCurrentEstimateTag.getPartitionDate()
...@@ -555,20 +600,19 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim ...@@ -555,20 +600,19 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"last_update_time" + "last_update_time" +
") values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')", ") values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')",
deviceCurrentEstimateTag.getDeviceId(), deviceCurrentEstimateTag.getDeviceId(),
deviceCurrentEstimateTag.getCommodityProject(), commodityFirstDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getCommodityFirstDemands(), commodityFirstPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getCommodityFirstPositions(), commodityFirstSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getCommodityFirstSolutions(), commoditySecondDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getCommoditySecondDemands(), commoditySecondPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getCommoditySecondPositions(), commoditySecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getCommoditySecondSolutions(), contentProjectObject.toJSONString(),
deviceCurrentEstimateTag.getContentProject(), contentFirstDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getContentFirstDemands(), contentFirstPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentFirstPositions(), contentFirstSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentFirstSolutions(), contentSecondDemandsObject.toJSONString(),
deviceCurrentEstimateTag.getContentSecondDemands(), contentSecondPositionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentSecondPositions(), contentSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getContentSecondSolutions(),
deviceCurrentEstimateTag.getPartitionDate(), deviceCurrentEstimateTag.getPartitionDate(),
deviceCurrentEstimateTag.getLastUpdateTime() deviceCurrentEstimateTag.getLastUpdateTime()
) )
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment