Skip to content

C
ctr-estimate
Commit 0b5f4091 authored by 赵建伟's avatar 赵建伟
Browse files
Options

update codes

parent 334c5b10
Hide whitespace changes
Inline Side-by-side
Showing with 162 additions and 118 deletions
bin/start_tag.sh
View file @ 0b5f4091
......@@ -27,7 +27,7 @@ $JAR_DIR/ctr-estimate-1.0-SNAPSHOT.jar \
--retryInteral 3000 \
--checkpointPath 'hdfs://bj-gmei-hdfs/user/data/flink/ctr-estimate-tag/checkpoint' \
--parallelism 6 \
--startTime '2020-04-04 16:08:00' \
--startTime '2020-04-04 17:00:00' \
>> /data/log/ctr-estimate/ctr-estimate-tag.out 2>&1 &
tail -f /data/log/ctr-estimate/ctr-estimate-tag.out
......
src/main/java/com/gmei/data/ctr/sink/CtrEstimateTagMysqlSink.java
View file @ 0b5f4091
......@@ -173,11 +173,14 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
}
String platProject = deviceCurrentEstimateTagPlat.getPlatProject();
if(StringUtils.isNotBlank(platProject)){
if(platProjectObject.keySet().contains(platProject)){
Long aLong = platProjectObject.getLong(platProject);
platProjectObject.put(platProject,aLong + 1);
}else{
platProjectObject.put(platProject,1);
String[] splits = platProject.split(",");
for(String split : splits){
if(platProjectObject.keySet().contains(split)){
Long aLong = platProjectObject.getLong(split);
platProjectObject.put(split,aLong + 1);
}else{
platProjectObject.put(split,1);
}
}
}
String platFirstDemands = deviceCurrentEstimateTagPlat.getPlatFirstDemands();
......@@ -247,13 +250,13 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"plat_second_solutions = '%s'," +
"last_update_time = '%s'"
+ "where device_id = '%s' and partition_date = '%s'",
deviceCurrentEstimateTagPlat.getPlatProject(),
deviceCurrentEstimateTagPlat.getPlatFirstDemands(),
deviceCurrentEstimateTagPlat.getPlatFirstPositions(),
deviceCurrentEstimateTagPlat.getPlatFirstSolutions(),
deviceCurrentEstimateTagPlat.getPlatSecondDemands(),
deviceCurrentEstimateTagPlat.getPlatSecondPositions(),
deviceCurrentEstimateTagPlat.getPlatFirstSolutions(),
platProjectObject.toJSONString(),
platFirstDemandsObject.toJSONString(),
platFirstPositionsObject.toJSONString(),
platFirstSolutionsObject.toJSONString(),
platSecondDemandsObject.toJSONString(),
platSecondPositionsObject.toJSONString(),
platSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getLastUpdateTime(),
deviceCurrentEstimateTagPlat.getDeviceId(),
deviceCurrentEstimateTagPlat.getPartitionDate()
......@@ -275,13 +278,13 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"last_update_time"
+ ") values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')",
deviceCurrentEstimateTagPlat.getDeviceId(),
deviceCurrentEstimateTagPlat.getPlatProject(),
deviceCurrentEstimateTagPlat.getPlatFirstDemands(),
deviceCurrentEstimateTagPlat.getPlatFirstPositions(),
deviceCurrentEstimateTagPlat.getPlatFirstSolutions(),
deviceCurrentEstimateTagPlat.getPlatSecondDemands(),
deviceCurrentEstimateTagPlat.getPlatSecondPositions(),
deviceCurrentEstimateTagPlat.getPlatSecondSolutions(),
platProjectObject.toJSONString(),
platFirstDemandsObject.toJSONString(),
platFirstPositionsObject.toJSONString(),
platFirstSolutionsObject.toJSONString(),
platSecondDemandsObject.toJSONString(),
platSecondPositionsObject.toJSONString(),
platSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTagPlat.getPartitionDate(),
deviceCurrentEstimateTagPlat.getLastUpdateTime()
)
......@@ -369,128 +372,170 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
}
String commodityProject = deviceCurrentEstimateTag.getCommodityProject();
if(StringUtils.isNotBlank(commodityProject)){
if(commodityProjectObject.keySet().contains(commodityProject)){
Long aLong = commodityProjectObject.getLong(commodityProject);
commodityProjectObject.put(commodityProject,aLong + 1);
}else{
commodityProjectObject.put(commodityProject,1);
String[] splits = commodityProject.split(",");
for (String split : splits) {
if(commodityProjectObject.keySet().contains(split)){
Long aLong = commodityProjectObject.getLong(split);
commodityProjectObject.put(split,aLong + 1);
}else{
commodityProjectObject.put(split,1);
}
}
}
String commodityFirstDemands = deviceCurrentEstimateTag.getCommodityFirstDemands();
if(StringUtils.isNotBlank(commodityFirstDemands)){
if(commodityFirstDemandsObject.keySet().contains(commodityFirstDemands)){
Long aLong = commodityFirstDemandsObject.getLong(commodityFirstDemands);
commodityFirstDemandsObject.put(commodityFirstDemands,aLong + 1);
}else{
commodityFirstDemandsObject.put(commodityFirstDemands,1);
String[] splits = commodityFirstDemands.split(",");
for (String split : splits) {
if(commodityFirstDemandsObject.keySet().contains(split)){
Long aLong = commodityFirstDemandsObject.getLong(split);
commodityFirstDemandsObject.put(split,aLong + 1);
}else{
commodityFirstDemandsObject.put(split,1);
}
}
}
String commodityFirstPositions = deviceCurrentEstimateTag.getCommodityFirstPositions();
if(StringUtils.isNotBlank(commodityFirstPositions)){
if(commodityFirstPositionsObject.keySet().contains(commodityFirstPositions)){
Long aLong = commodityFirstPositionsObject.getLong(commodityFirstPositions);
commodityFirstPositionsObject.put(commodityFirstPositions,aLong + 1);
}else{
commodityFirstPositionsObject.put(commodityFirstPositions,1);
String[] splits = commodityFirstPositions.split(",");
for (String split : splits) {
if(commodityFirstPositionsObject.keySet().contains(split)){
Long aLong = commodityFirstPositionsObject.getLong(split);
commodityFirstPositionsObject.put(split,aLong + 1);
}else{
commodityFirstPositionsObject.put(split,1);
}
}
}
String commodityFirstSolutions = deviceCurrentEstimateTag.getCommodityFirstSolutions();
if(StringUtils.isNotBlank(commodityFirstSolutions)){
if(commodityFirstSolutionsObject.keySet().contains(commodityFirstSolutions)){
Long aLong = commodityFirstSolutionsObject.getLong(commodityFirstSolutions);
commodityFirstSolutionsObject.put(commodityFirstSolutions,aLong + 1);
}else{
commodityFirstSolutionsObject.put(commodityFirstSolutions,1);
String[] splits = commodityFirstSolutions.split(",");
for (String split : splits) {
if(commodityFirstSolutionsObject.keySet().contains(split)){
Long aLong = commodityFirstSolutionsObject.getLong(split);
commodityFirstSolutionsObject.put(split,aLong + 1);
}else{
commodityFirstSolutionsObject.put(split,1);
}
}
}
String commoditySecondDemands = deviceCurrentEstimateTag.getCommoditySecondDemands();
if(StringUtils.isNotBlank(commoditySecondDemands)){
if(commoditySecondDemandsObject.keySet().contains(commoditySecondDemands)){
Long aLong = commoditySecondDemandsObject.getLong(commoditySecondDemands);
commoditySecondDemandsObject.put(commoditySecondDemands,aLong + 1);
}else{
commodityFirstSolutionsObject.put(commoditySecondDemands,1);
String[] splits = commoditySecondDemands.split(",");
for (String split : splits) {
if(commoditySecondDemandsObject.keySet().contains(split)){
Long aLong = commoditySecondDemandsObject.getLong(split);
commoditySecondDemandsObject.put(split,aLong + 1);
}else{
commodityFirstSolutionsObject.put(split,1);
}
}
}
String commoditySecondPositions = deviceCurrentEstimateTag.getCommoditySecondPositions();
if(StringUtils.isNotBlank(commoditySecondPositions)){
if(commoditySecondPositionsObject.keySet().contains(commoditySecondPositions)){
Long aLong = commoditySecondPositionsObject.getLong(commoditySecondPositions);
commoditySecondPositionsObject.put(commoditySecondPositions,aLong + 1);
}else{
commoditySecondPositionsObject.put(commoditySecondPositions,1);
String[] splits = commoditySecondPositions.split(",");
for (String split : splits) {
if(commoditySecondPositionsObject.keySet().contains(split)){
Long aLong = commoditySecondPositionsObject.getLong(split);
commoditySecondPositionsObject.put(split,aLong + 1);
}else{
commoditySecondPositionsObject.put(split,1);
}
}
}
String commoditySecondSolutions = deviceCurrentEstimateTag.getCommoditySecondSolutions();
if(StringUtils.isNotBlank(commoditySecondSolutions)){
if(commoditySecondSolutionsObject.keySet().contains(commoditySecondSolutions)){
Long aLong = commoditySecondSolutionsObject.getLong(commoditySecondSolutions);
commoditySecondSolutionsObject.put(commoditySecondSolutions,aLong + 1);
}else{
commoditySecondSolutionsObject.put(commoditySecondSolutions,1);
String[] splits = commoditySecondSolutions.split(",");
for (String split : splits) {
if(commoditySecondSolutionsObject.keySet().contains(split)){
Long aLong = commoditySecondSolutionsObject.getLong(split);
commoditySecondSolutionsObject.put(split,aLong + 1);
}else{
commoditySecondSolutionsObject.put(split,1);
}
}
}
String contentProject = deviceCurrentEstimateTag.getContentProject();
if(StringUtils.isNotBlank(contentProject)){
if(contentProjectObject.keySet().contains(contentProject)){
Long aLong = contentProjectObject.getLong(contentProject);
contentProjectObject.put(contentProject,aLong + 1);
}else{
contentProjectObject.put(contentProject,1);
String[] splits = contentProject.split(",");
for (String split :splits ) {
if(contentProjectObject.keySet().contains(split)){
Long aLong = contentProjectObject.getLong(split);
contentProjectObject.put(split,aLong + 1);
}else{
contentProjectObject.put(split,1);
}
}
}
String contentFirstDemands = deviceCurrentEstimateTag.getContentFirstDemands();
if(StringUtils.isNotBlank(contentFirstDemands)){
if(contentFirstDemandsObject.keySet().contains(contentFirstDemands)){
Long aLong = contentFirstDemandsObject.getLong(contentFirstDemands);
contentFirstDemandsObject.put(contentFirstDemands,aLong + 1);
}else{
contentFirstDemandsObject.put(contentFirstDemands,1);
String[] splits = contentFirstDemands.split(",");
for (String split : splits) {
if(contentFirstDemandsObject.keySet().contains(split)){
Long aLong = contentFirstDemandsObject.getLong(split);
contentFirstDemandsObject.put(split,aLong + 1);
}else{
contentFirstDemandsObject.put(split,1);
}
}
}
String contentFirstPositions = deviceCurrentEstimateTag.getContentFirstPositions();
if(StringUtils.isNotBlank(contentFirstPositions)){
if(contentFirstPositionsObject.keySet().contains(contentFirstPositions)){
Long aLong = contentFirstPositionsObject.getLong(contentFirstPositions);
contentFirstPositionsObject.put(contentFirstPositions,aLong + 1);
}else{
contentFirstPositionsObject.put(contentFirstPositions,1);
String[] splits = contentFirstPositions.split(",");
for (String split : splits) {
if(contentFirstPositionsObject.keySet().contains(split)){
Long aLong = contentFirstPositionsObject.getLong(split);
contentFirstPositionsObject.put(split,aLong + 1);
}else{
contentFirstPositionsObject.put(split,1);
}
}
}
String contentFirstSolutions = deviceCurrentEstimateTag.getContentFirstSolutions();
if(StringUtils.isNotBlank(contentFirstSolutions)){
if(contentFirstSolutionsObject.keySet().contains(contentFirstSolutions)){
Long aLong = contentFirstSolutionsObject.getLong(contentFirstSolutions);
contentFirstSolutionsObject.put(contentFirstSolutions,aLong + 1);
}else{
contentFirstSolutionsObject.put(contentFirstSolutions,1);
String[] splits = contentFirstSolutions.split(",");
for (String split : splits) {
if(contentFirstSolutionsObject.keySet().contains(split)){
Long aLong = contentFirstSolutionsObject.getLong(split);
contentFirstSolutionsObject.put(split,aLong + 1);
}else{
contentFirstSolutionsObject.put(split,1);
}
}
}
String contentSecondDemands = deviceCurrentEstimateTag.getContentSecondDemands();
if(StringUtils.isNotBlank(contentSecondDemands)){
if(contentSecondDemandsObject.keySet().contains(contentSecondDemands)){
Long aLong = contentSecondDemandsObject.getLong(contentSecondDemands);
contentSecondDemandsObject.put(contentSecondDemands,aLong + 1);
}else{
contentSecondDemandsObject.put(contentSecondDemands,1);
String[] splits = contentSecondDemands.split(",");
for (String split : splits) {
if(contentSecondDemandsObject.keySet().contains(split)){
Long aLong = contentSecondDemandsObject.getLong(split);
contentSecondDemandsObject.put(split,aLong + 1);
}else{
contentSecondDemandsObject.put(split,1);
}
}
}
String contentSecondPositions = deviceCurrentEstimateTag.getContentSecondPositions();
if(StringUtils.isNotBlank(contentSecondPositions)){
if(contentSecondPositionsObject.keySet().contains(contentSecondPositions)){
Long aLong = contentSecondPositionsObject.getLong(contentSecondPositions);
contentSecondPositionsObject.put(contentSecondPositions,aLong + 1);
}else{
contentSecondPositionsObject.put(contentSecondPositions,1);
String[] splits = contentSecondPositions.split(",");
for (String split : splits) {
if(contentSecondPositionsObject.keySet().contains(split)){
Long aLong = contentSecondPositionsObject.getLong(split);
contentSecondPositionsObject.put(split,aLong + 1);
}else{
contentSecondPositionsObject.put(split,1);
}
}
}
String contentSecondSolutions = deviceCurrentEstimateTag.getContentSecondSolutions();
if(StringUtils.isNotBlank(contentSecondSolutions)){
if(contentSecondSolutionsObject.keySet().contains(contentSecondSolutions)){
Long aLong = contentSecondSolutionsObject.getLong(contentSecondSolutions);
contentSecondSolutionsObject.put(contentSecondSolutions,aLong + 1);
}else{
contentSecondSolutionsObject.put(contentSecondSolutions,1);
String[] splits = contentSecondSolutions.split(",");
for (String split : splits) {
if(contentSecondSolutionsObject.keySet().contains(split)){
Long aLong = contentSecondSolutionsObject.getLong(split);
contentSecondSolutionsObject.put(split,aLong + 1);
}else{
contentSecondSolutionsObject.put(split,1);
}
}
}
if(isExist){
......@@ -513,20 +558,20 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"content_second_solutions = '%s'," +
"last_update_time = '%s'" +
"where device_id = '%s' and partition_date = '%s'",
deviceCurrentEstimateTag.getCommodityProject(),
deviceCurrentEstimateTag.getCommodityFirstDemands(),
deviceCurrentEstimateTag.getCommodityFirstPositions(),
deviceCurrentEstimateTag.getCommodityFirstSolutions(),
deviceCurrentEstimateTag.getCommoditySecondDemands(),
deviceCurrentEstimateTag.getCommoditySecondPositions(),
deviceCurrentEstimateTag.getCommoditySecondSolutions(),
deviceCurrentEstimateTag.getContentProject(),
deviceCurrentEstimateTag.getContentFirstDemands(),
deviceCurrentEstimateTag.getContentFirstPositions(),
deviceCurrentEstimateTag.getContentFirstSolutions(),
deviceCurrentEstimateTag.getContentSecondDemands(),
deviceCurrentEstimateTag.getContentSecondPositions(),
deviceCurrentEstimateTag.getContentSecondSolutions(),
commodityProjectObject.toJSONString(),
commodityFirstDemandsObject.toJSONString(),
commodityFirstPositionsObject.toJSONString(),
commodityFirstSolutionsObject.toJSONString(),
commoditySecondDemandsObject.toJSONString(),
commoditySecondPositionsObject.toJSONString(),
commoditySecondSolutionsObject.toJSONString(),
contentProjectObject.toJSONString(),
contentFirstDemandsObject.toJSONString(),
contentFirstPositionsObject.toJSONString(),
contentFirstSolutionsObject.toJSONString(),
contentSecondDemandsObject.toJSONString(),
contentSecondPositionsObject.toJSONString(),
contentSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getLastUpdateTime(),
deviceCurrentEstimateTag.getDeviceId(),
deviceCurrentEstimateTag.getPartitionDate()
......@@ -555,20 +600,19 @@ public class CtrEstimateTagMysqlSink extends RichSinkFunction<DeviceCurrentEstim
"last_update_time" +
") values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')",
deviceCurrentEstimateTag.getDeviceId(),
deviceCurrentEstimateTag.getCommodityProject(),
deviceCurrentEstimateTag.getCommodityFirstDemands(),
deviceCurrentEstimateTag.getCommodityFirstPositions(),
deviceCurrentEstimateTag.getCommodityFirstSolutions(),
deviceCurrentEstimateTag.getCommoditySecondDemands(),
deviceCurrentEstimateTag.getCommoditySecondPositions(),
deviceCurrentEstimateTag.getCommoditySecondSolutions(),
deviceCurrentEstimateTag.getContentProject(),
deviceCurrentEstimateTag.getContentFirstDemands(),
deviceCurrentEstimateTag.getContentFirstPositions(),
deviceCurrentEstimateTag.getContentFirstSolutions(),
deviceCurrentEstimateTag.getContentSecondDemands(),
deviceCurrentEstimateTag.getContentSecondPositions(),
deviceCurrentEstimateTag.getContentSecondSolutions(),
commodityFirstDemandsObject.toJSONString(),
commodityFirstPositionsObject.toJSONString(),
commodityFirstSolutionsObject.toJSONString(),
commoditySecondDemandsObject.toJSONString(),
commoditySecondPositionsObject.toJSONString(),
commoditySecondSolutionsObject.toJSONString(),
contentProjectObject.toJSONString(),
contentFirstDemandsObject.toJSONString(),
contentFirstPositionsObject.toJSONString(),
contentFirstSolutionsObject.toJSONString(),
contentSecondDemandsObject.toJSONString(),
contentSecondPositionsObject.toJSONString(),
contentSecondSolutionsObject.toJSONString(),
deviceCurrentEstimateTag.getPartitionDate(),
deviceCurrentEstimateTag.getLastUpdateTime()
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment