add checkpoint and uid and parallelism

src/main/java/com/gmei/cache/MlPreciseExposureDao.java

@@ -108,6 +108,7 @@ public class MlPreciseExposureDao {
                 "   ?,\n" +
                 "   ?,\n" +
                 "   ?,\n" +
+                "   ?,\n" +
                 "   ?)";
 
         List<Object> params = new ArrayList<Object>();

src/main/java/com/gmei/function/BlDistinctProcessAllWindowFunction.java → src/main/java/com/gmei/function/BlPreciseExposureProcessFunction.java

@@ -11,7 +11,7 @@ import java.util.Map;
 import java.util.Set;
 
 /**
- * ClassName: BlDistinctProcessAllWindowFunction
+ * ClassName: BlPreciseExposureProcessFunction
  * Function:
  * Reason: BL层数据去重器
  * Date: 2020/1/8 下午5:06
@@ -19,7 +19,7 @@ import java.util.Set;
  * @author liuzhe
  * @since JDK 1.8
  */
-public class BlDistinctProcessAllWindowFunction extends ProcessAllWindowFunction<BlPreciseExposureBean, BlPreciseExposureBean, TimeWindow> {
+public class BlPreciseExposureProcessFunction extends ProcessAllWindowFunction<BlPreciseExposureBean, BlPreciseExposureBean, TimeWindow> {
     @Override
     public void process(Context context, Iterable<BlPreciseExposureBean> iterable, Collector<BlPreciseExposureBean> collector) throws Exception {
         Iterator<BlPreciseExposureBean> blPreciseExposureBeanIterator = iterable.iterator();

src/main/java/com/gmei/function/BlPreciseExposureWatermark.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
+import org.apache.flink.streaming.api.watermark.Watermark;
+
+import javax.annotation.Nullable;
+
+/**
+ * ClassName: BlPreciseExposureWatermark
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/8 下午8:40
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureWatermark implements AssignerWithPeriodicWatermarks<BlPreciseExposureBean> {
+    private long maxOutOfOrderness = 10000;
+    private long currentMaxTimestamp;
+    @Override
+    public long extractTimestamp(BlPreciseExposureBean blPreciseExposureBean, long l) {
+        Double timestampDouble = Double.parseDouble(blPreciseExposureBean.getGm_nginx_timestamp());
+        long timestamp = new Double(timestampDouble * 1000).longValue();
+        currentMaxTimestamp = Math.max(timestamp, currentMaxTimestamp);
+        return timestamp;
+    }
+
+    @Nullable
+    @Override
+    public Watermark getCurrentWatermark() {
+        return new Watermark(currentMaxTimestamp - maxOutOfOrderness);
+    }
+}

src/main/java/com/gmei/source/BlMaiDianKafkaSource.java

@@ -6,6 +6,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
 
 import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
 import java.util.Properties;
 
 
@@ -23,15 +24,17 @@ public class BlMaiDianKafkaSource {
     private String brokers;
     private String topic;
     private String groupId;
+    private String startTime;
 
 
     public BlMaiDianKafkaSource() {
     }
 
-    public BlMaiDianKafkaSource(String brokers, String topic, String groupId) {
+    public BlMaiDianKafkaSource(String brokers, String topic, String groupId, String startTime) {
         this.brokers = brokers;
         this.topic = topic;
         this.groupId = groupId;
+        this.startTime = startTime;
     }
 
     /**
@@ -42,7 +45,7 @@ public class BlMaiDianKafkaSource {
     * @author liuzhe
     * @since JDK 1.8
     */
-    public FlinkKafkaConsumer011<String> addSource() {
+    public FlinkKafkaConsumer011<String> addSource() throws Exception{
         Properties props = new Properties();
         props.put("bootstrap.servers", brokers);
         props.put("group.id", groupId);
@@ -51,10 +54,14 @@ public class BlMaiDianKafkaSource {
 //        props.put("auto.offset.reset", "earliest");
         props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
         props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
-        FlinkKafkaConsumer011<String> myConsumer = new FlinkKafkaConsumer011<String>(topic, new SimpleStringSchema(Charset.forName("UTF-8")), props);
-        myConsumer.setStartFromGroupOffsets();//默认消费策略
-//        myConsumer.setStartFromEarliest();
-        return myConsumer;
+        FlinkKafkaConsumer011<String> flinkKafkaConsumer = new FlinkKafkaConsumer011<String>(topic, new SimpleStringSchema(Charset.forName("UTF-8")), props);
+//        flinkKafkaConsumer.setStartFromGroupOffsets();//默认消费策略
+        if(startTime != null){
+            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+            flinkKafkaConsumer.setStartFromTimestamp(simpleDateFormat.parse(startTime).getTime());
+        }
+//        flinkKafkaConsumer.setStartFromEarliest();
+        return flinkKafkaConsumer;
     }
 
     public DataStreamSource<String> addSource(StreamExecutionEnvironment streamExecutionEnvironment) {
@@ -71,10 +78,10 @@ public class BlMaiDianKafkaSource {
         props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
         props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
 
-        FlinkKafkaConsumer011<String> myConsumer = new FlinkKafkaConsumer011<String>(topicName, new SimpleStringSchema(), props);
-        myConsumer.setStartFromGroupOffsets();//默认消费策略
-//        myConsumer.setStartFromEarliest();
-        DataStreamSource<String> dataStreamSource = env.addSource(myConsumer);
+        FlinkKafkaConsumer011<String> flinkKafkaConsumer = new FlinkKafkaConsumer011<String>(topicName, new SimpleStringSchema(), props);
+        flinkKafkaConsumer.setStartFromGroupOffsets();//默认消费策略
+//        flinkKafkaConsumer.setStartFromEarliest();
+        DataStreamSource<String> dataStreamSource = env.addSource(flinkKafkaConsumer);
         return dataStreamSource;
     }
 }

src/test/java/com/gmei/source/BlMaiDianKafkaSourceTest.java

@@ -25,8 +25,10 @@ public class BlMaiDianKafkaSourceTest {
 //
 //        env.setStateBackend(new Sta("file:///Users/mac/opt/flink/checkpoints",true));
 
+        String brokers = "172.16.44.25:2181/gengmei";
         String topicName = "test";
         String groupId = "group1";
+        String startTime = "2019-12-31 00:01:02";
         //构造java.util.Properties对象
         Properties props = new Properties();
         // 必须指定属性。
@@ -45,9 +47,10 @@ public class BlMaiDianKafkaSourceTest {
         props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
 
 //        FlinkKafkaConsumer011<ObjectNode> MyConsumer = new FlinkKafkaConsumer011<ObjectNode>(topicName, new JSONDeserializationSchema(), props);
-        FlinkKafkaConsumer011<String> myConsumer = new FlinkKafkaConsumer011<String>(topicName, new SimpleStringSchema(), props);
+        FlinkKafkaConsumer011<String> myConsumer = new BlMaiDianKafkaSource(brokers, topicName, groupId, startTime).addSource();
         myConsumer.setStartFromGroupOffsets();//默认消费策略
 
+
         DataStreamSource<String> text = env.addSource(myConsumer);
 
         text.print().setParallelism(1);