add bl_et_mg_preciseexposure_inc_d sink

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/bean/bl/BlPreciseExposureBean.java

@@ -13,53 +13,6 @@ import java.util.ArrayList;
  * @author liuzhe
  * @since JDK 1.8
  */
-/*
-{
-    "gm_nginx_key":2,
-    "version":"110",
-    "params":{
-        "up_slide_times":0,
-        "down_loading_times":0,
-        "exposure_cards":{
-
-        },
-        "up_loading_times":0,
-        "is_exposure":1,
-        "tab_name":"精选",
-        "referrer_id":"",
-        "down_slide_times":0,
-        "referrer":"",
-        "page_name":"home",
-        "business_id":""
-    },
-    "app_session_id":"C3280044-5C8E-459D-AE6C-8E26BBACD6C9",
-    "gm_nginx_timestamp":1546307958.631,
-    "create_at":"1546307958",
-    "app":{
-        "channel":"AppStore",
-        "version":"7.7.35",
-        "serial_id":42,
-        "current_city_id":"worldwide",
-        "name":"gengmei_user",
-        "user_type":{
-
-        }
-    },
-    "device":{
-        "is_WiFi":"0",
-        "device_type":"ios",
-        "device_id":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
-        "lng":0,
-        "lat":0,
-        "ip":"10.156.100.97",
-        "manufacturer":"Apple",
-        "idfa":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
-        "idfv":"352A6D64-17CA-4520-831A-2CE9507631D8"
-    },
-    "user_id":"30864538",
-    "type":"home_choiceness_card_exposure"
-}
- */
 public class BlPreciseExposureBean {
 
     private String json;

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/cache/BlPreciseExposureDao.java

+package com.gmei.cache;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.bean.bl.BlPreciseExposureParamsExposureCardsBean;
+import com.gmei.jdbc.MysqlJdbcSink;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ClassName: BlPreciseExposureDao
+ * Function:
+ * Reason: bl_et_mg_preciseexposure_inc_d_rt数据下发操作类
+ * Date: 2019/12/19 上午11:35
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureDao {
+    private MysqlJdbcSink mysqlJdbcSink;
+    private String sql;
+    private String sinkJdbcUrl;
+    private String sinkTableName;
+
+    public BlPreciseExposureDao(String sinkJdbcUrl, String sinkTableName) {
+        this.sinkJdbcUrl = sinkJdbcUrl;
+        this.sinkTableName = sinkTableName;
+        this.mysqlJdbcSink = MysqlJdbcSink.getInstance(sinkJdbcUrl);
+    }
+
+    /**
+    * Function: insertBlPreciseExposure
+    * Reason: 向表中插入数据
+    * Date: 2019/12/25 下午5:38
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public void insertBlPreciseExposure(BlPreciseExposureBean blPreciseExposureBean) throws Exception {
+        sql = "insert into " + sinkTableName + "\n" +
+                "  (json,\n" +
+                "   gm_nginx_timestamp,\n" +
+                "   create_timestamp,\n" +
+                "   user_id,\n" +
+                "   action,\n" +
+                "   down_loading_times,\n" +
+                "   down_slide_times,\n" +
+                "   up_loading_times,\n" +
+                "   up_slide_times,\n" +
+                "   page_code,\n" +
+                "   tab_name,\n" +
+                "   business_id,\n" +
+                "   referrer_code,\n" +
+                "   referrer_id,\n" +
+                "   exposure_cards,\n" +
+                "   is_exposure,\n" +
+                "   is_popup,\n" +
+                "   filter,\n" +
+                "   query,\n" +
+                "   app_grey_type,\n" +
+                "   app_channel,\n" +
+                "   app_version,\n" +
+                "   app_current_city_id,\n" +
+                "   app_code,\n" +
+                "   device_os_type,\n" +
+                "   device_model,\n" +
+                "   device_id,\n" +
+                "   device_android_id,\n" +
+                "   device_idfv,\n" +
+                "   gm_nginx_time_date,\n" +
+                "   gm_nginx_time_day,\n" +
+                "   create_time_date,\n" +
+                "   create_time_day)\n" +
+                "values\n" +
+                "  (?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?)";
+
+        List<Object> params = new ArrayList<Object>();
+        String json = blPreciseExposureBean.getJson();
+//        Clob clob = connection.createClob();
+//        clob.setString(1, json);
+        params.add(json);
+        params.add(blPreciseExposureBean.getGm_nginx_timestamp());
+        params.add(blPreciseExposureBean.getCreate_timestamp());
+        params.add(blPreciseExposureBean.getUser_id());
+        params.add(blPreciseExposureBean.getAction());
+        params.add(blPreciseExposureBean.getDown_loading_times());
+        params.add(blPreciseExposureBean.getDown_slide_times());
+        params.add(blPreciseExposureBean.getUp_loading_times());
+        params.add(blPreciseExposureBean.getUp_slide_times());
+        params.add(blPreciseExposureBean.getPage_code());
+        params.add(blPreciseExposureBean.getTab_name());
+        params.add(blPreciseExposureBean.getBusiness_id());
+        params.add(blPreciseExposureBean.getReferrer_code());
+        params.add(blPreciseExposureBean.getReferrer_id());
+        ArrayList<BlPreciseExposureParamsExposureCardsBean> blPreciseExposureCardsBeans= blPreciseExposureBean.getExposure_cards();
+        JSONArray jsonArrayCardsBeans= JSONArray.parseArray(JSON.toJSONString(blPreciseExposureCardsBeans));
+        params.add(jsonArrayCardsBeans.toString());
+        params.add(blPreciseExposureBean.getIs_exposure());
+        params.add(blPreciseExposureBean.getIs_popup());
+        params.add(blPreciseExposureBean.getFilter());
+        params.add(blPreciseExposureBean.getQuery());
+        params.add(blPreciseExposureBean.getApp_grey_type());
+        params.add(blPreciseExposureBean.getApp_channel());
+        params.add(blPreciseExposureBean.getApp_version());
+        params.add(blPreciseExposureBean.getApp_current_city_id());
+        params.add(blPreciseExposureBean.getApp_code());
+        params.add(blPreciseExposureBean.getDevice_os_type());
+        params.add(blPreciseExposureBean.getDevice_model());
+        params.add(blPreciseExposureBean.getDevice_id());
+        params.add(blPreciseExposureBean.getDevice_android_id());
+        params.add(blPreciseExposureBean.getDevice_idfv());
+        params.add(blPreciseExposureBean.getGm_nginx_time_date());
+        params.add(blPreciseExposureBean.getGm_nginx_time_day());
+        params.add(blPreciseExposureBean.getCreate_time_date());
+        params.add(blPreciseExposureBean.getCreate_time_day());
+//        System.out.println(params.toString());
+        mysqlJdbcSink.update(sql, params);
+    }
+}

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/function/BlPreciseExposureMapFunction.java

@@ -67,7 +67,7 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
         String create_time_day = null;
 
         //去除掉json中的回车、换行、制表、空格
-        String jsonString = string.replaceAll("\\\\s+|\\\\\\\\\\\\\\\\n", "");
+        String jsonString = string.replaceAll("\\s+|\\\\n", "");
 
         JSONObject jsonObject = JSON.parseObject(jsonString);
 
@@ -121,7 +121,7 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
                 JSONObject jsonCard = jsonArrayExposureCards.getJSONObject(i);
                 String card_id = jsonCard.getString("card_id");
                 String transaction_type = jsonCard.getString("transaction_type");
-                String card_content_type = jsonCard.getString("card_content_type ");
+                String card_content_type = jsonCard.getString("card_content_type");
                 String card_type = jsonCard.getString("card_type");
                 String card_name = jsonCard.getString("card_name");
                 String target_name = jsonCard.getString("target_name");

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/function/BlPreciseExposureWatermark.java

@@ -16,12 +16,14 @@ import javax.annotation.Nullable;
  * @since JDK 1.8
  */
 public class BlPreciseExposureWatermark implements AssignerWithPeriodicWatermarks<BlPreciseExposureBean> {
-    private long maxOutOfOrderness = 10000;
+    private final long maxOutOfOrderness = 10000;
     private long currentMaxTimestamp;
     @Override
     public long extractTimestamp(BlPreciseExposureBean blPreciseExposureBean, long l) {
-        Double timestampDouble = Double.parseDouble(blPreciseExposureBean.getGm_nginx_timestamp());
-        long timestamp = new Double(timestampDouble * 1000).longValue();
+//        Double timestampDouble = Double.parseDouble(blPreciseExposureBean.getGm_nginx_timestamp());
+//        long timestamp = new Double(timestampDouble * 1000).longValue();
+        Double timestampDouble = Double.parseDouble(blPreciseExposureBean.getGm_nginx_timestamp()) * 1000;
+        long timestamp = timestampDouble.longValue();
         currentMaxTimestamp = Math.max(timestamp, currentMaxTimestamp);
         return timestamp;
     }

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/sink/BlPreciseExposureKafkaSink.java

+package com.gmei.sink;
+
+import com.alibaba.fastjson.JSON;
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.cache.BlPreciseExposureDao;
+import com.gmei.cache.SimpleCacheService;
+import com.gmei.jdbc.MysqlJdbcSink;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+/**
+ * ClassName: BlPreciseExposureMysqlSink
+ * Function:
+ * Reason: 数据下发mysql
+ * Date: 2019/12/16 下午6:45
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureKafkaSink extends RichSinkFunction<BlPreciseExposureBean> {
+
+    private String outBrokers;
+    private String outTopic;
+    private Properties props;
+    private KafkaProducer<String,String> kafkaProducer;
+
+
+    public BlPreciseExposureKafkaSink(String outBrokers, String outTopic) {
+        this.outBrokers = outBrokers;
+        this.outTopic = outTopic;
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        init();
+    }
+
+    @Override
+    public void invoke(BlPreciseExposureBean value, Context context) throws Exception {
+        kafkaProducer.send(new ProducerRecord<String,String>(outTopic, JSON.toJSONString(value)));
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        kafkaProducer.close();
+    }
+
+    private void init() throws Exception {
+        props = new Properties();
+        props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,outBrokers);
+        props.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG,"lz4");
+        props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
+        props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
+        kafkaProducer = new KafkaProducer(props);
+    }
+}
\ No newline at end of file

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/sink/BlPreciseExposureMysqlSink.java

+package com.gmei.sink;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.cache.BlPreciseExposureDao;
+import com.gmei.jdbc.MysqlJdbcSink;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+
+import java.sql.Connection;
+
+/**
+ * ClassName: BlPreciseExposureMysqlSink
+ * Function:
+ * Reason: 数据下发mysql
+ * Date: 2019/12/16 下午6:45
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureMysqlSink extends RichSinkFunction<BlPreciseExposureBean> {
+
+    private MysqlJdbcSink mysqlJdbcSink;
+    private Connection conn;
+    private BlPreciseExposureDao blPreciseExposureDao;
+    private int maxRetry = 1;
+    private long retryTime = 3000;
+    private String sinkJdbcUrl;
+    private String sinkTableName;
+
+    public BlPreciseExposureMysqlSink(String sinkJdbcUrl, String sinkTableName) {
+        this.sinkJdbcUrl = sinkJdbcUrl;
+        this.sinkTableName = sinkTableName;
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        mysqlJdbcSink = MysqlJdbcSink.getInstance(sinkJdbcUrl);
+        blPreciseExposureDao = new BlPreciseExposureDao(sinkJdbcUrl, sinkTableName);
+        conn = mysqlJdbcSink.getConnection();
+    }
+
+    @Override
+    public void invoke(BlPreciseExposureBean value, Context context) throws Exception {
+
+        try {
+            conn.setAutoCommit(false);
+            blPreciseExposureDao.insertBlPreciseExposure(value);
+            conn.commit();
+        } catch (Exception e) {
+            conn.rollback();
+            int numRetry = 1;
+            Exception lastException = e;
+            while (numRetry <= maxRetry) {
+                try {
+                    numRetry ++;
+                    Thread.sleep(retryTime);
+                    mysqlJdbcSink.close(conn, null, null);
+                    conn = mysqlJdbcSink.getConnection();
+                    conn.setAutoCommit(false);
+                    blPreciseExposureDao.insertBlPreciseExposure(value);
+                    conn.commit();
+                } catch (Exception e1) {
+                    conn.rollback();
+                    lastException = e1;
+                    continue;
+                }
+                return;
+            }
+            throw lastException;
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        mysqlJdbcSink.close(conn, null, null);
+    }
+}

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/source/BlMaiDianKafkaSource.java

@@ -55,10 +55,12 @@ public class BlMaiDianKafkaSource {
         props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
         props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
         FlinkKafkaConsumer011<String> flinkKafkaConsumer = new FlinkKafkaConsumer011<String>(topic, new SimpleStringSchema(Charset.forName("UTF-8")), props);
-//        flinkKafkaConsumer.setStartFromGroupOffsets();//默认消费策略
+//
         if(startTime != null){
             SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
             flinkKafkaConsumer.setStartFromTimestamp(simpleDateFormat.parse(startTime).getTime());
+        } else {
+            flinkKafkaConsumer.setStartFromGroupOffsets();//默认消费策略
         }
 //        flinkKafkaConsumer.setStartFromEarliest();
         return flinkKafkaConsumer;

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/sql/dim_table_ddl.sql

@@ -68,7 +68,42 @@ CREATE TABLE `dim_transaction_type` (
   `oid` int COMMENT '排序' DEFAULT NULL
 ) COMMENT '业务类型码表' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
 ;
-
+CREATE TABLE `bl_et_mg_preciseexposure_inc_d_rt` (
+`json` longtext comment '原始JSON',
+`gm_nginx_timestamp` varchar(200) comment '接受日志时间戳' default null,
+`create_timestamp` varchar(200) comment '创建日志时间戳' default null,
+`user_id` varchar(200) comment '用户ID' default null,
+`action` varchar(200) comment '事件接口' default null,
+`down_loading_times` int comment '下拉加载次数' default null,
+`down_slide_times` int comment '下拉滑动次数' default null,
+`up_loading_times` int comment '上拉加载次数' default null,
+`up_slide_times` int comment '上拉滑动次数' default null,
+`page_code` varchar(200) comment '页面编码' default null,
+`tab_name` varchar(200) comment 'TAB名称' default null,
+`business_id` varchar(200) comment '业务ID' default null,
+`referrer_code` varchar(200) comment '来源页编码' default null,
+`referrer_id` varchar(200) comment '来源页业务ID' default null,
+`exposure_cards` varchar(10000) comment '卡片列表' default null,
+`is_exposure` varchar(200) comment '是否精准曝光' default null,
+`is_popup` varchar(200) comment '是否弹窗' default null,
+`filter` varchar(200) comment '筛选器' default null,
+`query` varchar(200) comment '搜索词' default null,
+`app_grey_type` varchar(200) comment '灰度列表' default null,
+`app_channel` varchar(200) comment 'APP渠道' default null,
+`app_version` varchar(200) comment 'APP版本' default null,
+`app_current_city_id` varchar(200) comment '当前城市ID' default null,
+`app_code` varchar(200) comment 'APP编码' default null,
+`device_os_type` varchar(200) comment '设备系统类型' default null,
+`device_model` varchar(200) comment '设备型号' default null,
+`device_id` varchar(200) comment '设备ID' default null,
+`device_android_id` varchar(200) comment '设备安卓ID' default null,
+`device_idfv` varchar(200) comment '设备IDFV' default null,
+`gm_nginx_time_date` varchar(200) comment '日志接收时间' default null,
+`gm_nginx_time_day` varchar(200) comment '日志接收日期' default null,
+`create_time_date` varchar(200) comment '日志创建时间' default null,
+`create_time_day` varchar(200) comment '日志创建日期' default null
+) COMMENT 'BL层精准曝光实时表' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+;
 CREATE TABLE `ml_c_et_pe_preciseexposure_dimen_d_rt` (
 `user_id` varchar(200) comment '用户ID' default null,
 `action` varchar(200) comment '事件接口' default null,
@@ -106,7 +141,7 @@ CREATE TABLE `ml_c_et_pe_preciseexposure_dimen_d_rt` (
 `create_time_day` varchar(200) comment '日志创建时间' default null,
 `gm_nginx_time_day` varchar(200) comment '日志接收时间' default null,
 `preciseexposure_num` int comment '精准曝光数量' default null
-) COMMENT '精准曝光实时表' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+) COMMENT 'ML层精准曝光实时表' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
 ;
 
 

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/sql/dim_table_dml.sql

@@ -3,6 +3,7 @@ INSERT INTO `dim_transaction_type`(`code`, `pk`, `name`, `memo`, `oid`) VALUES (
 INSERT INTO `dim_transaction_type`(`code`, `pk`, `name`, `memo`, `oid`) VALUES ('community', 'community', '社区', '社区', 2);
 INSERT INTO `dim_transaction_type`(`code`, `pk`, `name`, `memo`, `oid`) VALUES ('operating', 'operating', '运营', '运营', 3);
 INSERT INTO `dim_transaction_type`(`code`, `pk`, `name`, `memo`, `oid`) VALUES ('supply', 'supply', '补位', '补位', 5);
+INSERT INTO `dim_transaction_type`(`code`, `pk`, `name`, `memo`, `oid`) VALUES ('operation', 'operation', '运营', '运营', 6);
 INSERT INTO `dim_city`(`code`, `pk`, `name`, `memo`, `parent_province_code`, `parent_province_pk`, `parent_province_name`, `parent_province_memo`, `parent_country_code`, `parent_country_pk`, `parent_country_name`, `parent_country_memo`, `parent_region_code`, `parent_region_pk`, `parent_region_name`, `parent_region_memo`, `oid`) VALUES ('aba', '298', '阿坝', '阿坝', 'sichuan', '289', '四川', '四川', 'china', '259', '中国', '中国', 'xinan', '6', '西南', '西南', 1);
 INSERT INTO `dim_city`(`code`, `pk`, `name`, `memo`, `parent_province_code`, `parent_province_pk`, `parent_province_name`, `parent_province_memo`, `parent_country_code`, `parent_country_pk`, `parent_country_name`, `parent_country_memo`, `parent_region_code`, `parent_region_pk`, `parent_region_name`, `parent_region_memo`, `oid`) VALUES ('akesu', '299', '阿克苏', '阿克苏', 'xinjiang', '293', '新疆', '新疆', 'china', '259', '中国', '中国', 'xinan', '6', '西南', '西南', 2);
 INSERT INTO `dim_city`(`code`, `pk`, `name`, `memo`, `parent_province_code`, `parent_province_pk`, `parent_province_name`, `parent_province_memo`, `parent_country_code`, `parent_country_pk`, `parent_country_name`, `parent_country_memo`, `parent_region_code`, `parent_region_pk`, `parent_region_name`, `parent_region_memo`, `oid`) VALUES ('alashan', '300', '阿拉善', '阿拉善', 'neimenggu', '281', '内蒙古', '内蒙古', 'china', '259', '中国', '中国', 'huabei', '2', '华北', '华北', 3);

ml_c_et_pe_preciseexposure_dimem_d_rt/src/main/java/com/gmei/streaming/PreciseExposureStreaming.java

@@ -3,6 +3,7 @@ package com.gmei.streaming;
 import com.gmei.bean.bl.BlPreciseExposureBean;
 import com.gmei.bean.ml.MlPreciseExposureBean;
 import com.gmei.function.*;
+import com.gmei.sink.BlPreciseExposureMysqlSink;
 import com.gmei.sink.MlPreciseExposureMysqlSink;
 import com.gmei.source.BlMaiDianKafkaSource;
 import org.apache.flink.api.common.restartstrategy.RestartStrategies;
@@ -14,6 +15,8 @@ import org.apache.flink.streaming.api.datastream.*;
 import org.apache.flink.streaming.api.environment.CheckpointConfig;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.streaming.api.windowing.triggers.CountTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
 import org.apache.flink.util.OutputTag;
 
@@ -35,7 +38,7 @@ public class PreciseExposureStreaming {
     * Function: main
     * Reason: 执行入口
     * Date: 2019/12/24 上午11:03
-    * args: inBrokes、inTopic、inzk、groupId、outTableName、outBrokers、outTopic、dimJdbcUrl、sinkJdbcUrl
+    * args: inBrokes、inTopic、inzk、groupId、sinkBlTableName、sinkMlTableName、outBrokers、outTopic、dimJdbcUrl、sinkJdbcUrl
     * 运行参数设置:
       * 时间属性: ProcessingTime
       * 聚合窗口时间: 1分钟
@@ -52,7 +55,8 @@ public class PreciseExposureStreaming {
         String groupId = null;
         String dimJdbcUrl = null;
         String sinkJdbcUrl = null;
-        String sinkTableName = null;
+        String sinkBlTableName = null;
+        String sinkMlTableName = null;
         Integer windowSize = null;
         Integer parallelism = null;
         String startTime = null;
@@ -69,7 +73,8 @@ public class PreciseExposureStreaming {
 
             dimJdbcUrl = parameterTool.getRequired("dimJdbcUrl");
             sinkJdbcUrl = parameterTool.getRequired("sinkJdbcUrl");
-            sinkTableName = parameterTool.getRequired("sinkTableName");
+            sinkBlTableName = parameterTool.getRequired("sinkBlTableName");
+            sinkMlTableName = parameterTool.getRequired("sinkMlTableName");
 //        Boolean startFromLatest = parameterTool.getBoolean("startFromLatest", false);
             windowSize = parameterTool.getInt("windowSize", 30);
             parallelism = parameterTool.getInt("parallelism", 1);
@@ -105,18 +110,20 @@ public class PreciseExposureStreaming {
         env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 3000));
 
         //设置时间属性（EventTime:事件时间;IngestionTime:接入时间;ProcessingTime:处理时间（默认））
-        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+        env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+//        DataStream<String> blMaiDianDataStream = env.socketTextStream("MacdeMacBook-Pro-9.local", 9000, "\n");
 
         //正式环境
         BlMaiDianKafkaSource blMaiDianKafkaSource = new BlMaiDianKafkaSource(inBrokers, inTopic, groupId, startTime);
         //测试环境
 //        BlMaiDianKafkaSource blMaiDianKafkaSource = new BlMaiDianKafkaSource("localhost:9092", "test", "group1");
 
-        FlinkKafkaConsumer011<String> myConsumer = blMaiDianKafkaSource.addSource();
+        FlinkKafkaConsumer011<String> flinkKafkaConsumer = blMaiDianKafkaSource.addSource();
 
         //maidian流分发
         SplitStream<String> blMaiDianDataStream  = env
-                .addSource(myConsumer).uid("id_blmaidiandata_source").setParallelism(parallelism)
+                .addSource(flinkKafkaConsumer).uid("id_blmaidiandata_source").setParallelism(parallelism)
                 .split(new BlMaiDianOutputSelector());
 
         OutputTag<BlPreciseExposureBean> outputTag = new OutputTag<BlPreciseExposureBean>("bl_preciseexposure_late_data"){};
@@ -128,43 +135,57 @@ public class PreciseExposureStreaming {
         @SuppressWarnings("unchecked")
         SingleOutputStreamOperator<BlPreciseExposureBean> blPreciseExposureStream = blMaiDianDataStream
                 .select("et_pe")
-                .map(new BlPreciseExposureMapFunction())
-                .uid("id_blpreciseexposure_map").setParallelism(parallelism)
-                .filter(new BlPreciseExposureFilterFunction())
-                .uid("id_blpreciseexposure_filter").setParallelism(parallelism)
-                .assignTimestampsAndWatermarks(new BlPreciseExposureWatermark())
+                .map(new BlPreciseExposureMapFunction()).uid("id_blpreciseexposure_map").setParallelism(parallelism)
+                .filter(new BlPreciseExposureFilterFunction()).uid("id_blpreciseexposure_filter").setParallelism(parallelism)
+//                .assignTimestampsAndWatermarks(new BlPreciseExposureWatermark())
                 .timeWindowAll(Time.of(windowSize, TimeUnit.SECONDS))
-                .allowedLateness(Time.of(windowSize, TimeUnit.SECONDS))
-                .sideOutputLateData(outputTag)
-                .process(new BlPreciseExposureProcessFunction())
-                .uid("id_blpreciseexposure_distinct_window").setParallelism(parallelism);
+//                .allowedLateness(Time.of(windowSize, TimeUnit.SECONDS))
+//                .trigger(EventTimeTrigger.create())
+//                .sideOutputLateData(outputTag)
+                .process(new BlPreciseExposureProcessFunction()).uid("id_blpreciseexposure_distinct_window").setParallelism(parallelism);
         /*
         BL层延迟到达数据
          */
-        DataStream<BlPreciseExposureBean> blPreciseExposureLateStream =  blPreciseExposureStream.getSideOutput(outputTag);
+//        DataStream<BlPreciseExposureBean> blPreciseExposureLateStream =  blPreciseExposureStream.getSideOutput(outputTag);
+        /*
+        BL层数据下发
+         */
+        blPreciseExposureStream
+//                .union(blPreciseExposureLateStream)
+                .addSink(new BlPreciseExposureMysqlSink(sinkJdbcUrl, sinkBlTableName))
+                .uid("id_blpreciseexposure_sink")
+                .setParallelism(parallelism);
 
         /*
         Ml层数据爆炸、聚合
          */
         DataStream<MlPreciseExposureBean> mlPreciseExposureStream = blPreciseExposureStream
-                .union(blPreciseExposureLateStream)
-                .flatMap(new MlpreciseExposureFlatMapFunction())
-                .uid("id_mlpreciseexposure_flatmap").setParallelism(parallelism)
+//                .union(blPreciseExposureLateStream)
+                .flatMap(new MlpreciseExposureFlatMapFunction()).uid("id_mlpreciseexposure_flatmap").setParallelism(parallelism)
                 .keyBy(new MlPreciseExposureKeySelector())
                 .timeWindow(Time.of(windowSize, TimeUnit.SECONDS))
-                .sum("preciseexposure_num")
-                .uid("id_mlpreciseexposure_aggregate_window").setParallelism(parallelism);
+                .sum("preciseexposure_num").uid("id_mlpreciseexposure_aggregate_window").setParallelism(parallelism);
 
         /*
         乱序异步调用关联维度码表
+        timeout：请求最长等待时间；capacity：请求最大并发数
          */
-        // timeout：请求最长等待时间；capacity：请求最大并发数
         DataStream<MlPreciseExposureBean> mlPreciseExposureJoinDimStream = AsyncDataStream
                 .unorderedWait(mlPreciseExposureStream, new DimRichAsyncFunction(dimJdbcUrl), 1, TimeUnit.MINUTES, 1000)
-                .uid("id_mlpreciseexposure_joindim").setParallelism(parallelism);
+                .uid("id_mlpreciseexposure_joindim")
+                .setParallelism(parallelism);
+
+//        //测试打印
 //        mlPreciseExposureJoinDimStream.print();
 //        mlPreciseExposureJoinDimStream.addSink(new PrintSinkFunction<>());
-        mlPreciseExposureJoinDimStream.addSink(new MlPreciseExposureMysqlSink(sinkJdbcUrl, sinkTableName)).uid("mlPreciseExposureSink");
+        /*
+        ML层数据下发
+         */
+        mlPreciseExposureJoinDimStream
+                .addSink(new MlPreciseExposureMysqlSink(sinkJdbcUrl, sinkMlTableName))
+                .uid("id_mlpreciseexposure_sink")
+                .setParallelism(parallelism);
+
         env.execute("ml_c_et_pe_preciseexposure_dimen_d_rt");
     }
 
@@ -187,7 +208,8 @@ public class PreciseExposureStreaming {
                 "    --windowSize <window size(second), default: 30 (s)> \n" +
                 "    --dimJdbcUrl <dim database url> \n" +
                 "    --sinkJdbcUrl <target database url> \n" +
-                "    --sinkTableName <target table name> \n" +
+                "    --sinkBlTableName <target bl table name> \n" +
+                "    --sinkMlTableName <target ml table name> \n" +
                 "    --parallelism <parallelism, default 1> \n" +
                 "    --startTime <kafka startTime, default null> \n" +
                 "    --checkpointPath <checkpointPath, hdfs> \n"
@@ -214,7 +236,8 @@ public class PreciseExposureStreaming {
                 "    --windowSize " + parameterTool.getInt("windowSize", 30) + " \n" +
                 "    --dimJdbcUrl " + parameterTool.getRequired("dimJdbcUrl") + " \n" +
                 "    --sinkJdbcUrl " + parameterTool.getRequired("sinkJdbcUrl") + " \n" +
-                "    --sinkTableName " + parameterTool.getRequired("sinkTableName") + " \n" +
+                "    --sinkBlTableName " + parameterTool.getRequired("sinkBlTableName") + " \n" +
+                "    --sinkMlTableName " + parameterTool.getRequired("sinkMlTableName") + " \n" +
                 "    --parallelism "+ parameterTool.getInt("parallelism", 1) + " \n" +
                 "    --startTime " + parameterTool.get("startTime", null) + " \n" +
                 "    --checkpointPath " + parameterTool.getRequired("checkpointPath") + " \n"

ml_c_et_pe_preciseexposure_dimem_d_rt/src/test/java/com/gmei/function/BlPreciseExposureMapFunctionTest.java

@@ -39,10 +39,12 @@ public class BlPreciseExposureMapFunctionTest {
 
     @Test
     public void testReplace() {
-        String json = "{\"{\n} \t\" }";
+//        String json = "{\"{\n} \t\" }";
+        String json = "{\"create_at\":\"1576854504\",\"gm_nginx_timestamp\":1578502215.724,\"user_id\":\"\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":\"0\",\"exposure_cards\":[{\"absolute_position\":\"0\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"0\",\"card_id\":2},{\"absolute_position\":\"0\",\"target_name\":\"\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"gadget\",\"relative_position\":\"\",\"card_id\":923},{\"absolute_position\":\"1\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"1\",\"card_id\":2},{\"absolute_position\":\"2\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"2\",\"card_id\":2},{\"target_name\":\"鼻部-双旦大促-banner\",\"absolute_position\":\"2\",\"card_content_type \":\"新专题聚合\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"relative_position\":\"\",\"in_page_pos\":\"top\",\"card_id\":46},{\"absolute_position\":\"3\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"3\",\"card_id\":2},{\"absolute_position\":\"4\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"4\",\"card_id\":2}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"is_exposure\":\"1\",\"referrer\":\"\",\"business_id\":\"\",\"referrer_id\":\"\"},\"app_session_id\":\"CCA64F08-C9F1-47FE-87ED-2B2993ADD067\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"report_result\\\" : 0,\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"video_steep\\\" : 0,\\n  \\\"face_scan\\\" : 0\\n}\",\"current_city_id\":\"worldwide\",\"version\":\"7.19.1\",\"user_type\":{\"config_type\":1},\"serial_id\":75,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"EB72DED5-75C3-410E-A759-388261BD232B\",\"sys_version\":\"13.3\",\"lng\":0,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"EB72DED5-75C3-410E-A759-388261BD232B\",\"lat\":0,\"ip\":\"192.168.0.167\",\"idfa\":\"00000000-0000-0000-0000-000000000000\",\"device_type\":\"ios\",\"model\":\"iPhone9,3\"},\"type\":\"page_precise_exposure\"}";
+
         System.out.println(json);
 //        String jsonOnly = json.replace("\n", "").replace("\r","");
-        String jsonAll = json.replaceAll("\\s+", "");
+        String jsonAll = json.replaceAll("\\s+|\\\\n", "");
 //        System.out.println(jsonOnly);
         System.out.println("------------");
         System.out.println(jsonAll);

ml_c_et_pe_preciseexposure_dimem_d_rt/src/test/java/com/gmei/kafka/MyProducter.java

@@ -52,7 +52,9 @@ public class MyProducter {
 //            ProducerConfig config = new ProducerConfig(props);
         //创建生产这对象
         Producer<String, String> producer = new KafkaProducer<String, String>(props);
-        String json = "{\"create_at\":\"1577447031\",\"gm_nginx_timestamp\":1577447032.059,\"user_id\":\"\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":0,\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type \":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type \":\"新专题聚合\",\"card_id\":46},{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type \":\"指定网页\",\"absolute_position\":3},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":4,\"card_content_type \":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type \":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type \":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"齿科-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type \":\"新专题聚合\",\"card_id\":41}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"referrer_id\":\"\",\"referrer\":\"\",\"is_exposure\":\"1\",\"business_id\":\"\"},\"app_session_id\":\"168871fb-1471-43ac-b4da-e07009d88be7\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"{\\\"video_steep\\\": 1, \\\"post_detail\\\": \\\"0\\\", \\\"face_detect_result\\\": \\\"B\\\", \\\"home\\\": \\\"0\\\", \\\"face_scan\\\": 0, \\\"report_result\\\": 0}\",\"current_city_id\":\"\",\"user_type\":{\"config_type\":\"1\"},\"name\":\"gengmei_user\",\"serial_id\":31,\"channel\":\"yiba5202\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_80a2a60d59729bda\",\"sys_version\":\"8.0.0\",\"lng\":\"0.0\",\"is_WiFi\":\"0\",\"lat\":\"0.0\",\"device_id\":\"869712034003715\",\"manufacturer\":\"HUAWEI\",\"net_type\":\"4G\",\"device_type\":\"android\",\"ip\":\"10.138.25.16\",\"model\":\"LDN-AL20\"},\"type\":\"page_precise_exposure\"}";
+        String json = "{\"create_at\":\"1576854504\",\"gm_nginx_timestamp\":1578502215.724,\"user_id\":\"\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":\"0\",\"exposure_cards\":[{\"absolute_position\":\"0\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"0\",\"card_id\":2},{\"absolute_position\":\"0\",\"target_name\":\"\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"gadget\",\"relative_position\":\"\",\"card_id\":923},{\"absolute_position\":\"1\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"1\",\"card_id\":2},{\"absolute_position\":\"2\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"2\",\"card_id\":2},{\"target_name\":\"鼻部-双旦大促-banner\",\"absolute_position\":\"2\",\"card_content_type \":\"新专题聚合\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"relative_position\":\"\",\"in_page_pos\":\"top\",\"card_id\":46},{\"absolute_position\":\"3\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"3\",\"card_id\":2},{\"absolute_position\":\"4\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"4\",\"card_id\":2}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"is_exposure\":\"1\",\"referrer\":\"\",\"business_id\":\"\",\"referrer_id\":\"\"},\"app_session_id\":\"CCA64F08-C9F1-47FE-87ED-2B2993ADD067\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"report_result\\\" : 0,\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"video_steep\\\" : 0,\\n  \\\"face_scan\\\" : 0\\n}\",\"current_city_id\":\"worldwide\",\"version\":\"7.19.1\",\"user_type\":{\"config_type\":1},\"serial_id\":75,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"EB72DED5-75C3-410E-A759-388261BD232B\",\"sys_version\":\"13.3\",\"lng\":0,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"EB72DED5-75C3-410E-A759-388261BD232B\",\"lat\":0,\"ip\":\"192.168.0.167\",\"idfa\":\"00000000-0000-0000-0000-000000000000\",\"device_type\":\"ios\",\"model\":\"iPhone9,3\"},\"type\":\"page_precise_exposure\"}";
+//        String json = "{\"create_at\":\"1578102778\",\"gm_nginx_timestamp\":1578102780.541,\"user_id\":\"32085659\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"business_id\":\"\",\"exposure_cards\":[{\"is_cpc\":1,\"result_status\":\"1\",\"card_id\":\"5754050\",\"absolute_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"relative_position\":0,\"card_content_type\":\"service\",\"cpc_referer\":\"0\"},{\"is_cpc\":1,\"result_status\":\"1\",\"card_id\":\"5760019\",\"absolute_position\":1,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"relative_position\":1,\"card_content_type\":\"service\",\"cpc_referer\":\"0\"},{\"is_cpc\":1,\"result_status\":\"1\",\"card_id\":\"5833115\",\"absolute_position\":2,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"relative_position\":2,\"card_content_type\":\"service\",\"cpc_referer\":\"0\"}],\"up_loading_times\":0,\"referrer_link\":[\"home\",\"search_home\"],\"tab_name\":\"美购\",\"down_slide_times\":0,\"query\":\"眼综合\",\"filter\":\"综合排序\",\"is_exposure\":\"1\",\"referrer\":\"search_result_more\",\"up_slide_times\":0,\"referrer_id\":\"\",\"page_name\":\"search_result_welfare\"},\"app_session_id\":\"dc92c61b-f9bf-4378-8ca1-e6c0d9bee116\",\"app\":{\"version\":\"7.20.1\",\"grey_type\":\"{\\\"video_steep\\\":0,\\\"post_detail\\\":\\\"0\\\",\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"face_scan\\\":0,\\\"report_result\\\":0}\",\"current_city_id\":\"jincheng\",\"user_type\":{\"config_type\":\"1\"},\"name\":\"gengmei_user\",\"serial_id\":25,\"channel\":\"oppo\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_c60514a82b1f4f68\",\"sys_version\":\"7.1.1\",\"lng\":\"112.31146\",\"is_WiFi\":\"0\",\"lat\":\"35.622659\",\"device_id\":\"861271049370690\",\"manufacturer\":\"OPPO\",\"net_type\":\"4G\",\"device_type\":\"android\",\"ip\":\"10.102.127.227\",\"model\":\"OPPOA83\"},\"type\":\"page_precise_exposure\"}";
+//        String json = "{\"create_at\":\"1577447031\",\"gm_nginx_timestamp\":1577447032.059,\"user_id\":\"\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":0,\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type \":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type \":\"新专题聚合\",\"card_id\":46},{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type \":\"指定网页\",\"absolute_position\":3},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":4,\"card_content_type \":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type \":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type \":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"齿科-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type \":\"新专题聚合\",\"card_id\":41}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"referrer_id\":\"\",\"referrer\":\"\",\"is_exposure\":\"1\",\"business_id\":\"\"},\"app_session_id\":\"168871fb-1471-43ac-b4da-e07009d88be7\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"{\\\"video_steep\\\": 1, \\\"post_detail\\\": \\\"0\\\", \\\"face_detect_result\\\": \\\"B\\\", \\\"home\\\": \\\"0\\\", \\\"face_scan\\\": 0, \\\"report_result\\\": 0}\",\"current_city_id\":\"\",\"user_type\":{\"config_type\":\"1\"},\"name\":\"gengmei_user\",\"serial_id\":31,\"channel\":\"yiba5202\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_80a2a60d59729bda\",\"sys_version\":\"8.0.0\",\"lng\":\"0.0\",\"is_WiFi\":\"0\",\"lat\":\"0.0\",\"device_id\":\"869712034003715\",\"manufacturer\":\"HUAWEI\",\"net_type\":\"4G\",\"device_type\":\"android\",\"ip\":\"10.138.25.16\",\"model\":\"LDN-AL20\"},\"type\":\"page_precise_exposure\"}";
 //        String json = "{\"create_at\":\"1577364648\",\"gm_nginx_timestamp\":1577364646.542,\"user_id\":\"\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":0,\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":3,\"card_content_type\":\"新专题聚合\",\"card_id\":46},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type\":\"指定网页\",\"absolute_position\":4},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type\":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type\":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type\":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":3,\"card_content_type\":\"新专题聚合\",\"card_id\":46},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type\":\"指定网页\",\"absolute_position\":4},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type\":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type\":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type\":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"referrer_id\":\"\",\"referrer\":\"\",\"is_exposure\":\"1\",\"business_id\":\"\"},\"app_session_id\":\"6d167db8-147d-4ded-b7a8-0d9f176f91a1\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"unknown\",\"current_city_id\":\"chongqing\",\"user_type\":{},\"name\":\"gengmei_user\",\"serial_id\":1,\"channel\":\"huawei\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_99a0301553dc4f12\",\"sys_version\":\"10\",\"lng\":\"106.56958\",\"is_WiFi\":\"1\",\"lat\":\"29.680635\",\"device_id\":\"864674045613095\",\"manufacturer\":\"HUAWEI\",\"net_type\":\"wifi\",\"device_type\":\"android\",\"ip\":\"192.168.0.100\",\"model\":\"VOG-AL00\"},\"type\":\"page_precise_exposure\"}";
 //        String json = "{\"create_at\":\"1577324083\",\"gm_nginx_timestamp\":1577324083.74,\"user_id\":\"32339790\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":13,\"exposure_cards\":[{\"absolute_position\":\"23\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"0\",\"card_id\":\"689089\"},{\"absolute_position\":\"26\",\"transaction_type\":\"\",\"relative_position\":\"3\",\"card_content_type\":\"search_query\",\"in_page_pos\":\"related_search\",\"card_id\":[\"注射美容\",\"硅胶丰额头\",\"祛颊脂垫\",\"胶原蛋白填充垫下巴\",\"面部不对称改善\",\"膨体丰额头\"]},{\"absolute_position\":\"27\",\"cpc_referer\":1,\"relative_position\":\"1\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"is_cpc\":0,\"card_content_type\":\"diary\",\"card_id\":15804523},{\"absolute_position\":\"28\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"article\",\"relative_position\":\"2\",\"card_id\":\"19514087\"},{\"absolute_position\":\"29\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"3\",\"card_id\":\"696977\"},{\"absolute_position\":\"30\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"4\",\"card_id\":\"686606\"},{\"absolute_position\":\"33\",\"cpc_referer\":1,\"relative_position\":\"4\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"is_cpc\":0,\"card_content_type\":\"diary\",\"card_id\":15965971},{\"absolute_position\":\"34\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"article\",\"relative_position\":\"2\",\"card_id\":\"19515896\"},{\"absolute_position\":\"35\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"3\",\"card_id\":\"700587\"},{\"absolute_position\":\"36\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"4\",\"card_id\":\"700403\"},{\"absolute_position\":\"39\",\"transaction_type\":\"\",\"relative_position\":\"3\",\"card_content_type\":\"search_query\",\"in_page_pos\":\"related_search\",\"card_id\":[\"长下巴矫正\",\"激光溶脂瘦脸\",\"额头\",\"人工骨垫下巴\",\"下巴截骨前移\",\"玻尿酸丰耳垂\"]},{\"absolute_position\":\"40\",\"cpc_referer\":1,\"relative_position\":\"4\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"is_cpc\":0,\"card_content_type\":\"diary\",\"card_id\":15953555}],\"tab_name\":\"综合\",\"down_slide_times\":1,\"query\":\"瘦脸针\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":\"search_home\",\"business_id\":\"\",\"page_name\":\"search_result_more\",\"referrer_id\":\"\"},\"app_session_id\":\"5A18488E-0965-45EE-B35A-BE82266B785B\",\"app\":{\"version\":\"7.19.1\",\"grey_type\":\"{\\\"report_result\\\":0,\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"post_detail\\\":\\\"0\\\",\\\"video_steep\\\":1,\\\"face_scan\\\":0}\",\"current_city_id\":\"qingdao\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":14,\"channel\":\"AppStore\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"F7B0BC42-DCDD-4AE8-A0A1-C205233A94A4\",\"sys_version\":\"12.4\",\"lng\":120.38120711884,\"is_WiFi\":\"0\",\"manufacturer\":\"Apple\",\"device_id\":\"BD96C788-434E-46EB-898C-DF2941882425\",\"device_type\":\"ios\",\"ip\":\"10.171.24.14\",\"idfa\":\"BD96C788-434E-46EB-898C-DF2941882425\",\"lat\":36.08928448005,\"model\":\"iPhone11,6\"},\"type\":\"page_precise_exposure\"}";
 //        String json = "{\"create_at\":\"1577275915\",\"gm_nginx_timestamp\":1577275916.509,\"user_id\":\"32368787\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_exposure\":\"1\",\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"card_type\":\"card\",\"relative_position\":0,\"card_content_type\":\"banner\",\"card_id\":\"\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5785839\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":1,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5785492\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":2,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5742979\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":3,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5750757\"},{\"cpc_referer\":\"1\",\"relative_position\":0,\"transaction_type\":\"\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"answer\",\"card_id\":\"332510\"},{\"cpc_referer\":\"1\",\"relative_position\":1,\"transaction_type\":\"\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"answer\",\"card_id\":\"307094\"},{\"cpc_referer\":\"1\",\"relative_position\":2,\"is_cpc\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"user_post\",\"card_id\":70366}],\"tab_name\":\"综合\",\"down_slide_times\":0,\"query\":\"魔镜\",\"filter\":\"\",\"referrer_id\":\"\",\"referrer\":\"search_home\",\"up_loading_times\":0,\"business_id\":\"\",\"page_name\":\"search_result_more\"},\"app_session_id\":\"7c55ad0f-a0a8-46c8-ad21-bd9f3f12449f\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"{\\\"video_steep\\\":0,\\\"post_detail\\\":\\\"0\\\",\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"face_scan\\\":0,\\\"report_result\\\":0}\",\"current_city_id\":\"xining\",\"user_type\":{\"config_type\":\"1\"},\"name\":\"gengmei_user\",\"serial_id\":32,\"channel\":\"guangdiantongand05\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_2117151825754036\",\"sys_version\":\"9\",\"lng\":\"101.689229\",\"is_WiFi\":\"1\",\"lat\":\"36.944768\",\"device_id\":\"865787042243564\",\"manufacturer\":\"Xiaomi\",\"net_type\":\"wifi\",\"device_type\":\"android\",\"ip\":\"192.168.0.10\",\"model\":\"RedmiNote7Pro\"},\"type\":\"page_precise_exposure\"}";