add module bl_et_pe_preciseexposure_inc_d_bm

bl_et_pe_preciseexposure_inc_d_bm/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>flink_warehouse_rt</artifactId>
+        <groupId>com.gmei.flink</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>bl_et_pe_preciseexposure_inc_d_bm</artifactId>
+    <dependencies>
+        <dependency>
+            <groupId>net.agkn</groupId>
+            <artifactId>hll</artifactId>
+            <version>1.6.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.roaringbitmap</groupId>
+            <artifactId>RoaringBitmap</artifactId>
+            <version>0.7.18</version>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/bean/bl/BlPreciseExposureBean.java

+package com.gmei.bean.bl;
+
+import java.util.ArrayList;
+
+
+/**
+ * ClassName: BlPreciseExposureBean
+ * TableName: Bl_Et_Mg_PreciseExposure_Inc_D
+ * Function:
+ * Reason: BL层精准曝光BEAN类
+ * Date: 2019/12/3 下午5:07
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureBean {
+
+    private String json;
+    private String gm_nginx_timestamp;
+    private String create_timestamp;
+    private String user_id;
+    private String action;
+    private Integer down_loading_times;
+    private Integer down_slide_times;
+    private Integer up_loading_times;
+    private Integer up_slide_times;
+    private String page_code;
+    private String tab_name;
+    private String business_id;
+    private String referrer_code;
+    private String referrer_id;
+    private ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards;
+    private String is_exposure;
+    private String is_popup;
+    private String filter;
+    private String query;
+    private String app_grey_type;
+    private String app_channel;
+    private String app_version;
+    private String app_current_city_id;
+    private String app_code;
+    private String device_os_type;
+    private String device_model;
+    private String device_id;
+    private String device_android_id;
+    private String device_idfv;
+    private String gm_nginx_time_date;
+    private String gm_nginx_time_day;
+    private String create_time_date;
+    private String create_time_day;
+
+    public BlPreciseExposureBean() {
+    }
+
+    public BlPreciseExposureBean(String json, String gm_nginx_timestamp, String create_timestamp, String user_id, String action, Integer down_loading_times, Integer down_slide_times, Integer up_loading_times, Integer up_slide_times, String page_code, String tab_name, String business_id, String referrer_code, String referrer_id, ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards, String is_exposure, String is_popup, String filter, String query, String app_grey_type, String app_channel, String app_version, String app_current_city_id, String app_code, String device_os_type, String device_model, String device_id, String device_android_id, String device_idfv, String gm_nginx_time_date, String gm_nginx_time_day, String create_time_date, String create_time_day) {
+        this.json = json;
+        this.gm_nginx_timestamp = gm_nginx_timestamp;
+        this.create_timestamp = create_timestamp;
+        this.user_id = user_id;
+        this.action = action;
+        this.down_loading_times = down_loading_times;
+        this.down_slide_times = down_slide_times;
+        this.up_loading_times = up_loading_times;
+        this.up_slide_times = up_slide_times;
+        this.page_code = page_code;
+        this.tab_name = tab_name;
+        this.business_id = business_id;
+        this.referrer_code = referrer_code;
+        this.referrer_id = referrer_id;
+        this.exposure_cards = exposure_cards;
+        this.is_exposure = is_exposure;
+        this.is_popup = is_popup;
+        this.filter = filter;
+        this.query = query;
+        this.app_grey_type = app_grey_type;
+        this.app_channel = app_channel;
+        this.app_version = app_version;
+        this.app_current_city_id = app_current_city_id;
+        this.app_code = app_code;
+        this.device_os_type = device_os_type;
+        this.device_model = device_model;
+        this.device_id = device_id;
+        this.device_android_id = device_android_id;
+        this.device_idfv = device_idfv;
+        this.gm_nginx_time_date = gm_nginx_time_date;
+        this.gm_nginx_time_day = gm_nginx_time_day;
+        this.create_time_date = create_time_date;
+        this.create_time_day = create_time_day;
+    }
+
+    public String getJson() {
+        return json;
+    }
+
+    public void setJson(String json) {
+        this.json = json;
+    }
+
+    public String getGm_nginx_timestamp() {
+        return gm_nginx_timestamp;
+    }
+
+    public void setGm_nginx_timestamp(String gm_nginx_timestamp) {
+        this.gm_nginx_timestamp = gm_nginx_timestamp;
+    }
+
+    public String getCreate_timestamp() {
+        return create_timestamp;
+    }
+
+    public void setCreate_timestamp(String create_timestamp) {
+        this.create_timestamp = create_timestamp;
+    }
+
+    public String getUser_id() {
+        return user_id;
+    }
+
+    public void setUser_id(String user_id) {
+        this.user_id = user_id;
+    }
+
+    public String getAction() {
+        return action;
+    }
+
+    public void setAction(String action) {
+        this.action = action;
+    }
+
+    public Integer getDown_loading_times() {
+        return down_loading_times;
+    }
+
+    public void setDown_loading_times(Integer down_loading_times) {
+        this.down_loading_times = down_loading_times;
+    }
+
+    public Integer getDown_slide_times() {
+        return down_slide_times;
+    }
+
+    public void setDown_slide_times(Integer down_slide_times) {
+        this.down_slide_times = down_slide_times;
+    }
+
+    public Integer getUp_loading_times() {
+        return up_loading_times;
+    }
+
+    public void setUp_loading_times(Integer up_loading_times) {
+        this.up_loading_times = up_loading_times;
+    }
+
+    public Integer getUp_slide_times() {
+        return up_slide_times;
+    }
+
+    public void setUp_slide_times(Integer up_slide_times) {
+        this.up_slide_times = up_slide_times;
+    }
+
+    public String getPage_code() {
+        return page_code;
+    }
+
+    public void setPage_code(String page_code) {
+        this.page_code = page_code;
+    }
+
+    public String getTab_name() {
+        return tab_name;
+    }
+
+    public void setTab_name(String tab_name) {
+        this.tab_name = tab_name;
+    }
+
+    public String getBusiness_id() {
+        return business_id;
+    }
+
+    public void setBusiness_id(String business_id) {
+        this.business_id = business_id;
+    }
+
+    public String getReferrer_code() {
+        return referrer_code;
+    }
+
+    public void setReferrer_code(String referrer_code) {
+        this.referrer_code = referrer_code;
+    }
+
+    public String getReferrer_id() {
+        return referrer_id;
+    }
+
+    public void setReferrer_id(String referrer_id) {
+        this.referrer_id = referrer_id;
+    }
+
+    public ArrayList<BlPreciseExposureParamsExposureCardsBean> getExposure_cards() {
+        return exposure_cards;
+    }
+
+    public void setExposure_cards(ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards) {
+        this.exposure_cards = exposure_cards;
+    }
+
+    public String getIs_exposure() {
+        return is_exposure;
+    }
+
+    public void setIs_exposure(String is_exposure) {
+        this.is_exposure = is_exposure;
+    }
+
+    public String getIs_popup() {
+        return is_popup;
+    }
+
+    public void setIs_popup(String is_popup) {
+        this.is_popup = is_popup;
+    }
+
+    public String getFilter() {
+        return filter;
+    }
+
+    public void setFilter(String filter) {
+        this.filter = filter;
+    }
+
+    public String getQuery() {
+        return query;
+    }
+
+    public void setQuery(String query) {
+        this.query = query;
+    }
+
+    public String getApp_grey_type() {
+        return app_grey_type;
+    }
+
+    public void setApp_grey_type(String app_grey_type) {
+        this.app_grey_type = app_grey_type;
+    }
+
+    public String getApp_channel() {
+        return app_channel;
+    }
+
+    public void setApp_channel(String app_channel) {
+        this.app_channel = app_channel;
+    }
+
+    public String getApp_version() {
+        return app_version;
+    }
+
+    public void setApp_version(String app_version) {
+        this.app_version = app_version;
+    }
+
+    public String getApp_current_city_id() {
+        return app_current_city_id;
+    }
+
+    public void setApp_current_city_id(String app_current_city_id) {
+        this.app_current_city_id = app_current_city_id;
+    }
+
+    public String getApp_code() {
+        return app_code;
+    }
+
+    public void setApp_code(String app_code) {
+        this.app_code = app_code;
+    }
+
+    public String getDevice_os_type() {
+        return device_os_type;
+    }
+
+    public void setDevice_os_type(String device_os_type) {
+        this.device_os_type = device_os_type;
+    }
+
+    public String getDevice_model() {
+        return device_model;
+    }
+
+    public void setDevice_model(String device_model) {
+        this.device_model = device_model;
+    }
+
+    public String getDevice_id() {
+        return device_id;
+    }
+
+    public void setDevice_id(String device_id) {
+        this.device_id = device_id;
+    }
+
+    public String getDevice_android_id() {
+        return device_android_id;
+    }
+
+    public void setDevice_android_id(String device_android_id) {
+        this.device_android_id = device_android_id;
+    }
+
+    public String getDevice_idfv() {
+        return device_idfv;
+    }
+
+    public void setDevice_idfv(String device_idfv) {
+        this.device_idfv = device_idfv;
+    }
+
+    public String getGm_nginx_time_date() {
+        return gm_nginx_time_date;
+    }
+
+    public void setGm_nginx_time_date(String gm_nginx_time_date) {
+        this.gm_nginx_time_date = gm_nginx_time_date;
+    }
+
+    public String getGm_nginx_time_day() {
+        return gm_nginx_time_day;
+    }
+
+    public void setGm_nginx_time_day(String gm_nginx_time_day) {
+        this.gm_nginx_time_day = gm_nginx_time_day;
+    }
+
+    public String getCreate_time_date() {
+        return create_time_date;
+    }
+
+    public void setCreate_time_date(String create_time_date) {
+        this.create_time_date = create_time_date;
+    }
+
+    public String getCreate_time_day() {
+        return create_time_day;
+    }
+
+    public void setCreate_time_day(String create_time_day) {
+        this.create_time_day = create_time_day;
+    }
+
+    @Override
+    public String toString() {
+        return "BlPreciseExposureBean{" +
+                "json='" + json + '\'' +
+                ", gm_nginx_timestamp='" + gm_nginx_timestamp + '\'' +
+                ", create_timestamp='" + create_timestamp + '\'' +
+                ", user_id='" + user_id + '\'' +
+                ", action='" + action + '\'' +
+                ", down_loading_times=" + down_loading_times +
+                ", down_slide_times=" + down_slide_times +
+                ", up_loading_times=" + up_loading_times +
+                ", up_slide_times=" + up_slide_times +
+                ", page_code='" + page_code + '\'' +
+                ", tab_name='" + tab_name + '\'' +
+                ", business_id='" + business_id + '\'' +
+                ", referrer_code='" + referrer_code + '\'' +
+                ", referrer_id='" + referrer_id + '\'' +
+                ", exposure_cards=" + exposure_cards +
+                ", is_exposure='" + is_exposure + '\'' +
+                ", is_popup='" + is_popup + '\'' +
+                ", filter='" + filter + '\'' +
+                ", query='" + query + '\'' +
+                ", app_grey_type='" + app_grey_type + '\'' +
+                ", app_channel='" + app_channel + '\'' +
+                ", app_version='" + app_version + '\'' +
+                ", app_current_city_id='" + app_current_city_id + '\'' +
+                ", app_code='" + app_code + '\'' +
+                ", device_os_type='" + device_os_type + '\'' +
+                ", device_model='" + device_model + '\'' +
+                ", device_id='" + device_id + '\'' +
+                ", device_android_id='" + device_android_id + '\'' +
+                ", device_idfv='" + device_idfv + '\'' +
+                ", gm_nginx_time_date='" + gm_nginx_time_date + '\'' +
+                ", gm_nginx_time_day='" + gm_nginx_time_day + '\'' +
+                ", create_time_date='" + create_time_date + '\'' +
+                ", create_time_day='" + create_time_day + '\'' +
+                '}';
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/bean/bl/BlPreciseExposureParamsExposureCardsBean.java

+package com.gmei.bean.bl;
+
+/**
+ * ClassName: BlPreciseExposureBean
+ * TableName: Bl_Et_Mg_PreciseExposure_Inc_D
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2019/12/3 下午5:07
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+/*
+{
+    "gm_nginx_key":2,
+    "version":"110",
+    "params":{
+        "up_slide_times":0,
+        "down_loading_times":0,
+        "exposure_cards":{
+
+        },
+        "up_loading_times":0,
+        "is_exposure":1,
+        "tab_name":"精选",
+        "referrer_id":"",
+        "down_slide_times":0,
+        "referrer":"",
+        "page_name":"home",
+        "business_id":""
+    },
+    "app_session_id":"C3280044-5C8E-459D-AE6C-8E26BBACD6C9",
+    "gm_nginx_timestamp":1546307958.631,
+    "create_at":"1546307958",
+    "app":{
+        "channel":"AppStore",
+        "version":"7.7.35",
+        "serial_id":42,
+        "current_city_id":"worldwide",
+        "name":"gengmei_user",
+        "user_type":{
+
+        }
+    },
+    "device":{
+        "is_WiFi":"0",
+        "device_type":"ios",
+        "device_id":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
+        "lng":0,
+        "lat":0,
+        "ip":"10.156.100.97",
+        "manufacturer":"Apple",
+        "idfa":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
+        "idfv":"352A6D64-17CA-4520-831A-2CE9507631D8"
+    },
+    "user_id":"30864538",
+    "type":"home_choiceness_card_exposure"
+}
+ */
+
+/*
+  get_json_object(result_card,"$.card_id") as card_id, --卡片id
+  get_json_object(result_card,"$.card_content_type") as card_content_type, --卡片内容类型
+  get_json_object(result_card,"$.card_type") as card_type, --卡片类型
+  get_json_object(result_card,"$.transaction_type") as transaction_type, --卡片业务类型
+  get_json_object(result_card,"$.absolute_position") as absolute_position, --绝对位置
+  get_json_object(result_card,"$.relative_position") as relative_position, --相对位置
+  get_json_object(result_card,"$.filter_f") as filter_f,
+  get_json_object(result_card,"$.is_cpc") as is_cpc, --是否为cpc收费卡片
+  get_json_object(result_card,"$.cpc_referer") as cpc_referer, --cpc_referer
+  get_json_object(result_card,"$.in_page_pose ") as in_page_pose,
+  get_json_object(result_card,"$.result_status") as result_status, --结果状态
+  get_json_object(result_card,"$.sec_tab_name") as sec_tab_name, --二级页面名称
+  --非正常卡片通用参数（banner 豆腐块 icon）
+  get_json_object(result_card,"$.module_id") as module_id, --模块id
+  get_json_object(result_card,"$.card_name ") as card_name, --卡片名字
+  get_json_object(result_card,"$.url") as url, --跳转链接
+  --豆腐块额外参数
+  get_json_object(result_card,"$.module_type") as module_type, --模块类型
+  get_json_object(result_card,"$.grid") as grid, --区域 （a、b、c、d，目前线上最多四张图片）
+  --icon的额外参数
+  get_json_object(result_card,"$.total_position") as total_position,--总共有几屏
+  get_json_object(result_card,"$.current_position") as current_position --icon在第几屏
+*/
+public class BlPreciseExposureParamsExposureCardsBean {
+
+
+    private String card_id;
+    private String card_content_type;
+    private String card_type;
+    private String card_name;
+    private String target_name;
+    private String transaction_type;
+    private String is_cpc;
+    private String cpc_referer;
+    private String absolute_position;
+    private String relative_position;
+
+
+    public BlPreciseExposureParamsExposureCardsBean() {
+    }
+
+    public String getCard_type() {
+        return card_type;
+    }
+
+    public void setCard_type(String card_type) {
+        this.card_type = card_type;
+    }
+
+    public String getCard_name() {
+        return card_name;
+    }
+
+    public void setCard_name(String card_name) {
+        this.card_name = card_name;
+    }
+
+    public String getTarget_name() {
+        return target_name;
+    }
+
+    public void setTarget_name(String target_name) {
+        this.target_name = target_name;
+    }
+
+    public String getCpc_referer() {
+        return cpc_referer;
+    }
+
+    public void setCpc_referer(String cpc_referer) {
+        this.cpc_referer = cpc_referer;
+    }
+
+    public String getCard_id() {
+        return card_id;
+    }
+
+    public void setCard_id(String card_id) {
+        this.card_id = card_id;
+    }
+
+    public String getCard_content_type() {
+        return card_content_type;
+    }
+
+    public void setCard_content_type(String card_content_type) {
+        this.card_content_type = card_content_type;
+    }
+
+    public String getIs_cpc() {
+        return is_cpc;
+    }
+
+    public void setIs_cpc(String is_cpc) {
+        this.is_cpc = is_cpc;
+    }
+
+    public String getAbsolute_position() {
+        return absolute_position;
+    }
+
+    public void setAbsolute_position(String absolute_position) {
+        this.absolute_position = absolute_position;
+    }
+
+    public String getTransaction_type() {
+        return transaction_type;
+    }
+
+    public void setTransaction_type(String transaction_type) {
+        this.transaction_type = transaction_type;
+    }
+
+    public String getRelative_position() {
+        return relative_position;
+    }
+
+    public void setRelative_position(String relative_position) {
+        this.relative_position = relative_position;
+    }
+
+    public BlPreciseExposureParamsExposureCardsBean(String card_id, String card_content_type, String card_type, String card_name, String target_name, String transaction_type, String is_cpc, String cpc_referer, String absolute_position, String relative_position) {
+        this.card_id = card_id;
+        this.card_content_type = card_content_type;
+        this.card_type = card_type;
+        this.card_name = card_name;
+        this.target_name = target_name;
+        this.transaction_type = transaction_type;
+        this.is_cpc = is_cpc;
+        this.cpc_referer = cpc_referer;
+        this.absolute_position = absolute_position;
+        this.relative_position = relative_position;
+    }
+
+    @Override
+    public String toString() {
+        return "BlPreciseExposureParamsExposureCardsBean{" +
+                "card_id='" + card_id + '\'' +
+                ", card_content_type='" + card_content_type + '\'' +
+                ", card_type='" + card_type + '\'' +
+                ", card_name='" + card_name + '\'' +
+                ", target_name='" + target_name + '\'' +
+                ", transaction_type='" + transaction_type + '\'' +
+                ", is_cpc='" + is_cpc + '\'' +
+                ", cpc_referer='" + cpc_referer + '\'' +
+                ", absolute_position='" + absolute_position + '\'' +
+                ", relative_position='" + relative_position + '\'' +
+                '}';
+    }
+}
+

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/bean/dim/DimCardContentType.java

+package com.gmei.bean.dim;
+
+/**
+ * ClassName: DimCardContentType
+ * Function:
+ * Reason: 卡片内容类型DIM_CARD_CONTENT_TYPE
+ * Date: 2019/12/6 下午8:20
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class DimCardContentType {
+
+    private String code; //编码
+    private String pk; //主键
+    private String name; //名称
+    private String memo; //说明
+    private String new_code; //转换后编码
+    private Integer oid; //排序
+
+    public DimCardContentType() {
+    }
+
+    public DimCardContentType(String code, String pk, String name, String memo, String new_code, Integer oid) {
+        this.code = code;
+        this.pk = pk;
+        this.name = name;
+        this.memo = memo;
+        this.new_code = new_code;
+        this.oid = oid;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public void setCode(String code) {
+        this.code = code;
+    }
+
+    public String getPk() {
+        return pk;
+    }
+
+    public void setPk(String pk) {
+        this.pk = pk;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getMemo() {
+        return memo;
+    }
+
+    public void setMemo(String memo) {
+        this.memo = memo;
+    }
+
+    public String getNew_code() {
+        return new_code;
+    }
+
+    public void setNew_code(String new_code) {
+        this.new_code = new_code;
+    }
+
+    public Integer getOid() {
+        return oid;
+    }
+
+    public void setOid(Integer oid) {
+        this.oid = oid;
+    }
+
+    @Override
+    public String toString() {
+        return "DimCity{" +
+                "code='" + code + '\'' +
+                ", pk='" + pk + '\'' +
+                ", name='" + name + '\'' +
+                ", memo='" + memo + '\'' +
+                ", new_code='" + new_code + '\'' +
+                ", oid=" + oid +
+                '}';
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/bean/dim/DimCity.java

+package com.gmei.bean.dim;
+
+/**
+ * ClassName: DimCity
+ * Function:
+ * Reason: 城市码表DIM_CITY
+ * Date: 2019/12/6 下午8:20
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class DimCity {
+
+    private String code; //城市编码
+    private String pk; //城市TAGID
+    private String name; //城市名称
+    private String memo; //城市说明
+    private String parent_province_code; //省份编码
+    private String parent_province_pk; //省份TAGID
+    private String parent_province_name; //省份名称
+    private String parent_province_memo; //省份说明
+    private String parent_country_code; //国家编码
+    private String parent_country_pk; //国家TAGID
+    private String parent_country_name; //国家名称
+    private String parent_country_memo; //国家说明
+    private String parent_region_code; //区域编码
+    private String parent_region_pk; //区域ID
+    private String parent_region_name; //区域名称
+    private String parent_region_memo; //区域说明
+    private Integer oid; //排序
+
+    public DimCity() {
+    }
+
+    public DimCity(String code, String pk, String name, String memo, String parent_province_code, String parent_province_pk, String parent_province_name, String parent_province_memo, String parent_country_code, String parent_country_pk, String parent_country_name, String parent_country_memo, String parent_region_code, String parent_region_pk, String parent_region_name, String parent_region_memo, Integer oid) {
+        this.code = code;
+        this.pk = pk;
+        this.name = name;
+        this.memo = memo;
+        this.parent_province_code = parent_province_code;
+        this.parent_province_pk = parent_province_pk;
+        this.parent_province_name = parent_province_name;
+        this.parent_province_memo = parent_province_memo;
+        this.parent_country_code = parent_country_code;
+        this.parent_country_pk = parent_country_pk;
+        this.parent_country_name = parent_country_name;
+        this.parent_country_memo = parent_country_memo;
+        this.parent_region_code = parent_region_code;
+        this.parent_region_pk = parent_region_pk;
+        this.parent_region_name = parent_region_name;
+        this.parent_region_memo = parent_region_memo;
+        this.oid = oid;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public void setCode(String code) {
+        this.code = code;
+    }
+
+    public String getPk() {
+        return pk;
+    }
+
+    public void setPk(String pk) {
+        this.pk = pk;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getMemo() {
+        return memo;
+    }
+
+    public void setMemo(String memo) {
+        this.memo = memo;
+    }
+
+    public String getParent_province_code() {
+        return parent_province_code;
+    }
+
+    public void setParent_province_code(String parent_province_code) {
+        this.parent_province_code = parent_province_code;
+    }
+
+    public String getParent_province_pk() {
+        return parent_province_pk;
+    }
+
+    public void setParent_province_pk(String parent_province_pk) {
+        this.parent_province_pk = parent_province_pk;
+    }
+
+    public String getParent_province_name() {
+        return parent_province_name;
+    }
+
+    public void setParent_province_name(String parent_province_name) {
+        this.parent_province_name = parent_province_name;
+    }
+
+    public String getParent_province_memo() {
+        return parent_province_memo;
+    }
+
+    public void setParent_province_memo(String parent_province_memo) {
+        this.parent_province_memo = parent_province_memo;
+    }
+
+    public String getParent_country_code() {
+        return parent_country_code;
+    }
+
+    public void setParent_country_code(String parent_country_code) {
+        this.parent_country_code = parent_country_code;
+    }
+
+    public String getParent_country_pk() {
+        return parent_country_pk;
+    }
+
+    public void setParent_country_pk(String parent_country_pk) {
+        this.parent_country_pk = parent_country_pk;
+    }
+
+    public String getParent_country_name() {
+        return parent_country_name;
+    }
+
+    public void setParent_country_name(String parent_country_name) {
+        this.parent_country_name = parent_country_name;
+    }
+
+    public String getParent_country_memo() {
+        return parent_country_memo;
+    }
+
+    public void setParent_country_memo(String parent_country_memo) {
+        this.parent_country_memo = parent_country_memo;
+    }
+
+    public String getParent_region_code() {
+        return parent_region_code;
+    }
+
+    public void setParent_region_code(String parent_region_code) {
+        this.parent_region_code = parent_region_code;
+    }
+
+    public String getParent_region_pk() {
+        return parent_region_pk;
+    }
+
+    public String getParent_region_name() {
+        return parent_region_name;
+    }
+
+    public void setParent_region_name(String parent_region_name) {
+        this.parent_region_name = parent_region_name;
+    }
+
+    public String getParent_region_memo() {
+        return parent_region_memo;
+    }
+
+    public void setParent_region_memo(String parent_region_memo) {
+        this.parent_region_memo = parent_region_memo;
+    }
+
+    @Override
+    public String toString() {
+        return "DimCity{" +
+                "code='" + code + '\'' +
+                ", pk='" + pk + '\'' +
+                ", name='" + name + '\'' +
+                ", memo='" + memo + '\'' +
+                ", parent_province_code='" + parent_province_code + '\'' +
+                ", parent_province_pk='" + parent_province_pk + '\'' +
+                ", parent_province_name='" + parent_province_name + '\'' +
+                ", parent_province_memo='" + parent_province_memo + '\'' +
+                ", parent_country_code='" + parent_country_code + '\'' +
+                ", parent_country_pk='" + parent_country_pk + '\'' +
+                ", parent_country_name='" + parent_country_name + '\'' +
+                ", parent_country_memo='" + parent_country_memo + '\'' +
+                ", parent_region_code='" + parent_region_code + '\'' +
+                ", parent_region_pk='" + parent_region_pk + '\'' +
+                ", parent_region_name='" + parent_region_name + '\'' +
+                ", parent_region_memo='" + parent_region_memo + '\'' +
+                ", oid=" + oid +
+                '}';
+    }
+
+    public void setParent_region_pk(String parent_region_pk) {
+        this.parent_region_pk = parent_region_pk;
+    }
+
+    public Integer getOid() {
+        return oid;
+    }
+
+    public void setOid(Integer oid) {
+        this.oid = oid;
+    }
+
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/bean/dim/DimPageType.java

+package com.gmei.bean.dim;
+
+/**
+ * ClassName: DimPageType
+ * Function:
+ * Reason: 页面类型码表DIM_PAGE_TYPE
+ * Date: 2019/12/6 下午8:20
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class DimPageType {
+
+    private String code; //页面编码
+    private String pk; //页面主键
+    private String name; //页面名称
+    private String memo; //页面说明
+    private String parent_app_code; //上级APP编码
+    private String parent_app_pk; //上级APP主键
+    private String parent_app_name; //上级APP名称
+    private String parent_app_memo; //上级APP说明
+    private String type_s_code; //页面分类小类编码
+    private String type_s_name; //页面分类小类名称
+    private String type_m_code; //页面分类中类编码
+    private String type_m_name; //页面分类中类名称
+    private String type_l_code; //页面分类大类编码
+    private String type_l_name; //页面分类大类名称
+    private Integer oid; //排序
+
+    public DimPageType() {
+    }
+
+    public DimPageType(String code, String pk, String name, String memo, String parent_app_code, String parent_app_pk, String parent_app_name, String parent_app_memo, String type_s_code, String type_s_name, String type_m_code, String type_m_name, String type_l_code, String type_l_name, Integer oid) {
+        this.code = code;
+        this.pk = pk;
+        this.name = name;
+        this.memo = memo;
+        this.parent_app_code = parent_app_code;
+        this.parent_app_pk = parent_app_pk;
+        this.parent_app_name = parent_app_name;
+        this.parent_app_memo = parent_app_memo;
+        this.type_s_code = type_s_code;
+        this.type_s_name = type_s_name;
+        this.type_m_code = type_m_code;
+        this.type_m_name = type_m_name;
+        this.type_l_code = type_l_code;
+        this.type_l_name = type_l_name;
+        this.oid = oid;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public void setCode(String code) {
+        this.code = code;
+    }
+
+    public String getPk() {
+        return pk;
+    }
+
+    public void setPk(String pk) {
+        this.pk = pk;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getMemo() {
+        return memo;
+    }
+
+    public void setMemo(String memo) {
+        this.memo = memo;
+    }
+
+    public String getParent_app_code() {
+        return parent_app_code;
+    }
+
+    public void setParent_app_code(String parent_app_code) {
+        this.parent_app_code = parent_app_code;
+    }
+
+    public String getParent_app_pk() {
+        return parent_app_pk;
+    }
+
+    public void setParent_app_pk(String parent_app_pk) {
+        this.parent_app_pk = parent_app_pk;
+    }
+
+    public String getParent_app_name() {
+        return parent_app_name;
+    }
+
+    public void setParent_app_name(String parent_app_name) {
+        this.parent_app_name = parent_app_name;
+    }
+
+    public String getParent_app_memo() {
+        return parent_app_memo;
+    }
+
+    public void setParent_app_memo(String parent_app_memo) {
+        this.parent_app_memo = parent_app_memo;
+    }
+
+    public String getType_s_code() {
+        return type_s_code;
+    }
+
+    public void setType_s_code(String type_s_code) {
+        this.type_s_code = type_s_code;
+    }
+
+    public String getType_s_name() {
+        return type_s_name;
+    }
+
+    public void setType_s_name(String type_s_name) {
+        this.type_s_name = type_s_name;
+    }
+
+    public String getType_m_code() {
+        return type_m_code;
+    }
+
+    public void setType_m_code(String type_m_code) {
+        this.type_m_code = type_m_code;
+    }
+
+    public String getType_m_name() {
+        return type_m_name;
+    }
+
+    public void setType_m_name(String type_m_name) {
+        this.type_m_name = type_m_name;
+    }
+
+    public String getType_l_code() {
+        return type_l_code;
+    }
+
+    public void setType_l_code(String type_l_code) {
+        this.type_l_code = type_l_code;
+    }
+
+    public String getType_l_name() {
+        return type_l_name;
+    }
+
+    public void setType_l_name(String type_l_name) {
+        this.type_l_name = type_l_name;
+    }
+
+    public Integer getOid() {
+        return oid;
+    }
+
+    public void setOid(Integer oid) {
+        this.oid = oid;
+    }
+
+    @Override
+    public String toString() {
+        return "DimPageType{" +
+                "code='" + code + '\'' +
+                ", pk='" + pk + '\'' +
+                ", name='" + name + '\'' +
+                ", memo='" + memo + '\'' +
+                ", parent_app_code='" + parent_app_code + '\'' +
+                ", parent_app_pk='" + parent_app_pk + '\'' +
+                ", parent_app_name='" + parent_app_name + '\'' +
+                ", parent_app_memo='" + parent_app_memo + '\'' +
+                ", type_s_code='" + type_s_code + '\'' +
+                ", type_s_name='" + type_s_name + '\'' +
+                ", type_m_code='" + type_m_code + '\'' +
+                ", type_m_name='" + type_m_name + '\'' +
+                ", type_l_code='" + type_l_code + '\'' +
+                ", type_l_name='" + type_l_name + '\'' +
+                ", oid=" + oid +
+                '}';
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/bean/dim/DimTable.java

+package com.gmei.bean.dim;
+
+/**
+ * ClassName: DimTable
+ * Function:
+ * Reason: 标准码表共用
+ * include DIM_CARD_TYPE、DIM_TRANSACTION_TYPE、DIM_CARD_CONTENT_TYPE
+ * Date: 2019/12/6 下午8:20
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class DimTable {
+
+    private String code; //编码
+    private String pk; //主键
+    private String name; //名称
+    private String memo; //说明
+    private Integer oid; //排序
+
+    public DimTable() {
+    }
+
+    public DimTable(String code, String pk, String name, String memo, Integer oid) {
+        this.code = code;
+        this.pk = pk;
+        this.name = name;
+        this.memo = memo;
+        this.oid = oid;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public void setCode(String code) {
+        this.code = code;
+    }
+
+    public String getPk() {
+        return pk;
+    }
+
+    public void setPk(String pk) {
+        this.pk = pk;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getMemo() {
+        return memo;
+    }
+
+    public void setMemo(String memo) {
+        this.memo = memo;
+    }
+
+    public Integer getOid() {
+        return oid;
+    }
+
+    public void setOid(Integer oid) {
+        this.oid = oid;
+    }
+
+    @Override
+    public String toString() {
+        return "DimCity{" +
+                "code='" + code + '\'' +
+                ", pk='" + pk + '\'' +
+                ", name='" + name + '\'' +
+                ", memo='" + memo + '\'' +
+                ", oid=" + oid +
+                '}';
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/cache/BlPreciseExposureDao.java

+package com.gmei.cache;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.bean.bl.BlPreciseExposureParamsExposureCardsBean;
+import com.gmei.jdbc.MysqlJdbcSink;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ClassName: BlPreciseExposureDao
+ * Function:
+ * Reason: bl_et_mg_preciseexposure_inc_d_rt数据下发操作类
+ * Date: 2019/12/19 上午11:35
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureDao {
+    private MysqlJdbcSink mysqlJdbcSink;
+    private String sql;
+    private String sinkJdbcUrl;
+    private String sinkTableName;
+
+    public BlPreciseExposureDao(String sinkJdbcUrl, String sinkTableName) {
+        this.sinkJdbcUrl = sinkJdbcUrl;
+        this.sinkTableName = sinkTableName;
+        this.mysqlJdbcSink = MysqlJdbcSink.getInstance(sinkJdbcUrl);
+    }
+
+    /**
+    * Function: insertBlPreciseExposure
+    * Reason: 向表中插入数据
+    * Date: 2019/12/25 下午5:38
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public void insertBlPreciseExposure(BlPreciseExposureBean blPreciseExposureBean) throws Exception {
+        sql = "insert into " + sinkTableName + "\n" +
+                "  (json,\n" +
+                "   gm_nginx_timestamp,\n" +
+                "   create_timestamp,\n" +
+                "   user_id,\n" +
+                "   action,\n" +
+                "   down_loading_times,\n" +
+                "   down_slide_times,\n" +
+                "   up_loading_times,\n" +
+                "   up_slide_times,\n" +
+                "   page_code,\n" +
+                "   tab_name,\n" +
+                "   business_id,\n" +
+                "   referrer_code,\n" +
+                "   referrer_id,\n" +
+                "   exposure_cards,\n" +
+                "   is_exposure,\n" +
+                "   is_popup,\n" +
+                "   filter,\n" +
+                "   query,\n" +
+                "   app_grey_type,\n" +
+                "   app_channel,\n" +
+                "   app_version,\n" +
+                "   app_current_city_id,\n" +
+                "   app_code,\n" +
+                "   device_os_type,\n" +
+                "   device_model,\n" +
+                "   device_id,\n" +
+                "   device_android_id,\n" +
+                "   device_idfv,\n" +
+                "   gm_nginx_time_date,\n" +
+                "   gm_nginx_time_day,\n" +
+                "   create_time_date,\n" +
+                "   create_time_day)\n" +
+                "values\n" +
+                "  (?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?,\n" +
+                "   ?)";
+
+        List<Object> params = new ArrayList<Object>();
+        String json = blPreciseExposureBean.getJson();
+//        Clob clob = connection.createClob();
+//        clob.setString(1, json);
+        params.add(json);
+        params.add(blPreciseExposureBean.getGm_nginx_timestamp());
+        params.add(blPreciseExposureBean.getCreate_timestamp());
+        params.add(blPreciseExposureBean.getUser_id());
+        params.add(blPreciseExposureBean.getAction());
+        params.add(blPreciseExposureBean.getDown_loading_times());
+        params.add(blPreciseExposureBean.getDown_slide_times());
+        params.add(blPreciseExposureBean.getUp_loading_times());
+        params.add(blPreciseExposureBean.getUp_slide_times());
+        params.add(blPreciseExposureBean.getPage_code());
+        params.add(blPreciseExposureBean.getTab_name());
+        params.add(blPreciseExposureBean.getBusiness_id());
+        params.add(blPreciseExposureBean.getReferrer_code());
+        params.add(blPreciseExposureBean.getReferrer_id());
+        ArrayList<BlPreciseExposureParamsExposureCardsBean> blPreciseExposureCardsBeans= blPreciseExposureBean.getExposure_cards();
+        JSONArray jsonArrayCardsBeans= JSONArray.parseArray(JSON.toJSONString(blPreciseExposureCardsBeans));
+        params.add(jsonArrayCardsBeans.toString());
+        params.add(blPreciseExposureBean.getIs_exposure());
+        params.add(blPreciseExposureBean.getIs_popup());
+        params.add(blPreciseExposureBean.getFilter());
+        params.add(blPreciseExposureBean.getQuery());
+        params.add(blPreciseExposureBean.getApp_grey_type());
+        params.add(blPreciseExposureBean.getApp_channel());
+        params.add(blPreciseExposureBean.getApp_version());
+        params.add(blPreciseExposureBean.getApp_current_city_id());
+        params.add(blPreciseExposureBean.getApp_code());
+        params.add(blPreciseExposureBean.getDevice_os_type());
+        params.add(blPreciseExposureBean.getDevice_model());
+        params.add(blPreciseExposureBean.getDevice_id());
+        params.add(blPreciseExposureBean.getDevice_android_id());
+        params.add(blPreciseExposureBean.getDevice_idfv());
+        params.add(blPreciseExposureBean.getGm_nginx_time_date());
+        params.add(blPreciseExposureBean.getGm_nginx_time_day());
+        params.add(blPreciseExposureBean.getCreate_time_date());
+        params.add(blPreciseExposureBean.getCreate_time_day());
+//        System.out.println(params.toString());
+        mysqlJdbcSink.update(sql, params);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/cache/PreciseAccumulator.java

+package com.gmei.cache;
+
+import org.roaringbitmap.longlong.Roaring64NavigableMap;
+
+/**
+ * ClassName: PreciseAccumulator
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/14 下午8:24
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+
+public class PreciseAccumulator {
+    private Roaring64NavigableMap bitmap;
+
+    public Roaring64NavigableMap getBitmap() {
+        return bitmap;
+    }
+
+    public void setBitmap(Roaring64NavigableMap bitmap) {
+        this.bitmap = bitmap;
+    }
+
+    public PreciseAccumulator(){
+        bitmap=new Roaring64NavigableMap();
+    }
+
+    public PreciseAccumulator(long defaultValue) {
+        bitmap=new Roaring64NavigableMap();
+        bitmap.addLong(defaultValue);
+    }
+
+    public void add(long id){
+        bitmap.addLong(id);
+    }
+
+    public long getCardinality(){
+        return bitmap.getLongCardinality();
+    }
+
+    public boolean contains(long value) {
+        return bitmap.contains(value);
+    }
+
+    public void clear() {
+        bitmap.clear();
+    }
+
+    @Override
+    public String toString() {
+        return bitmap.toString();
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BitMapFunction.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.cache.PreciseAccumulator;
+import com.gmei.utils.DateUtil;
+import com.gmei.utils.MurmurHash;
+import org.apache.flink.api.common.state.StateTtlConfig;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.util.Collector;
+import org.roaringbitmap.longlong.Roaring64NavigableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class BitMapFunction extends KeyedProcessFunction<String, BlPreciseExposureBean,BlPreciseExposureBean> {
+    private static final long serialVersionUID = 1L;
+    private static final Logger LOGGER = LoggerFactory.getLogger(BitMapFunction.class);
+    private volatile PreciseAccumulator bitMap;
+
+    @Override
+    public void onTimer(long timestamp, OnTimerContext ctx, Collector<BlPreciseExposureBean> out) throws Exception {
+        super.onTimer(timestamp, ctx, out);
+        bitMap.clear();
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        bitMap = new PreciseAccumulator();
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        bitMap.clear();
+    }
+
+    @Override
+    public void processElement(BlPreciseExposureBean blPreciseExposureBean, Context context, Collector<BlPreciseExposureBean> collector) throws Exception {
+        String blPreciseExposureBeanId = blPreciseExposureBean.getJson();
+        long hashCode = MurmurHash.hash64(blPreciseExposureBeanId);
+        if (!bitMap.contains(hashCode)) {
+            bitMap.add(hashCode);
+            collector.collect(blPreciseExposureBean);
+        } else {
+            System.out.println(blPreciseExposureBean);
+        }
+        //凌晨0点0分0秒
+        context.timerService().registerProcessingTimeTimer(DateUtil.tomorrowZeroTimestampMs(Double.valueOf(blPreciseExposureBean.getGm_nginx_timestamp()).longValue() * 1000, 8) + 1);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BlMaiDianOutputSelector.java

+package com.gmei.function;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.streaming.api.collector.selector.OutputSelector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ClassName: BlMaiDianOutputSelector
+ * Function:
+ * Reason: BL层数据选择器
+ * Date: 2019/12/7 上午11:01
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlMaiDianOutputSelector implements OutputSelector<String> {
+    @Override
+    public Iterable<String> select(String s) {
+        List<String> output = new ArrayList<String>();
+        JSONObject jsonObject = JSON.parseObject(s);
+        String action = jsonObject.getString("type");
+//        System.out.println(s);
+        if ("home_choiceness_card_exposure".equals(action) || "page_precise_exposure".equals(action)) {
+            output.add("et_pe");
+        } else if ("page_view".equals(action)) {
+            output.add("et_le");
+        } else if (StringUtils.containsIgnoreCase(action,"click")) {
+            output.add("et_ck");
+        } else {
+            output.add("et_ot");
+        }
+        return output;
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BlPreciseExposureFilterFunction.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import org.apache.flink.api.common.functions.FilterFunction;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * ClassName: BlPreciseExposureFilterFunction
+ * Function:
+ * Reason: BL层数据筛选器
+ * Date: 2019/12/7 上午11:01
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureFilterFunction implements FilterFunction<BlPreciseExposureBean> {
+    @Override
+    public boolean filter(BlPreciseExposureBean blPreciseExposureBean) throws Exception {
+        String app_code = blPreciseExposureBean.getApp_code();
+        String is_exposure = blPreciseExposureBean.getIs_exposure();
+        String is_popup = blPreciseExposureBean.getIs_popup();
+        String app_version = blPreciseExposureBean.getApp_version();
+        String page_code = blPreciseExposureBean.getPage_code();
+        Boolean flag = true;
+        //清洗掉app_code不等于gengmei_user与is_exposure不等于1的
+        //1.5 Home页精准曝光触发两次
+        //问题描述：在7.7.70','7.7.71','7.7.72','7.7.75','7.7.76','7.8.0','7.8.1' (7.7.70--7.9.0)版本中的首页精准曝光会触发两次，经确认，需要去除page_name='home'
+        //1.6 品类聚合页的精准曝光问题
+        //问题描述：品类聚合页的精准曝光没做，但是数据中却有精准曝光、而且有问题，因此过滤掉，目前还不能确定什么版本可以解决，先暂时根据page_name过滤掉page_name='category'
+        //1.15 当首页精准曝光是通过弹窗触发或者点击push的时候，会有额外的精准曝光数据，需要去掉这部分数据
+        List<String> app_version_list = Arrays.asList("7.7.70","7.7.71","7.7.72","7.7.75","7.7.76","7.8.0","7.8.1");
+        String[] version = app_version.split("\\.");
+        if(version.length != 3) {
+            flag = false;
+        } else if(!"gengmei_user".equals(app_code) || !"1".equals(is_exposure)) {
+            flag = false;
+        } else if(app_version_list.contains(app_version) && "home".equals(page_code)) {
+            flag = false;
+        } else if("category".equals(page_code) && "7".equals(version[0]) && Integer.parseInt(version[1]) < 14 ) {
+            flag = false;
+        } else if("home".equals(page_code) && "1".equals(is_popup)) {
+            flag = false;
+        }
+        return flag;
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BlPreciseExposureKeySelector.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import org.apache.flink.api.java.functions.KeySelector;
+
+
+/**
+ * ClassName: BlPreciseExposureKeySelector
+ * Function:
+ * Reason: BL层数据聚合分组器
+ * Date: 2019/12/7 上午11:01
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureKeySelector implements KeySelector<BlPreciseExposureBean, String> {
+    @Override
+    public String getKey(BlPreciseExposureBean blPreciseExposureBean) throws Exception {
+        return blPreciseExposureBean.getGm_nginx_time_day();
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BlPreciseExposureMapFunction.java

+package com.gmei.function;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.bean.bl.BlPreciseExposureParamsExposureCardsBean;
+import com.gmei.utils.BeanReflectUtil;
+import com.gmei.utils.DateUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.common.functions.MapFunction;
+
+import java.util.ArrayList;
+
+
+/**
+ * ClassName: BlPreciseExposureMapFunction
+ * Function:
+ * Reason: BL层数据清洗转换器（一对一）
+ * Date: 2019/12/7 上午11:01
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreciseExposureBean> {
+    @Override
+    public BlPreciseExposureBean map(String string) throws Exception {
+        BlPreciseExposureBean blPreciseExposureBean = new BlPreciseExposureBean();
+        JSONObject app = null;
+        String gm_nginx_timestamp = null;
+        String create_timestamp = null;
+        String user_id = null;
+        String action = null;
+
+        String app_grey_type = null;
+        String app_channel = null;
+        String app_version = null;
+        String app_current_city_id = null;
+        String app_code = null;
+
+        JSONObject params = null;
+        Integer down_loading_times = null;
+        Integer down_slide_times = null;
+        Integer up_loading_times = null;
+        Integer up_slide_times = null;
+        String page_code = null;
+        String tab_name = null;
+        String business_id = null;
+        String referrer_code = null;
+        String referrer_id = null;
+        String is_exposure = null;
+        String is_popup = null;
+        String filter = null;
+        String query = null;
+        JSONArray jsonArrayExposureCards = null;
+
+        JSONObject device = null;
+        String device_os_type = null;
+        String device_model = null;
+        String device_id = null;
+        String device_idfv = null;
+        String device_android_id = null;
+
+        String gm_nginx_time_date = null;
+        String gm_nginx_time_day = null;
+        String create_time_date = null;
+        String create_time_day = null;
+
+        //去除掉json中的回车、换行、制表、空格
+        String jsonString = string.replaceAll("\\s+|\\\\n", "");
+
+        JSONObject jsonObject = JSON.parseObject(jsonString);
+
+        //一级别JSON
+        app = jsonObject.getJSONObject("app");
+        gm_nginx_timestamp = String.valueOf(jsonObject.get("gm_nginx_timestamp"));
+        create_timestamp = String.valueOf(jsonObject.get("create_at"));
+        user_id = jsonObject.getString("user_id");
+        action = jsonObject.getString("type");
+
+        //二级别JSON
+        //APP
+        if (app != null) {
+            app_grey_type = app.getString("grey_type");
+            app_channel = app.getString("channel");
+            app_version = app.getString("version");
+            app_current_city_id = app.getString("current_city_id");
+            app_code = app.getString("name");
+        }
+        //1.2 absolute_position参数类型不完整
+        //问题描述：在'7.7.35', '7.7.36', '7.7.40'params的absolute_position缺少左边引号
+        String paramsString = String.valueOf(jsonObject.get("params"));
+        if("7.7.35".equals(app_version) || "7.7.36".equals(app_version) || "7.7.40".equals(app_version)) {
+            paramsString = paramsString.replace("absolute_position","\\\"absolute_position");
+        }
+
+        //PARAMS
+        params =  JSON.parseObject(paramsString);
+        if (params != null) {
+            down_loading_times = params.getInteger("down_loading_times");
+            down_slide_times = params.getInteger("down_slide_times");
+            up_loading_times = params.getInteger("up_loading_times");
+            up_slide_times = params.getInteger("up_slide_times");
+            page_code = params.getString("page_name");
+            tab_name = params.getString("tab_name");
+            business_id = params.getString("business_id");
+            referrer_code = params.getString("referrer");
+            referrer_id = params.getString("referrer_id");
+            is_exposure = params.getString("is_exposure");
+            is_popup = params.getString("is_popup");
+            filter = params.getString("filter");
+            query = params.getString("query");
+        }
+        //1.4 初版精准曝光格式错误
+        //问题描述：在'7.7.60'版本中美购页卡片的card_id和transaction_type这两个参数是数组的形式
+//        ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards =(ArrayList<BlPreciseExposureParamsExposureCardsBean>)JSONArray.parseArray(params.getJSONArray("exposure_cards").toString(), BlPreciseExposureParamsExposureCardsBean.class);
+        ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards = new ArrayList<>();
+        try {
+            jsonArrayExposureCards = JSONArray.parseArray(params.getJSONArray("exposure_cards").toString());
+            for (int i = 0; i < jsonArrayExposureCards.size(); i++) {
+                JSONObject jsonCard = jsonArrayExposureCards.getJSONObject(i);
+                String card_id = jsonCard.getString("card_id");
+                String transaction_type = jsonCard.getString("transaction_type");
+                String card_content_type = jsonCard.getString("card_content_type");
+                String card_type = jsonCard.getString("card_type");
+                String card_name = jsonCard.getString("card_name");
+                String target_name = jsonCard.getString("target_name");
+                String is_cpc = jsonCard.getString("is_cpc");
+                String cpc_referer = jsonCard.getString("cpc_referer");
+                String absolute_posttion = jsonCard.getString("absolute_position");
+                String relative_position = jsonCard.getString("relative_position");
+//                if("service".equals(card_content_type) && "7.7.60".equals(app_version)) {
+                if(StringUtils.startsWithIgnoreCase(card_id, "[") && StringUtils.endsWithIgnoreCase(card_id, "]")) {
+                    JSONArray card_id_list = JSONArray.parseArray(card_id);
+                    JSONArray transaction_type_list = null;
+                    if(StringUtils.startsWithIgnoreCase(transaction_type, "[") && StringUtils.endsWithIgnoreCase(transaction_type, "]")) {
+                        transaction_type_list = JSONArray.parseArray(transaction_type);
+                    }
+                    for (int j = 0; j < card_id_list.size(); j++) {
+                        BlPreciseExposureParamsExposureCardsBean cardsBean = new BlPreciseExposureParamsExposureCardsBean();
+                        cardsBean.setCard_id(card_id_list.getString(j));
+                        try {
+                            cardsBean.setTransaction_type(transaction_type_list.getString(j));
+                        } catch (Exception e) {
+                            cardsBean.setTransaction_type(null);
+                        }
+                        cardsBean.setCard_content_type(card_content_type);
+                        cardsBean.setCard_type(card_type);
+                        cardsBean.setCard_name(card_name);
+                        cardsBean.setTarget_name(target_name);
+                        cardsBean.setIs_cpc(is_cpc);
+                        cardsBean.setCpc_referer(cpc_referer);
+                        cardsBean.setAbsolute_position(absolute_posttion);
+                        cardsBean.setRelative_position(relative_position);
+                        exposure_cards.add(cardsBean);
+                    }
+                } else {
+                    BlPreciseExposureParamsExposureCardsBean cardsBean = new BlPreciseExposureParamsExposureCardsBean();
+                    cardsBean.setCard_id(card_id);
+                    cardsBean.setTransaction_type(transaction_type);
+                    cardsBean.setCard_content_type(card_content_type);
+                    cardsBean.setCard_type(card_type);
+                    cardsBean.setCard_name(card_name);
+                    cardsBean.setTarget_name(target_name);
+                    cardsBean.setIs_cpc(is_cpc);
+                    cardsBean.setCpc_referer(cpc_referer);
+                    cardsBean.setAbsolute_position(absolute_posttion);
+                    cardsBean.setRelative_position(relative_position);
+//                    BlPreciseExposureParamsExposureCardsBean cardsBean = JSON.toJavaObject(jsonCard, BlPreciseExposureParamsExposureCardsBean.class);
+                    exposure_cards.add(cardsBean);
+                }
+            }
+        } catch (Exception e) {
+//            System.out.println(JSON.parseObject(params.getString("exposure_cards")).toString());
+        }
+
+        //DEVICE
+        device = jsonObject.getJSONObject("device");
+        if (device != null) {
+            device_os_type = device.getString("device_type");
+            device_model = device.getString("model");
+            device_id = device.getString("device_id");
+            device_idfv = device.getString("idfv");
+            device_android_id = device.getString("android_device_id");
+        }
+
+        //1.1 ios系统精准曝光埋点channel字段
+        //问题描述：ios系统精准曝光埋点channel字段AppStore应该修改为App Store
+        if("AppStore".equals(app_channel)) {
+            app_channel = "App Store";
+        }
+
+        //日志时间处理
+        gm_nginx_time_date = DateUtil.timestampFormat(gm_nginx_timestamp, DateUtil.DateType.SECOND);
+        gm_nginx_time_day = DateUtil.timestampFormat(gm_nginx_timestamp, DateUtil.DateType.DAY);
+        create_time_date = DateUtil.timestampFormat(create_timestamp, DateUtil.DateType.SECOND);
+        create_time_day = DateUtil.timestampFormat(create_timestamp, DateUtil.DateType.DAY);
+
+        //1.7 result_status参数位置问题(未处理)
+        //问题描述：搜索结果百科（search_result_wiki)和搜索结果美购（search_result_welfare）的result_status埋点埋在exposure_data里面了为了都获得该参数值，因此需要从两处拿取，默认先取exposure_data里的result_satus字段值（数据中发现不只是这两个页面）search_result_diary、search_result_doctor、search_result_hospital也存在
+
+        //1.9 搜索结果页tab_name参数不该存在
+        //问题描述:在搜索结果页出现tab_name参数，参数值和page_name一样，在点击的时候页面有跳转，因此，不应该有tab_name参数
+        if (StringUtils.containsIgnoreCase(page_code, "search_result")) {
+            tab_name = null;
+        }
+
+        blPreciseExposureBean.setJson(jsonString);
+        blPreciseExposureBean.setGm_nginx_timestamp(gm_nginx_timestamp);
+        blPreciseExposureBean.setCreate_timestamp(create_timestamp);
+        blPreciseExposureBean.setUser_id(user_id);
+        blPreciseExposureBean.setAction(action);
+        blPreciseExposureBean.setApp_grey_type(app_grey_type);
+        blPreciseExposureBean.setApp_channel(app_channel);
+        blPreciseExposureBean.setApp_version(app_version);
+        blPreciseExposureBean.setApp_current_city_id(app_current_city_id);
+        blPreciseExposureBean.setApp_code(app_code);
+        blPreciseExposureBean.setDown_loading_times(down_loading_times);
+        blPreciseExposureBean.setDown_slide_times(down_slide_times);
+        blPreciseExposureBean.setUp_loading_times(up_loading_times);
+        blPreciseExposureBean.setUp_slide_times(up_slide_times);
+        blPreciseExposureBean.setPage_code(page_code);
+        blPreciseExposureBean.setTab_name(tab_name);
+        blPreciseExposureBean.setBusiness_id(business_id);
+        blPreciseExposureBean.setReferrer_code(referrer_code);
+        blPreciseExposureBean.setReferrer_id(referrer_id);
+        blPreciseExposureBean.setExposure_cards(exposure_cards);
+        blPreciseExposureBean.setIs_exposure(is_exposure);
+        blPreciseExposureBean.setIs_popup(is_popup);
+        blPreciseExposureBean.setFilter(filter);
+        blPreciseExposureBean.setQuery(query);
+        blPreciseExposureBean.setDevice_os_type(device_os_type);
+        blPreciseExposureBean.setDevice_model(device_model);
+        blPreciseExposureBean.setDevice_id(device_id != null && device_id !="" ? device_id : device_idfv);
+        blPreciseExposureBean.setDevice_android_id(device_android_id);
+        blPreciseExposureBean.setDevice_idfv(device_idfv);
+        blPreciseExposureBean.setGm_nginx_time_date(gm_nginx_time_date);
+        blPreciseExposureBean.setGm_nginx_time_day(gm_nginx_time_day);
+        blPreciseExposureBean.setCreate_time_date(create_time_date);
+        blPreciseExposureBean.setCreate_time_day(create_time_day);
+        return BeanReflectUtil.setNullValue(blPreciseExposureBean);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BlPreciseExposureProcessFunction.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction;
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.apache.flink.util.Collector;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * ClassName: BlPreciseExposureProcessFunction
+ * Function:
+ * Reason: BL层数据去重器
+ * Date: 2020/1/8 下午5:06
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureProcessFunction extends ProcessAllWindowFunction<BlPreciseExposureBean, BlPreciseExposureBean, TimeWindow> {
+    @Override
+    public void process(Context context, Iterable<BlPreciseExposureBean> iterable, Collector<BlPreciseExposureBean> collector) throws Exception {
+        Iterator<BlPreciseExposureBean> blPreciseExposureBeanIterator = iterable.iterator();
+        Map<String, BlPreciseExposureBean> blPreciseExposureBeanMap = new HashMap<>();
+        while (blPreciseExposureBeanIterator.hasNext()) {
+            BlPreciseExposureBean blPreciseExposureBean = blPreciseExposureBeanIterator.next();
+            blPreciseExposureBeanMap.put(blPreciseExposureBean.getJson(), blPreciseExposureBean);
+        }
+
+        Set<Map.Entry<String, BlPreciseExposureBean>> blPreciseExposureBeanSet = blPreciseExposureBeanMap.entrySet();
+        for(Map.Entry<String, BlPreciseExposureBean> blPreciseExposureBeanEntry : blPreciseExposureBeanSet) {
+            collector.collect(blPreciseExposureBeanEntry.getValue());
+        }
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BlPreciseExposureWatermark.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
+import org.apache.flink.streaming.api.watermark.Watermark;
+
+import javax.annotation.Nullable;
+
+/**
+ * ClassName: BlPreciseExposureWatermark
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/8 下午8:40
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureWatermark implements AssignerWithPeriodicWatermarks<BlPreciseExposureBean> {
+    private final long maxOutOfOrderness = 10000;
+    private long currentMaxTimestamp;
+    @Override
+    public long extractTimestamp(BlPreciseExposureBean blPreciseExposureBean, long l) {
+//        Double timestampDouble = Double.parseDouble(blPreciseExposureBean.getGm_nginx_timestamp());
+//        long timestamp = new Double(timestampDouble * 1000).longValue();
+        Double timestampDouble = Double.parseDouble(blPreciseExposureBean.getGm_nginx_timestamp()) * 1000;
+        long timestamp = timestampDouble.longValue();
+        currentMaxTimestamp = Math.max(timestamp, currentMaxTimestamp);
+        return timestamp;
+    }
+
+    @Nullable
+    @Override
+    public Watermark getCurrentWatermark() {
+        return new Watermark(currentMaxTimestamp - maxOutOfOrderness);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/BloomFilterFunction.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.utils.DateUtil;
+import com.google.common.hash.BloomFilter;
+import com.google.common.hash.Funnels;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.Charset;
+
+public class BloomFilterFunction extends KeyedProcessFunction<String, BlPreciseExposureBean,BlPreciseExposureBean> {
+    private static final long serialVersionUID = 1L;
+    private static final Logger LOGGER = LoggerFactory.getLogger(BloomFilterFunction.class);
+    private static final int BF_CARDINAL_THRESHOLD = 3000000;
+    private static final double BF_FALSE_POSITIVE_RATE = 0.01;
+    private volatile BloomFilter<String> blPreciseExposureBF;
+
+    @Override
+    public void onTimer(long timestamp, OnTimerContext ctx, Collector<BlPreciseExposureBean> out) throws Exception {
+        super.onTimer(timestamp, ctx, out);
+//        long s = System.currentTimeMillis();
+        blPreciseExposureBF = BloomFilter.create(Funnels.stringFunnel(Charset.forName("utf-8")), BF_CARDINAL_THRESHOLD, BF_FALSE_POSITIVE_RATE);
+//        long e = System.currentTimeMillis();
+//        LOGGER.info("Timer triggered & resetted BloomFilter, time cost: " + (e - s));
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+//        long s = System.currentTimeMillis();
+        blPreciseExposureBF = BloomFilter.create(Funnels.stringFunnel(Charset.forName("utf-8")), BF_CARDINAL_THRESHOLD, BF_FALSE_POSITIVE_RATE);
+//        long e = System.currentTimeMillis();
+//        LOGGER.info("Created Guava BloomFilter, time cost: " + (e - s));
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        blPreciseExposureBF = null;
+    }
+
+    @Override
+    public void processElement(BlPreciseExposureBean blPreciseExposureBean, Context context, Collector<BlPreciseExposureBean> collector) throws Exception {
+        String blPreciseExposureBeanId = blPreciseExposureBean.getJson();
+        if (!blPreciseExposureBF.mightContain(blPreciseExposureBeanId)) {
+            blPreciseExposureBF.put(blPreciseExposureBeanId);
+            collector.collect(blPreciseExposureBean);
+        }
+        context.timerService().registerProcessingTimeTimer(DateUtil.tomorrowZeroTimestampMs(Double.valueOf(blPreciseExposureBean.getGm_nginx_timestamp()).longValue() * 1000, 8) + 1);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/HyperLogLogFunction.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.utils.DateUtil;
+import com.gmei.utils.HyperLogLog;
+import net.agkn.hll.HLL;
+import org.apache.flink.api.common.state.StateTtlConfig;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * ClassName: HyperLogLogFunction
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/13 下午7:16
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class HyperLogLogFunction extends KeyedProcessFunction<String, BlPreciseExposureBean, BlPreciseExposureBean> {
+    private static final long serialVersionUID = 1L;
+    private static final Logger LOGGER = LoggerFactory.getLogger(HyperLogLogFunction.class);
+    private static final int HLL_CARDINAL_THRESHOLD = 3000000;
+    private static final double HLL_FALSE_POSITIVE_RATE = 0.01; //误差
+    private volatile HyperLogLog blPreciseExposureBeanHLL;
+    private ValueState<Long> hllState;
+    private ValueStateDescriptor<Long> hllStateDescriptor;
+
+    @Override
+    public void onTimer(long timestamp, OnTimerContext ctx, Collector<BlPreciseExposureBean> out) throws Exception {
+        super.onTimer(timestamp, ctx, out);
+        hllState.clear();
+//        long s = System.currentTimeMillis();
+        blPreciseExposureBeanHLL = new HyperLogLog(HLL_FALSE_POSITIVE_RATE);
+//        long e = System.currentTimeMillis();
+//        LOGGER.info("Timer triggered & resetted HyperLogLog, time cost: " + (e - s));
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        //设置ValueState的TTL的生命周期为24个小时，自动会清除ValueState的里内容
+        StateTtlConfig ttlConfig = StateTtlConfig
+                .newBuilder(Time.days(1))
+                .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
+                .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
+                .build();
+        blPreciseExposureBeanHLL = new HyperLogLog(HLL_FALSE_POSITIVE_RATE);
+        hllStateDescriptor = new ValueStateDescriptor("hll", Types.OBJECT_ARRAY(Types.LONG));
+        hllStateDescriptor.enableTimeToLive(ttlConfig);
+        hllState = getRuntimeContext().getState(hllStateDescriptor);
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        blPreciseExposureBeanHLL = null;
+    }
+
+    @Override
+    public void processElement(BlPreciseExposureBean blPreciseExposureBean, Context context, Collector<BlPreciseExposureBean> collector) throws Exception {
+        blPreciseExposureBeanHLL.offer(blPreciseExposureBean.getJson());
+        Long countDistinct = blPreciseExposureBeanHLL.cardinality();
+        Long hllStateValue = this.hllState.value();
+        if(!countDistinct.equals(this.hllState.value())) {
+            hllState.update(blPreciseExposureBeanHLL.cardinality());
+            collector.collect(blPreciseExposureBean);
+        }
+        context.timerService().registerProcessingTimeTimer(DateUtil.tomorrowZeroTimestampMs(Double.valueOf(blPreciseExposureBean.getGm_nginx_timestamp()).longValue() * 1000, 8) + 1);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/function/HyperLogLogFunction2.java

+package com.gmei.function;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.utils.DateUtil;
+import net.agkn.hll.HLL;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.util.hashing.MurmurHash3;
+
+/**
+ * ClassName: HyperLogLogFunction
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/13 下午7:16
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class HyperLogLogFunction2 extends KeyedProcessFunction<String, BlPreciseExposureBean, BlPreciseExposureBean> {
+    private static final long serialVersionUID = 1L;
+    private static final Logger LOGGER = LoggerFactory.getLogger(HyperLogLogFunction2.class);
+    private static final int HLL_CARDINAL_THRESHOLD = 3000000;
+    private static final double HLL_FALSE_POSITIVE_RATE = 0.01; //误差
+//    private volatile HyperLogLog blPreciseExposureBeanHLL;
+    private ValueState<Byte[]> hllState;
+    private volatile HLL blPreciseExposureBeanHLL;
+    private ValueStateDescriptor<Byte[]> hllStateDescriptor;
+
+    @Override
+    public void onTimer(long timestamp, OnTimerContext ctx, Collector<BlPreciseExposureBean> out) throws Exception {
+        super.onTimer(timestamp, ctx, out);
+        hllState.clear();
+        long s = System.currentTimeMillis();
+//        blPreciseExposureBeanHLL = new HyperLogLog(HLL_FALSE_POSITIVE_RATE);
+        long e = System.currentTimeMillis();
+        LOGGER.info("Timer triggered & resetted HyperLogLog, time cost: " + (e - s));
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+//        blPreciseExposureBeanHLL = new HyperLogLog(HLL_FALSE_POSITIVE_RATE);
+        blPreciseExposureBeanHLL = new HLL(14, 5);
+        hllStateDescriptor = new ValueStateDescriptor("hll", Types.OBJECT_ARRAY(Types.BYTE));
+        hllState = getRuntimeContext().getState(hllStateDescriptor);
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        blPreciseExposureBeanHLL.clear();
+    }
+
+    @Override
+    public void processElement(BlPreciseExposureBean blPreciseExposureBean, Context context, Collector<BlPreciseExposureBean> collector) throws Exception {
+        HLL hll = null;
+        if (this.hllState.value() == null) {
+            hll = new HLL(14, 5);
+        } else {
+            hll = HLL.fromBytes(ArrayUtils.toPrimitive(hllState.value()));
+//            hll.addRaw(blPreciseExposureBean.getJson());
+        }
+        hllState.update(ArrayUtils.toObject(hll.toBytes()));
+
+        if(hll.cardinality() != blPreciseExposureBeanHLL.cardinality()) {
+            collector.collect(blPreciseExposureBean);
+        }
+        context.timerService().registerProcessingTimeTimer(DateUtil.tomorrowZeroTimestampMs(Double.valueOf(blPreciseExposureBean.getGm_nginx_timestamp()).longValue() * 1000, 8) + 1);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/jdbc/MysqlJdbcSink.java

+package com.gmei.jdbc;
+
+import java.sql.*;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * ClassName: MysqlJdbcSink
+ * Function:
+ * Reason: 单例模式的数据下发jdbc工具
+ * Date: 2019/12/7 上午11:01
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class MysqlJdbcSink {
+    private volatile static MysqlJdbcSink mysqlJdbcSink = null;
+
+//    private static String driver = "com.mysql.cj.jdbc.Driver";
+//    public static String url = "jdbc:mysql://localhost:3306/zhengxing?useSSL=false&serverTimezone=UTC";
+//    //?useUnicode=true&characterEncoding=utf8&serverTimezone=GMT&allowMultiQueries=true&useSSL=false
+//    public static String username = "jayden";
+//    public static String password = "jayden548493";
+
+//    private String driver = "com.mysql.jdbc.Driver";
+//    private String url = "jdbc:mysql://172.22.30.12:3506/test?useUnicode=true&characterEncoding=UTF-8";//设置连接字符串
+//    //rewriteBatchedStatements=true
+//    private String username = "work";
+//    private String password = "zJnxVEhyyxeC7ciqxdMITVyWqOFc2mew";
+
+    private String driver = "com.mysql.cj.jdbc.Driver";
+    private String url;
+    private Connection conn;
+
+    private MysqlJdbcSink(String url) {
+        this.url = url;
+    }
+
+    public static MysqlJdbcSink getInstance(String url) {
+        if (mysqlJdbcSink == null) {
+            synchronized (MysqlJdbcSink.class) {
+                if(mysqlJdbcSink == null) {
+                    mysqlJdbcSink = new MysqlJdbcSink(url);
+                }
+            }
+        }
+        return mysqlJdbcSink;
+    }
+
+    public Connection getConnection() {
+        try {
+            Class.forName(driver);
+            if(conn == null || conn.isClosed()) {
+//                conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/zhengxing?useSSL=false&serverTimezone=UTC", "jayden", "jayden548943");
+//                conn = DriverManager.getConnection(url, username, password);
+                conn = DriverManager.getConnection(url);
+            }
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        return conn;
+    }
+
+    public void update(String sql, List<Object> params) throws SQLException {
+        PreparedStatement ps = null;
+        try {
+            ps = conn.prepareStatement(sql);
+            if (params != null && params.size() > 0) {
+                for (int i = 0; i < params.size(); i++) {
+                    ps.setObject(i + 1, params.get(i));
+                }
+            }
+            int resultNum = ps.executeUpdate();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        } finally {
+            ps.close();
+        }
+//        System.out.println(printRealSql(sql, params).replaceAll("(\\s+|\\\\n)", ""));
+    }
+
+    public void insert(String sql) throws SQLException {
+        Statement st = null;
+        try {
+            st = conn.createStatement();
+            st.executeUpdate(sql);
+        } catch (SQLException e) {
+            e.printStackTrace();
+        } finally {
+            st.close();
+        }
+    }
+
+    public String printRealSql(String sql, List<Object> params) {
+        if(params == null || params.size() == 0) {
+//            System.out.println("The SQL is------------>\n" + sql);
+            return sql;
+        }
+
+        if (!match(sql, params)) {
+            System.out.println("SQL 语句中的占位符与参数个数不匹配。SQL：" + sql);
+            return null;
+        }
+
+        int cols = params.size();
+//         values = params.toArray(index, count);
+//        Object[] values = new Object[cols];
+//        System.arraycopy(params, 0, values, 0, cols);
+        Object[] values = params.toArray(new Object[cols]);
+        for (int i = 0; i < cols; i++) {
+            Object value = values[i];
+            if (value instanceof Date) {
+                values[i] = "'" + value + "'";
+            } else if (value instanceof String) {
+                values[i] = "'" + value + "'";
+            } else if (value instanceof Boolean) {
+                values[i] = (Boolean) value ? 1 : 0;
+            }
+        }
+
+        String statement = String.format(sql.replaceAll("\\?", "%s"), values);
+
+//        System.out.println("The SQL is------------>\n" + statement);
+        return statement;
+    }
+
+    private static boolean match(String sql, List<Object> params) {
+        if(params == null || params.size() == 0) return true; // 没有参数，完整输出
+
+        Matcher m = Pattern.compile("(\\?)").matcher(sql);
+        int count = 0;
+        while (m.find()) {
+            count++;
+        }
+
+        return count == params.size();
+    }
+
+//    public ResultSet query(String sql, List<Object> param) {
+//        try {
+//            ps = conn.prepareStatement(sql);
+//
+//            if (param != null && param.size() > 0) {
+//                for (int i = 0; i < param.size(); i++) {
+//                    ps.setObject(i + 1, param.get(i));
+//                }
+//            }
+//            if(!ps.isClosed())
+//                System.out.println("Succeeded connecting to the prepareStatement!");
+//            rs = ps.executeQuery();
+//        } catch (SQLException e) {
+//            e.printStackTrace();
+//        }
+//        return rs;
+//    }
+
+    public ResultSet query(String sql) {
+//        System.out.println(sql);
+        if(conn == null){
+            conn = getConnection();
+        }
+        ResultSet rs = null;
+        Statement st = null;
+        try {
+
+            st = conn.createStatement();
+            rs = st.executeQuery(sql);
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        return rs;
+    }
+
+    public void close(Connection con, Statement st, ResultSet rs) {
+        if (rs != null) {
+            try {
+                rs.close();
+            } catch (SQLException e) {
+                e.printStackTrace();
+            }
+            if (st != null) {
+                try {
+                    st.close();
+                } catch (SQLException e) {
+                    e.printStackTrace();
+                }
+            }
+            if (con != null) {
+                try {
+                    con.close();
+                } catch (SQLException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/sink/BlPreciseExposureKafkaSink.java

+package com.gmei.sink;
+
+import com.alibaba.fastjson.JSON;
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+
+import java.util.Properties;
+
+/**
+ * ClassName: BlPreciseExposureMysqlSink
+ * Function:
+ * Reason: 数据下发mysql
+ * Date: 2019/12/16 下午6:45
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureKafkaSink extends RichSinkFunction<BlPreciseExposureBean> {
+
+    private String outBrokers;
+    private String outTopic;
+    private Properties props;
+    private KafkaProducer<String,String> kafkaProducer;
+
+
+    public BlPreciseExposureKafkaSink(String outBrokers, String outTopic) {
+        this.outBrokers = outBrokers;
+        this.outTopic = outTopic;
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        init();
+    }
+
+    @Override
+    public void invoke(BlPreciseExposureBean value, Context context) throws Exception {
+        kafkaProducer.send(new ProducerRecord<String,String>(outTopic, JSON.toJSONString(value)));
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        kafkaProducer.close();
+    }
+
+    private void init() throws Exception {
+        props = new Properties();
+        props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,outBrokers);
+        props.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG,"lz4");
+        props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
+        props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
+        kafkaProducer = new KafkaProducer(props);
+    }
+}
\ No newline at end of file

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/sink/BlPreciseExposureMysqlSink.java

+package com.gmei.sink;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.cache.BlPreciseExposureDao;
+import com.gmei.jdbc.MysqlJdbcSink;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+
+import java.sql.Connection;
+
+/**
+ * ClassName: BlPreciseExposureMysqlSink
+ * Function:
+ * Reason: 数据下发mysql
+ * Date: 2019/12/16 下午6:45
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlPreciseExposureMysqlSink extends RichSinkFunction<BlPreciseExposureBean> {
+
+    private MysqlJdbcSink mysqlJdbcSink;
+    private Connection conn;
+    private BlPreciseExposureDao blPreciseExposureDao;
+    private int maxRetry = 1;
+    private long retryTime = 3000;
+    private String sinkJdbcUrl;
+    private String sinkTableName;
+
+    public BlPreciseExposureMysqlSink(String sinkJdbcUrl, String sinkTableName) {
+        this.sinkJdbcUrl = sinkJdbcUrl;
+        this.sinkTableName = sinkTableName;
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        mysqlJdbcSink = MysqlJdbcSink.getInstance(sinkJdbcUrl);
+        blPreciseExposureDao = new BlPreciseExposureDao(sinkJdbcUrl, sinkTableName);
+        conn = mysqlJdbcSink.getConnection();
+    }
+
+    @Override
+    public void invoke(BlPreciseExposureBean value, Context context) throws Exception {
+
+        try {
+            conn.setAutoCommit(false);
+            blPreciseExposureDao.insertBlPreciseExposure(value);
+            conn.commit();
+        } catch (Exception e) {
+            conn.rollback();
+            int numRetry = 1;
+            Exception lastException = e;
+            while (numRetry <= maxRetry) {
+                try {
+                    numRetry ++;
+                    Thread.sleep(retryTime);
+                    mysqlJdbcSink.close(conn, null, null);
+                    conn = mysqlJdbcSink.getConnection();
+                    conn.setAutoCommit(false);
+                    blPreciseExposureDao.insertBlPreciseExposure(value);
+                    conn.commit();
+                } catch (Exception e1) {
+                    conn.rollback();
+                    lastException = e1;
+                    continue;
+                }
+                return;
+            }
+            throw lastException;
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        super.close();
+        mysqlJdbcSink.close(conn, null, null);
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/source/BlMaiDianKafkaSource.java

+package com.gmei.source;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
+
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.Properties;
+
+
+/**
+ * ClassName: BlMaiDianKafkaSource
+ * Function:
+ * Reason: 获取kafka资源
+ * Date: 2019/12/5 下午3:54
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BlMaiDianKafkaSource {
+
+    private String brokers;
+    private String topic;
+    private String groupId;
+    private String startTime;
+
+
+    public BlMaiDianKafkaSource() {
+    }
+
+    public BlMaiDianKafkaSource(String brokers, String topic, String groupId, String startTime) {
+        this.brokers = brokers;
+        this.topic = topic;
+        this.groupId = groupId;
+        this.startTime = startTime;
+    }
+
+    /**
+    * Function: addSource
+    * Reason: 获取kafka资源
+    * Date: 2019/12/25 下午4:49
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public FlinkKafkaConsumer011<String> addSource() throws Exception{
+        Properties props = new Properties();
+        props.put("bootstrap.servers", brokers);
+        props.put("group.id", groupId);
+        props.put("enable.auto.commit", "true");
+        props.put("auto.commit.interval.ms", "1000");
+//        props.put("auto.offset.reset", "earliest");
+        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+        FlinkKafkaConsumer011<String> flinkKafkaConsumer = new FlinkKafkaConsumer011<String>(topic, new SimpleStringSchema(Charset.forName("UTF-8")), props);
+//
+        if(startTime != null){
+            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+            flinkKafkaConsumer.setStartFromTimestamp(simpleDateFormat.parse(startTime).getTime());
+        } else {
+            flinkKafkaConsumer.setStartFromGroupOffsets();//默认消费策略
+        }
+//        flinkKafkaConsumer.setStartFromEarliest();
+        return flinkKafkaConsumer;
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/streaming/PreciseExposureStreaming.java

+package com.gmei.streaming;
+
+import com.gmei.bean.bl.BlPreciseExposureBean;
+import com.gmei.function.*;
+import com.gmei.sink.BlPreciseExposureMysqlSink;
+import com.gmei.source.BlMaiDianKafkaSource;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.runtime.state.filesystem.FsStateBackend;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.KeyedStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.datastream.SplitStream;
+import org.apache.flink.streaming.api.environment.CheckpointConfig;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
+import org.apache.flink.util.OutputTag;
+
+
+
+/**
+ * ClassName: PreciseExposureStreaming
+ * Function:
+ * Reason: flink主类
+ * Date: 2019/12/5 下午3:54
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class PreciseExposureStreaming {
+
+    /**
+    * Function: main
+    * Reason: 执行入口
+    * Date: 2019/12/24 上午11:03
+    * args: inBrokes、inTopic、inzk、groupId、sinkBFBlTableName、sinkHLLBlTableName、outBrokers、outTopic、dimJdbcUrl、sinkJdbcUrl
+    * 运行参数设置:
+      * 时间属性: ProcessingTime
+      * 聚合窗口时间: 1分钟
+      * 并行度: 1
+      * 异步查询最长等待时间: 1分钟
+      * 缓存大小: 2000
+      * 缓存过期时间: 24小时
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public static void main(String[] args) throws Exception {
+        String inBrokers = null;
+        String inTopic = null;
+        String groupId = null;
+        String dimJdbcUrl = null;
+        String sinkJdbcUrl = null;
+        String sinkBFBlTableName = null;
+        String sinkHLLBlTableName = null;
+        String sinkBMBlTableName = null;
+        Integer windowSize = null;
+        Integer parallelism = null;
+        String startTime = null;
+        String checkpointPath = null;
+        ParameterTool parameterTool = null;
+
+        try {
+//        ./flink run flink-stat-1.0-SNAPSHOT.jar -brokers kafka.dev.gengmei:9092 -topic gm-logging-test -sinkUrl jdbc:phoenix:localhost:2181 -sinkTable data.device_for_flink -zxUrl jdbc:mysql://localhost/seven?user=root&password=123123 -zxTable statistic_promotion_channel
+//        ./flink run flink-stat-1.0-SNAPSHOT.jar -brokers 172.21.40.14:9092 -topic gm-precise_exposure-test
+            parameterTool = ParameterTool.fromArgs(args);
+            inBrokers = parameterTool.getRequired("inBrokers");
+            inTopic = parameterTool.getRequired("inTopic");
+            groupId = parameterTool.get("groupId", "flink_preciseexposure_group");
+
+            dimJdbcUrl = parameterTool.getRequired("dimJdbcUrl");
+            sinkJdbcUrl = parameterTool.getRequired("sinkJdbcUrl");
+            sinkBFBlTableName = parameterTool.getRequired("sinkBFBlTableName");
+            sinkHLLBlTableName = parameterTool.getRequired("sinkHLLBlTableName");
+            sinkBMBlTableName = parameterTool.getRequired("sinkBMBlTableName");
+//        Boolean startFromLatest = parameterTool.getBoolean("startFromLatest", false);
+            windowSize = parameterTool.getInt("windowSize", 30);
+            parallelism = parameterTool.getInt("parallelism", 1);
+            startTime = parameterTool.get("startTime", null);
+            checkpointPath = parameterTool.getRequired("checkpointPath");
+            printUsage(parameterTool);
+        } catch (Exception e) {
+            e.printStackTrace();
+            printUsage();
+        }
+        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+        /*
+        checkpoint设置
+         */
+        //start a checkpoint every 1000 ms
+        env.enableCheckpointing(1000);
+        //set mode to exactly-once (this is the default)
+        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
+        //make sure 500 ms of progress happen between checkpoints
+        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500);
+        //checkpoints have to complete within one minute, or are discarded
+        env.getCheckpointConfig().setCheckpointTimeout(60000);
+        //allow only one checkpoint to be in progress at the same time
+        env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
+        //enable externalized checkpoints which are retained after job cancellation
+        env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
+        //This determines if a task will be failed if an error occurs in the execution of the task’s checkpoint procedure.
+        env.getCheckpointConfig().setFailOnCheckpointingErrors(true);
+        //设置statebackend
+        env.setStateBackend(new FsStateBackend(checkpointPath,true));
+        //重试次数1，重试间隔时间30s
+        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 3000));
+
+        //设置时间属性（EventTime:事件时间;IngestionTime:接入时间;ProcessingTime:处理时间（默认））
+        env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+//        DataStream<String> blMaiDianDataStream = env.socketTextStream("MacdeMacBook-Pro-9.local", 9000, "\n");
+
+        //正式环境
+        BlMaiDianKafkaSource blMaiDianKafkaSource = new BlMaiDianKafkaSource(inBrokers, inTopic, groupId, startTime);
+        //测试环境
+//        BlMaiDianKafkaSource blMaiDianKafkaSource = new BlMaiDianKafkaSource("localhost:9092", "test", "group1");
+
+        FlinkKafkaConsumer011<String> flinkKafkaConsumer = blMaiDianKafkaSource.addSource();
+
+        //maidian流分发
+        SplitStream<String> blMaiDianDataStream  = env
+                .addSource(flinkKafkaConsumer).uid("id_blmaidiandata_source").setParallelism(parallelism)
+                .split(new BlMaiDianOutputSelector());
+
+        OutputTag<BlPreciseExposureBean> outputTag = new OutputTag<BlPreciseExposureBean>("bl_preciseexposure_late_data"){};
+
+        /*
+        BL层数据清洗、转换、去重
+        Time.of(windowSize, TimeUnit.SECONDS)等价与Time.seconds(windowSize):30秒
+         */
+        @SuppressWarnings("unchecked")
+        KeyedStream<BlPreciseExposureBean, String> blPreciseExposureKeyedStream = blMaiDianDataStream
+                .select("et_pe")
+                .map(new BlPreciseExposureMapFunction()).uid("id_blpreciseexposure_map").setParallelism(parallelism)
+                .filter(new BlPreciseExposureFilterFunction()).uid("id_blpreciseexposure_filter").setParallelism(parallelism)
+                .keyBy(new BlPreciseExposureKeySelector());
+
+//        SingleOutputStreamOperator<BlPreciseExposureBean> blPreciseExposureStreamBF = blPreciseExposureKeyedStream.process(new BloomFilterFunction());
+//        SingleOutputStreamOperator<BlPreciseExposureBean> blPreciseExposureStreamHLL = blPreciseExposureKeyedStream.process(new HyperLogLogFunction());
+        SingleOutputStreamOperator<BlPreciseExposureBean> blPreciseExposureStreamBM = blPreciseExposureKeyedStream.process(new BitMapFunction());
+
+        /*
+        BL层数据下发
+         */
+//        blPreciseExposureStreamBF
+//                .print();
+//                .addSink(new BlPreciseExposureMysqlSink(sinkJdbcUrl, sinkBFBlTableName))
+//                .uid("id_blpreciseexposurebf_sink")
+//                .setParallelism(parallelism);
+
+//        blPreciseExposureStreamHLL
+//                .print();
+//                .addSink(new BlPreciseExposureMysqlSink(sinkJdbcUrl, sinkHLLBlTableName))
+//                .uid("id_blpreciseexposurehll_sink")
+//                .setParallelism(parallelism);
+
+        blPreciseExposureStreamBM
+                .print();
+//                .addSink(new BlPreciseExposureMysqlSink(sinkJdbcUrl, sinkBMBlTableName))
+//                .uid("id_blpreciseexposurebm_sink")
+//                .setParallelism(parallelism);
+//        //测试打印
+//        blPreciseExposureStreamBF.print();
+//        blPreciseExposureStreamHLL.print();
+//        blPreciseExposureStreamBF.addSink(new PrintSinkFunction<>());
+//        blPreciseExposureStreamHLL.addSink(new PrintSinkFunction<>());
+        env.execute("bl_et_pe_preciseexposure_inc_d_distinct");
+    }
+
+    /**
+    * Function: printUsage
+    * Reason: 打印参数说明
+    * Date: 2019/12/26 下午2:39
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+
+    public static void printUsage(){
+        System.out.println("Missing parameters!\n" +
+                "Usage:\n" +
+                "    --inBrokers <source kafka brokers> \n" +
+                "    --inTopic <source kafka topic>\n" +
+                "    --groupid <source kafka groupid, default: flink_preciseexposure_group> \n" +
+                "    --startFromLatest <start from the latest kafka record, default: false> \n" +
+                "    --windowSize <window size(second), default: 30 (s)> \n" +
+                "    --dimJdbcUrl <dim database url> \n" +
+                "    --sinkJdbcUrl <target database url> \n" +
+                "    --sinkBFBlTableName <target bl table name> \n" +
+                "    --sinkHLLBlTableName <target bl table name> \n" +
+                "    --sinkBMBlTableName <target bl table name> \n" +
+                "    --parallelism <parallelism, default 1> \n" +
+                "    --startTime <kafka startTime, default null> \n" +
+                "    --checkpointPath <checkpointPath, hdfs> \n"
+        );
+    }
+
+    /**
+    * Function: printUsage
+    * Reason: 打印参数
+    * Date: 2019/12/25 下午5:00
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+
+    public static void printUsage(ParameterTool args){
+        ParameterTool parameterTool = args;
+        System.out.println("Print parameters!\n" +
+                "Usage:\n" +
+                "    --inBrokers " + parameterTool.getRequired("inBrokers") + " \n" +
+                "    --inTopic " + parameterTool.getRequired("inTopic") + "\n" +
+                "    --groupid " + parameterTool.get("groupId", "flink_preciseexposure_group") + " \n" +
+                "    --startFromLatest <start from the latest kafka record, default: false> \n" +
+                "    --windowSize " + parameterTool.getInt("windowSize", 30) + " \n" +
+                "    --dimJdbcUrl " + parameterTool.getRequired("dimJdbcUrl") + " \n" +
+                "    --sinkJdbcUrl " + parameterTool.getRequired("sinkJdbcUrl") + " \n" +
+                "    --sinkBFBlTableName " + parameterTool.getRequired("sinkBFBlTableName") + " \n" +
+                "    --sinkHLLBlTableName " + parameterTool.getRequired("sinkHLLBlTableName") + " \n" +
+                "    --sinkBMBlTableName " + parameterTool.getRequired("sinkBMBlTableName") + " \n" +
+                "    --parallelism "+ parameterTool.getInt("parallelism", 1) + " \n" +
+                "    --startTime " + parameterTool.get("startTime", null) + " \n" +
+                "    --checkpointPath " + parameterTool.getRequired("checkpointPath") + " \n"
+        );
+    }
+
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/utils/BeanReflectUtil.java

+package com.gmei.utils;
+
+import java.lang.reflect.Field;
+
+/**
+ * ClassName: BeanReflectUtil
+ * Function:
+ * Reason: 类反射工具类
+ * Date: 2019/12/5 下午3:54
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class BeanReflectUtil {
+
+    /**
+    * Function: getFieldValue
+    * Reason: 通过类反射获取属性值
+    * Date: 2019/12/25 下午4:41
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public static <T> Object getFieldValue(T obj, String field) throws Exception {
+        Class<?> claz = obj.getClass();
+        Field f = null;
+        Object fieldValue = null;
+
+        Field[] fields = claz.getDeclaredFields();
+        for (int i = 0; i < fields.length; i++) {
+            if (fields[i].getName().equals(field)) {
+                f = claz.getDeclaredField(field);
+                f.setAccessible(true);
+                fieldValue = f.get(obj);
+            }
+        }
+        return fieldValue;
+    }
+
+    /**
+    * Function: setFieldValue
+    * Reason: 通过类反射设置属性值
+    * Date: 2019/12/25 下午4:41
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public static <T> void setFieldValue(T obj, String field, Object value) {
+        Class<?> claz = obj.getClass();
+        Field f = null;
+        try {
+            Field[] fields = claz.getDeclaredFields();
+            for (int i = 0; i < fields.length; i++) {
+                if (fields[i].getName().equals(field)) {
+                    f = claz.getDeclaredField(field);
+                    f.setAccessible(true);
+                    f.set(obj, value);
+                }
+            }
+        } catch (Exception e) {
+            e.getStackTrace();
+        }
+    }
+
+    /**
+    * Function: setNullValue
+    * Reason: 通过类反射设置所有空字符串为NULL
+    * Date: 2019/12/25 下午4:42
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public static <T> T setNullValue(T source) throws IllegalArgumentException, IllegalAccessException, SecurityException {
+        Field[] fields = source.getClass().getDeclaredFields();
+        for (Field field : fields) {
+            if (field.getGenericType().toString().equals("class java.lang.String")) {
+                field.setAccessible(true);
+                Object obj = field.get(source);
+                if (obj != null && obj.equals("")) {
+                    field.set(source, null);
+                }
+            }
+        }
+        return source;
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/utils/DateUtil.java

+package com.gmei.utils;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+/**
+ * ClassName: DateUtil
+ * Function:
+ * Reason: 日期格式工具类
+ * Date: 2019/12/17 下午4:51
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class DateUtil {
+
+    static SimpleDateFormat secDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    static SimpleDateFormat dayDate = new SimpleDateFormat("yyyyMMdd");
+
+    public enum DateType {
+        DAY,
+        SECOND;
+    }
+
+    /**
+    * Function: timestampFormat.
+     * DAY:yyyyMMdd
+     * SECOND:yyyy-MM-dd HH:mm:ss
+    * Reason: timestamp时间戳格式化.
+    * Date: 2019/12/25 下午4:43
+    *
+    * @author liuzhe
+    * @since JDK 1.8
+    */
+    public static String timestampFormat(String timestamp, DateType type) {
+        Double timestampDouble = Double.parseDouble(timestamp);
+        Long timestampLong = new Double(timestampDouble * 1000).longValue();
+        Date date= new Date(timestampLong);
+        switch (type) {
+            case DAY:
+                return dayDate.format(date);
+            case SECOND:
+                return secDate.format(date);
+            default:
+                return null;
+        }
+    }
+
+    public static long changeDateToLong(String time, SimpleDateFormat simpleDateFormat) throws ParseException {
+        return simpleDateFormat.parse(time).getTime();
+    }
+
+    // 根据当前时间戳获取第二天0时0分0秒的时间戳
+    public static long tomorrowZeroTimestampMs(long now, int timeZone) {
+        return now - (now + timeZone * 3600000) % 86400000 + 86400000;
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/utils/HyperLogLog.java

+package com.gmei.utils;
+
+/**
+ * ClassName: HyperLogLog
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/13 下午6:02
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class HyperLogLog {
+
+    private final RegisterSet registerSet;
+    private final int log2m;   //log(m)
+    private final double alphaMM;
+
+
+    /**
+     *
+     *  rsd = 1.04/sqrt(m)
+     * @param rsd  相对标准偏差
+     */
+    public HyperLogLog(double rsd) {
+        this(log2m(rsd));
+    }
+
+    /**
+     * rsd = 1.04/sqrt(m)
+     * m = (1.04 / rsd)^2
+     * @param rsd 相对标准偏差
+     * @return
+     */
+    private static int log2m(double rsd) {
+        return (int) (Math.log((1.106 / rsd) * (1.106 / rsd)) / Math.log(2));
+    }
+
+    private static double rsd(int log2m) {
+        return 1.106 / Math.sqrt(Math.exp(log2m * Math.log(2)));
+    }
+
+
+    /**
+     * accuracy = 1.04/sqrt(2^log2m)
+     *
+     * @param log2m
+     */
+    public HyperLogLog(int log2m) {
+        this(log2m, new RegisterSet(1 << log2m));
+    }
+
+    /**
+     *
+     * @param registerSet
+     */
+    public HyperLogLog(int log2m, RegisterSet registerSet) {
+        this.registerSet = registerSet;
+        this.log2m = log2m;
+        int m = 1 << this.log2m; //从log2m中算出m
+
+        alphaMM = getAlphaMM(log2m, m);
+    }
+
+
+    public boolean offerHashed(int hashedValue) {
+        // j 代表第几个桶,取hashedValue的前log2m位即可
+        // j 介于 0 到 m
+        final int j = hashedValue >>> (Integer.SIZE - log2m);
+        // r代表 除去前log2m位剩下部分的前导零 + 1
+        final int r = Integer.numberOfLeadingZeros((hashedValue << this.log2m) | (1 << (this.log2m - 1)) + 1) + 1;
+        return registerSet.updateIfGreater(j, r);
+    }
+
+    /**
+     * 添加元素
+     * @param o  要被添加的元素
+     * @return
+     */
+    public boolean offer(Object o) {
+        final int x = MurmurHash.hash(o);
+        return offerHashed(x);
+    }
+
+
+    public long cardinality() {
+        double registerSum = 0;
+        int count = registerSet.count;
+        double zeros = 0.0;
+        //count是桶的数量
+        for (int j = 0; j < registerSet.count; j++) {
+            int val = registerSet.get(j);
+            registerSum += 1.0 / (1 << val);
+            if (val == 0) {
+                zeros++;
+            }
+        }
+
+        double estimate = alphaMM * (1 / registerSum);
+
+        if (estimate <= (5.0 / 2.0) * count) {  //小数据量修正
+            return Math.round(linearCounting(count, zeros));
+        } else {
+            return Math.round(estimate);
+        }
+    }
+
+
+    /**
+     *  计算constant常数的取值
+     * @param p   log2m
+     * @param m   m
+     * @return
+     */
+    protected static double getAlphaMM(final int p, final int m) {
+        // See the paper.
+        switch (p) {
+            case 4:
+                return 0.673 * m * m;
+            case 5:
+                return 0.697 * m * m;
+            case 6:
+                return 0.709 * m * m;
+            default:
+                return (0.7213 / (1 + 1.079 / m)) * m * m;
+        }
+    }
+
+    /**
+     *
+     * @param m   桶的数目
+     * @param V   桶中0的数目
+     * @return
+     */
+    protected static double linearCounting(int m, double V) {
+        return m * Math.log(m / V);
+    }
+
+    public static void main(String[] args) {
+        HyperLogLog hyperLogLog = new HyperLogLog(0.1325);//64个桶
+        //集合中只有下面这些元素
+        hyperLogLog.offer("hhh");
+        hyperLogLog.offer("mmm");
+        hyperLogLog.offer("ccc");
+        hyperLogLog.offer("ccc");
+        hyperLogLog.offer("ccc");
+        hyperLogLog.offer("abc");
+        hyperLogLog.offer("def");
+        hyperLogLog.offer("def");
+        hyperLogLog.offer("xyz");
+        //估算基数
+        System.out.println(hyperLogLog.cardinality());
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/utils/MurmurHash.java

+package com.gmei.utils;
+
+/**
+ * ClassName: MurmurHash
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/13 下午6:03
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+/**
+ * 一种快速的非加密hash
+ * 适用于对保密性要求不高以及不在意hash碰撞攻击的场合
+ */
+public class MurmurHash {
+
+    public static int hash(Object o) {
+        if (o == null) {
+            return 0;
+        }
+        if (o instanceof Long) {
+            return hashLong((Long) o);
+        }
+        if (o instanceof Integer) {
+            return hashLong((Integer) o);
+        }
+        if (o instanceof Double) {
+            return hashLong(Double.doubleToRawLongBits((Double) o));
+        }
+        if (o instanceof Float) {
+            return hashLong(Float.floatToRawIntBits((Float) o));
+        }
+        if (o instanceof String) {
+            return hash(((String) o).getBytes());
+        }
+        if (o instanceof byte[]) {
+            return hash((byte[]) o);
+        }
+        return hash(o.toString());
+    }
+
+    public static int hash(byte[] data) {
+        return hash(data, data.length, -1);
+    }
+
+    public static int hash(byte[] data, int seed) {
+        return hash(data, data.length, seed);
+    }
+
+    public static int hash(byte[] data, int length, int seed) {
+        int m = 0x5bd1e995;
+        int r = 24;
+
+        int h = seed ^ length;
+
+        int len_4 = length >> 2;
+
+        for (int i = 0; i < len_4; i++) {
+            int i_4 = i << 2;
+            int k = data[i_4 + 3];
+            k = k << 8;
+            k = k | (data[i_4 + 2] & 0xff);
+            k = k << 8;
+            k = k | (data[i_4 + 1] & 0xff);
+            k = k << 8;
+            k = k | (data[i_4 + 0] & 0xff);
+            k *= m;
+            k ^= k >>> r;
+            k *= m;
+            h *= m;
+            h ^= k;
+        }
+
+        // avoid calculating modulo
+        int len_m = len_4 << 2;
+        int left = length - len_m;
+
+        if (left != 0) {
+            if (left >= 3) {
+                h ^= (int) data[length - 3] << 16;
+            }
+            if (left >= 2) {
+                h ^= (int) data[length - 2] << 8;
+            }
+            if (left >= 1) {
+                h ^= (int) data[length - 1];
+            }
+
+            h *= m;
+        }
+
+        h ^= h >>> 13;
+        h *= m;
+        h ^= h >>> 15;
+
+        return h;
+    }
+
+    public static int hashLong(long data) {
+        int m = 0x5bd1e995;
+        int r = 24;
+
+        int h = 0;
+
+        int k = (int) data * m;
+        k ^= k >>> r;
+        h ^= k * m;
+
+        k = (int) (data >> 32) * m;
+        k ^= k >>> r;
+        h *= m;
+        h ^= k * m;
+
+        h ^= h >>> 13;
+        h *= m;
+        h ^= h >>> 15;
+
+        return h;
+    }
+
+    public static long hash64(Object o) {
+        if (o == null) {
+            return 0l;
+        } else if (o instanceof String) {
+            final byte[] bytes = ((String) o).getBytes();
+            return hash64(bytes, bytes.length);
+        } else if (o instanceof byte[]) {
+            final byte[] bytes = (byte[]) o;
+            return hash64(bytes, bytes.length);
+        }
+        return hash64(o.toString());
+    }
+
+    // 64 bit implementation copied from here:  https://github.com/tnm/murmurhash-java
+
+    /**
+     * Generates 64 bit hash from byte array with default seed value.
+     *
+     * @param data   byte array to hash
+     * @param length length of the array to hash
+     * @return 64 bit hash of the given string
+     */
+    public static long hash64(final byte[] data, int length) {
+        return hash64(data, length, 0xe17a1465);
+    }
+
+
+    /**
+     * Generates 64 bit hash from byte array of the given length and seed.
+     *
+     * @param data   byte array to hash
+     * @param length length of the array to hash
+     * @param seed   initial seed value
+     * @return 64 bit hash of the given array
+     */
+    public static long hash64(final byte[] data, int length, int seed) {
+        final long m = 0xc6a4a7935bd1e995L;
+        final int r = 47;
+
+        long h = (seed & 0xffffffffl) ^ (length * m);
+
+        int length8 = length / 8;
+
+        for (int i = 0; i < length8; i++) {
+            final int i8 = i * 8;
+            long k = ((long) data[i8 + 0] & 0xff) + (((long) data[i8 + 1] & 0xff) << 8)
+                    + (((long) data[i8 + 2] & 0xff) << 16) + (((long) data[i8 + 3] & 0xff) << 24)
+                    + (((long) data[i8 + 4] & 0xff) << 32) + (((long) data[i8 + 5] & 0xff) << 40)
+                    + (((long) data[i8 + 6] & 0xff) << 48) + (((long) data[i8 + 7] & 0xff) << 56);
+
+            k *= m;
+            k ^= k >>> r;
+            k *= m;
+
+            h ^= k;
+            h *= m;
+        }
+
+        switch (length % 8) {
+            case 7:
+                h ^= (long) (data[(length & ~7) + 6] & 0xff) << 48;
+            case 6:
+                h ^= (long) (data[(length & ~7) + 5] & 0xff) << 40;
+            case 5:
+                h ^= (long) (data[(length & ~7) + 4] & 0xff) << 32;
+            case 4:
+                h ^= (long) (data[(length & ~7) + 3] & 0xff) << 24;
+            case 3:
+                h ^= (long) (data[(length & ~7) + 2] & 0xff) << 16;
+            case 2:
+                h ^= (long) (data[(length & ~7) + 1] & 0xff) << 8;
+            case 1:
+                h ^= (long) (data[length & ~7] & 0xff);
+                h *= m;
+        }
+        ;
+
+        h ^= h >>> r;
+        h *= m;
+        h ^= h >>> r;
+
+        return h;
+    }
+}

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/utils/RegisterSet.java

+package com.gmei.utils;
+
+/**
+ * ClassName: RegisterSet
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/13 下午6:03
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+public class RegisterSet {
+
+    public final static int LOG2_BITS_PER_WORD = 6;  //2的6次方是64
+    public final static int REGISTER_SIZE = 5;     //每个register占5位,代码里有一些细节涉及到这个5位，所以仅仅改这个参数是会报错的
+
+    public final int count;
+    public final int size;
+
+    private final int[] M;
+
+    //传入m
+    public RegisterSet(int count) {
+        this(count, null);
+    }
+
+    public RegisterSet(int count, int[] initialValues) {
+        this.count = count;
+
+        if (initialValues == null) {
+            /**
+             * 分配(m / 6)个int给M
+             *
+             * 因为一个register占五位，所以每个int（32位）有6个register
+             */
+            this.M = new int[getSizeForCount(count)];
+        } else {
+            this.M = initialValues;
+        }
+        //size代表RegisterSet所占字的大小
+        this.size = this.M.length;
+    }
+
+    public static int getBits(int count) {
+        return count / LOG2_BITS_PER_WORD;
+    }
+
+    public static int getSizeForCount(int count) {
+        int bits = getBits(count);
+        if (bits == 0) {
+            return 1;
+        } else if (bits % Integer.SIZE == 0) {
+            return bits;
+        } else {
+            return bits + 1;
+        }
+    }
+
+    public void set(int position, int value) {
+        int bucketPos = position / LOG2_BITS_PER_WORD;
+        int shift = REGISTER_SIZE * (position - (bucketPos * LOG2_BITS_PER_WORD));
+        this.M[bucketPos] = (this.M[bucketPos] & ~(0x1f << shift)) | (value << shift);
+    }
+
+    public int get(int position) {
+        int bucketPos = position / LOG2_BITS_PER_WORD;
+        int shift = REGISTER_SIZE * (position - (bucketPos * LOG2_BITS_PER_WORD));
+        return (this.M[bucketPos] & (0x1f << shift)) >>> shift;
+    }
+
+    public boolean updateIfGreater(int position, int value) {
+        int bucket = position / LOG2_BITS_PER_WORD;    //M下标
+        int shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD));  //M偏移
+        int mask = 0x1f << shift;      //register大小为5位
+
+        // 这里使用long是为了避免int的符号位的干扰
+        long curVal = this.M[bucket] & mask;
+        long newVal = value << shift;
+        if (curVal < newVal) {
+            //将M的相应位置为新的值
+            this.M[bucket] = (int) ((this.M[bucket] & ~mask) | newVal);
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public void merge(RegisterSet that) {
+        for (int bucket = 0; bucket < M.length; bucket++) {
+            int word = 0;
+            for (int j = 0; j < LOG2_BITS_PER_WORD; j++) {
+                int mask = 0x1f << (REGISTER_SIZE * j);
+
+                int thisVal = (this.M[bucket] & mask);
+                int thatVal = (that.M[bucket] & mask);
+                word |= (thisVal < thatVal) ? thatVal : thisVal;
+            }
+            this.M[bucket] = word;
+        }
+    }
+
+    int[] readOnlyBits() {
+        return M;
+    }
+
+    public int[] bits() {
+        int[] copy = new int[size];
+        System.arraycopy(M, 0, copy, 0, M.length);
+        return copy;
+    }
+}
\ No newline at end of file

bl_et_pe_preciseexposure_inc_d_bm/src/main/java/com/gmei/utils/SnowFlake.java

+package com.gmei.utils;
+
+/**
+ * ClassName: SnowFlake
+ * Function: TODO ADD FUNCTION.
+ * Reason: TODO ADD REASON.
+ * Date: 2020/1/14 下午8:21
+ *
+ * @author liuzhe
+ * @since JDK 1.8
+ */
+
+public class SnowFlake {
+    /**
+     * 起始的时间戳
+     */
+    private final static long START_STMP = 1480166465631L;
+    /**
+     * 每一部分占用的位数
+     */
+    private final static long SEQUENCE_BIT = 12; //序列号占用的位数
+    private final static long MACHINE_BIT = 5;   //机器标识占用的位数
+    private final static long DATACENTER_BIT = 5;//数据中心占用的位数
+    /**
+     * 每一部分的最大值
+     */
+    private final static long MAX_DATACENTER_NUM = -1L ^ (-1L << DATACENTER_BIT);
+    private final static long MAX_MACHINE_NUM = -1L ^ (-1L << MACHINE_BIT);
+    private final static long MAX_SEQUENCE = -1L ^ (-1L << SEQUENCE_BIT);
+    /**
+     * 每一部分向左的位移
+     */
+    private final static long MACHINE_LEFT = SEQUENCE_BIT;
+    private final static long DATACENTER_LEFT = SEQUENCE_BIT + MACHINE_BIT;
+    private final static long TIMESTMP_LEFT = DATACENTER_LEFT + DATACENTER_BIT;
+    private long datacenterId;  //数据中心
+    private long machineId;     //机器标识
+    private long sequence = 0L; //序列号
+    private long lastStmp = -1L;//上一次时间戳
+
+    public SnowFlake(long datacenterId, long machineId) {
+        if (datacenterId > MAX_DATACENTER_NUM || datacenterId < 0) {
+            throw new IllegalArgumentException("datacenterId can't be greater than MAX_DATACENTER_NUM or less than 0");
+        }
+        if (machineId > MAX_MACHINE_NUM || machineId < 0) {
+            throw new IllegalArgumentException("machineId can't be greater than MAX_MACHINE_NUM or less than 0");
+        }
+        this.datacenterId = datacenterId;
+        this.machineId = machineId;
+    }
+
+    /**
+     * 产生下一个ID
+     *
+     * @return
+     */
+    public synchronized long nextId() {
+        long currStmp = getNewstmp();
+        if (currStmp < lastStmp) {
+            throw new RuntimeException("Clock moved backwards.  Refusing to generate id");
+        }
+        if (currStmp == lastStmp) {
+            //相同毫秒内，序列号自增
+            sequence = (sequence + 1) & MAX_SEQUENCE;
+            //同一毫秒的序列数已经达到最大
+            if (sequence == 0L) {
+                currStmp = getNextMill();
+            }
+        } else {
+            //不同毫秒内，序列号置为0
+            sequence = 0L;
+        }
+        lastStmp = currStmp;
+        return (currStmp - START_STMP) << TIMESTMP_LEFT //时间戳部分
+                | datacenterId << DATACENTER_LEFT       //数据中心部分
+                | machineId << MACHINE_LEFT             //机器标识部分
+                | sequence;                             //序列号部分
+    }
+
+    private long getNextMill() {
+        long mill = getNewstmp();
+        while (mill <= lastStmp) {
+            mill = getNewstmp();
+        }
+        return mill;
+    }
+
+    private long getNewstmp() {
+        return System.currentTimeMillis();
+    }
+
+    public static void main(String[] args) {
+        SnowFlake snowFlake = new SnowFlake(00001,00002);
+        System.out.println(snowFlake.nextId());
+    }
+}
\ No newline at end of file

bl_et_pe_preciseexposure_inc_d_distinct/src/main/java/com/gmei/function/BitMapFunction.java

@@ -67,6 +67,8 @@ public class BitMapFunction extends KeyedProcessFunction<String, BlPreciseExposu
             bitMapState.update(bitMapValue);
 //            bitMap.add(hashCode);
             collector.collect(blPreciseExposureBean);
+        } else {
+            System.out.println(blPreciseExposureBean);
         }
 //        context.timerService().registerProcessingTimeTimer(DateUtil.tomorrowZeroTimestampMs(Double.valueOf(blPreciseExposureBean.getGm_nginx_timestamp()).longValue() * 1000, 8) + 1);
     }

bl_et_pe_preciseexposure_inc_d_distinct/src/test/java/com/gmei/function/BitMapFunctionTest.java

@@ -4,6 +4,9 @@ import com.gmei.cache.PreciseAccumulator;
 import com.gmei.utils.MurmurHash;
 import org.junit.Test;
 
+import java.util.HashSet;
+import java.util.Set;
+
 /**
  * ClassName: BitMapFunctionTest
  * Function: TODO ADD FUNCTION.
@@ -44,4 +47,35 @@ public class BitMapFunctionTest {
         preciseAccumulator.add(jsonHash3);
         System.out.println("3:" + preciseAccumulator.getCardinality());
     }
+
+    @Test
+    public void hashCodeTest() {
+        String json1 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746257.232,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json2 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746257.233,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"version\":\"7.20.0\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json3 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746257.665,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"version\":\"7.20.0\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json4 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746257.666,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"version\":\"7.20.0\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"lat\":31.331567703747,\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"ip\":\"10.10.205.62\",\"device_type\":\"ios\",\"manufacturer\":\"Apple\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json5 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746258.895,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"version\":\"7.20.0\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json6 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746258.895,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"version\":\"7.20.0\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json7 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746258.895,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"ip\":\"10.10.205.62\",\"lat\":31.331567703747,\"device_type\":\"ios\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json8 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746258.895,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json9 = "{\"create_at\":\"1578746235\",\"gm_nginx_timestamp\":1578746258.896,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":96,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":4,\"page_name\":\"question_detail\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":7,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"783034\"},{\"absolute_position\":8,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"version\":\"7.20.0\",\"user_type\":{\"config_type\":1},\"serial_id\":17,\"channel\":\"App Store\"},\"gm_nginx_key\":5,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json10 = "{\"create_at\":\"1578746311\",\"gm_nginx_timestamp\":1578746311.862,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":138,\"referrer_link\":[\"question_detail\"],\"tab_name\":\"\",\"down_slide_times\":5,\"page_name\":\"question_detail\",\"up_loading_times\":4,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":46,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":3,\"card_id\":\"19603546\"},{\"absolute_position\":47,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":4,\"card_id\":\"707281\"},{\"absolute_position\":48,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"79248\"},{\"absolute_position\":49,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"19590583\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":50,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json11 = "{\"create_at\":\"1578746311\",\"gm_nginx_timestamp\":1578746311.863,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":138,\"referrer_link\":[\"question_detail\"],\"tab_name\":\"\",\"down_slide_times\":5,\"page_name\":\"question_detail\",\"up_loading_times\":4,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":46,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":3,\"card_id\":\"19603546\"},{\"absolute_position\":47,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":4,\"card_id\":\"707281\"},{\"absolute_position\":48,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"79248\"},{\"absolute_position\":49,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"19590583\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"version\":\"7.20.0\",\"user_type\":{\"config_type\":1},\"serial_id\":50,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json12 = "{\"create_at\":\"1578746312\",\"gm_nginx_timestamp\":1578746312.978,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":138,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":5,\"page_name\":\"question_detail\",\"up_loading_times\":5,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":42,\"in_page_pos\":\"recommend\",\"card_content_type\":\"diary\",\"transaction_type\":\"-1\",\"card_type\":\"video\",\"card_style\":\"cascade\",\"relative_position\":0,\"card_id\":\"16187543\"},{\"absolute_position\":44,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":1,\"card_id\":\"80165\"},{\"absolute_position\":45,\"in_page_pos\":\"recommend\",\"card_content_type\":\"diary\",\"transaction_type\":\"-1\",\"card_type\":\"video\",\"card_style\":\"cascade\",\"relative_position\":2,\"card_id\":\"16430639\"},{\"absolute_position\":46,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":3,\"card_id\":\"19603546\"},{\"absolute_position\":47,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":4,\"card_id\":\"707281\"},{\"absolute_position\":48,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"79248\"},{\"absolute_position\":49,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"19590583\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":53,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json13 = "{\"create_at\":\"1578746318\",\"gm_nginx_timestamp\":1578746319.142,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":140,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":5,\"page_name\":\"question_detail\",\"up_loading_times\":5,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":46,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":0,\"card_id\":\"19603546\"},{\"absolute_position\":47,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":1,\"card_id\":\"707281\"},{\"absolute_position\":48,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":2,\"card_id\":\"79248\"},{\"absolute_position\":49,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":3,\"card_id\":\"19590583\"},{\"absolute_position\":50,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":4,\"card_id\":\"598515\"},{\"absolute_position\":51,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"20311\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":59,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json14 = "{\"create_at\":\"1578746321\",\"gm_nginx_timestamp\":1578746321.492,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":142,\"referrer_link\":{},\"tab_name\":\"\",\"down_slide_times\":5,\"page_name\":\"question_detail\",\"up_loading_times\":5,\"is_exposure\":\"1\",\"referrer\":null,\"exposure_cards\":[{\"absolute_position\":48,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":0,\"card_id\":\"79248\"},{\"absolute_position\":49,\"in_page_pos\":\"recommend\",\"card_content_type\":\"article\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":1,\"card_id\":\"19590583\"},{\"absolute_position\":50,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":2,\"card_id\":\"598515\"},{\"absolute_position\":51,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":3,\"card_id\":\"20311\"},{\"absolute_position\":52,\"in_page_pos\":\"recommend\",\"card_content_type\":\"diary\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":4,\"card_id\":\"16179569\"},{\"absolute_position\":53,\"in_page_pos\":\"recommend\",\"card_content_type\":\"answer\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":5,\"card_id\":\"745966\"},{\"absolute_position\":54,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"10707\"}],\"business_id\":\"230926\",\"referrer_id\":null},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":65,\"channel\":\"App Store\"},\"gm_nginx_key\":3,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String json15 = "{\"create_at\":\"1578746444\",\"gm_nginx_timestamp\":1578746444.682,\"user_id\":\"31293003\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":65,\"referrer_link\":[\"question_detail\"],\"tab_name\":\"\",\"down_slide_times\":1,\"page_name\":\"user_post_detail\",\"up_loading_times\":3,\"is_exposure\":\"1\",\"referrer\":\"question_detail\",\"exposure_cards\":[{\"absolute_position\":37,\"in_page_pos\":\"recommend\",\"card_content_type\":\"user_post\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_style\":\"cascade\",\"relative_position\":6,\"card_id\":\"79248\"}],\"business_id\":\"10707\",\"referrer_id\":\"230926\"},\"app_session_id\":\"39B82FF5-DB3F-4CE6-BE35-10115051A859\",\"app\":{\"version\":\"7.20.0\",\"grey_type\":\"{\\n  \\\"face_detect_result\\\" : \\\"B\\\",\\n  \\\"search_result_welfare\\\" : false,\\n  \\\"post_detail\\\" : \\\"0\\\",\\n  \\\"home\\\" : \\\"0\\\",\\n  \\\"face_scan\\\" : 0,\\n  \\\"search_gray\\\" : false,\\n  \\\"video_steep\\\" : 0,\\n  \\\"launch_gray\\\" : false,\\n  \\\"report_result\\\" : 0\\n}\",\"current_city_id\":\"shanghai\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":100,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"F421E310-5A13-4694-9E6B-5B3EC8228E25\",\"sys_version\":\"13.3\",\"lng\":120.74191122379,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"device_type\":\"ios\",\"ip\":\"10.10.205.62\",\"idfa\":\"B7A61B8E-5423-4CCB-8BAE-1260BFBDF7D0\",\"lat\":31.331567703747,\"model\":\"iPhone11,8\"},\"type\":\"page_precise_exposure\"}";
+        String[] jsonArray = new String[] {json1,json2,json3,json4,json5,json6,json7,json8,json9,json10,json11,json12,json13,json14,json15};
+        Set jsonSet = new HashSet<>();
+        Set jsonHashSet = new HashSet<>();
+
+        for (int i = 0; i<jsonArray.length; i++) {
+            jsonSet.add(jsonArray[i]);
+            System.out.println(MurmurHash.hash64(jsonArray[i]));
+            jsonHashSet.add(MurmurHash.hash64(jsonArray[i]));
+        }
+        System.out.println(jsonArray.length);
+        System.out.println(jsonSet.size());
+        System.out.println(jsonHashSet.size());
+    }
 }

pom.xml

@@ -11,6 +11,7 @@
     <modules>
         <module>ml_c_et_pe_preciseexposure_dimen_d_rt</module>
         <module>bl_et_pe_preciseexposure_inc_d_distinct</module>
+        <module>bl_et_pe_preciseexposure_inc_d_bm</module>
     </modules>
 
     <properties>