ml_c_et_pe_preciseexposure_dimen_d_rt add day_id

ml_c_et_pe_preciseexposure_dimen_d_rt/src/main/java/com/gmei/bean/bl/BlPreciseExposureBean.java

 package com.gmei.bean.bl;
 
+import java.util.Date;
 import java.util.ArrayList;
 
 
@@ -15,6 +16,7 @@ import java.util.ArrayList;
  */
 public class BlPreciseExposureBean {
 
+    private Date day_id;
     private String json;
     private String gm_nginx_timestamp;
     private String create_timestamp;
@@ -52,7 +54,8 @@ public class BlPreciseExposureBean {
     public BlPreciseExposureBean() {
     }
 
-    public BlPreciseExposureBean(String json, String gm_nginx_timestamp, String create_timestamp, String user_id, String action, Integer down_loading_times, Integer down_slide_times, Integer up_loading_times, Integer up_slide_times, String page_code, String tab_name, String business_id, String referrer_code, String referrer_id, ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards, String is_exposure, String is_popup, String filter, String query, String app_grey_type, String app_channel, String app_version, String app_current_city_id, String app_code, String device_os_type, String device_model, String device_id, String device_android_id, String device_idfv, String gm_nginx_time_date, String gm_nginx_time_day, String create_time_date, String create_time_day) {
+    public BlPreciseExposureBean(Date day_id, String json, String gm_nginx_timestamp, String create_timestamp, String user_id, String action, Integer down_loading_times, Integer down_slide_times, Integer up_loading_times, Integer up_slide_times, String page_code, String tab_name, String business_id, String referrer_code, String referrer_id, ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards, String is_exposure, String is_popup, String filter, String query, String app_grey_type, String app_channel, String app_version, String app_current_city_id, String app_code, String device_os_type, String device_model, String device_id, String device_android_id, String device_idfv, String gm_nginx_time_date, String gm_nginx_time_day, String create_time_date, String create_time_day) {
+        this.day_id = day_id;
         this.json = json;
         this.gm_nginx_timestamp = gm_nginx_timestamp;
         this.create_timestamp = create_timestamp;
@@ -88,6 +91,14 @@ public class BlPreciseExposureBean {
         this.create_time_day = create_time_day;
     }
 
+    public Date getDay_id() {
+        return day_id;
+    }
+
+    public void setDay_id(Date day_id) {
+        this.day_id = day_id;
+    }
+
     public String getJson() {
         return json;
     }
@@ -355,7 +366,8 @@ public class BlPreciseExposureBean {
     @Override
     public String toString() {
         return "BlPreciseExposureBean{" +
-                "json='" + json + '\'' +
+                "day_id='" + day_id + '\'' +
+                ", json='" + json + '\'' +
                 ", gm_nginx_timestamp='" + gm_nginx_timestamp + '\'' +
                 ", create_timestamp='" + create_timestamp + '\'' +
                 ", user_id='" + user_id + '\'' +

ml_c_et_pe_preciseexposure_dimen_d_rt/src/main/java/com/gmei/bean/ml/MlPreciseExposureBean.java

 package com.gmei.bean.ml;
 
+
+import java.util.Date;
+
 /**
  * ClassName: MlPreciseExposureBean
  * TableName: ML_C_ET_PE_PRECISEEXPOSURE_DIMEN_D_RT
@@ -37,6 +40,7 @@ package com.gmei.bean.ml;
 */
 public class MlPreciseExposureBean {
 
+    private Date day_id;
 //    private String gm_nginx_timestamp;
 //    private String create_timestamp;
     private String user_id;
@@ -51,7 +55,6 @@ public class MlPreciseExposureBean {
     private String referrer_name; //匹配码表DIM_PAGE_TYPE
     private String referrer_id;
 
-
     private String card_id;
     private String card_name;
     private String card_content_type;
@@ -79,7 +82,6 @@ public class MlPreciseExposureBean {
     private String current_region_id; //匹配码表DIM_CITY
     private String current_region_name; //匹配码表DIM_CITY
 
-
     private String app_code;
     private String device_os_type;
 //    private String device_model;
@@ -95,7 +97,8 @@ public class MlPreciseExposureBean {
     public MlPreciseExposureBean() {
     }
 
-    public MlPreciseExposureBean(String user_id, String action, String page_code, String page_name, String tab_name, String business_id, String referrer_code, String referrer_name, String referrer_id, String card_id, String card_name, String card_content_type, String card_content_type_name, String card_type, String card_type_name, String is_cpc, String cpc_referer, String transaction_type, String transaction_type_name, String filter, String query, String app_version, String current_city_id, String current_city_name, String current_province_id, String current_province_name, String current_country_id, String current_country_name, String current_region_id, String current_region_name, String app_code, String device_os_type, String device_id, String create_time_day, String gm_nginx_time_day, Integer preciseexposure_num) {
+    public MlPreciseExposureBean(Date day_id, String user_id, String action, String page_code, String page_name, String tab_name, String business_id, String referrer_code, String referrer_name, String referrer_id, String card_id, String card_name, String card_content_type, String card_content_type_name, String card_type, String card_type_name, String is_cpc, String cpc_referer, String transaction_type, String transaction_type_name, String filter, String query, String app_version, String current_city_id, String current_city_name, String current_province_id, String current_province_name, String current_country_id, String current_country_name, String current_region_id, String current_region_name, String app_code, String device_os_type, String device_id, String create_time_day, String gm_nginx_time_day, Integer preciseexposure_num) {
+        this.day_id = day_id;
         this.user_id = user_id;
         this.action = action;
 //        this.action_name = action_name;
@@ -136,6 +139,14 @@ public class MlPreciseExposureBean {
         this.preciseexposure_num = preciseexposure_num;
     }
 
+    public Date getDay_id() {
+        return day_id;
+    }
+
+    public void setDay_id(Date day_id) {
+        this.day_id = day_id;
+    }
+
     public String getUser_id() {
         return user_id;
     }
@@ -459,7 +470,8 @@ public class MlPreciseExposureBean {
     @Override
     public String toString() {
         return "MlPreciseExposureBean{" +
-                "user_id='" + user_id + '\'' +
+                "day_id='" + day_id + '\'' +
+                ", user_id='" + user_id + '\'' +
                 ", action='" + action + '\'' +
                 ", page_code='" + page_code + '\'' +
                 ", page_name='" + page_name + '\'' +

ml_c_et_pe_preciseexposure_dimen_d_rt/src/main/java/com/gmei/cache/BlPreciseExposureDao.java

@@ -40,7 +40,8 @@ public class BlPreciseExposureDao {
     */
     public void insertBlPreciseExposure(BlPreciseExposureBean blPreciseExposureBean) throws Exception {
         sql = "insert into " + sinkTableName + "\n" +
-                "  (json,\n" +
+                "  (day_id,\n" +
+                "   json,\n" +
                 "   gm_nginx_timestamp,\n" +
                 "   create_timestamp,\n" +
                 "   user_id,\n" +
@@ -106,12 +107,14 @@ public class BlPreciseExposureDao {
                 "   ?,\n" +
                 "   ?,\n" +
                 "   ?,\n" +
+                "   ?,\n" +
                 "   ?)";
 
         List<Object> params = new ArrayList<Object>();
         String json = blPreciseExposureBean.getJson();
 //        Clob clob = connection.createClob();
 //        clob.setString(1, json);
+        params.add(blPreciseExposureBean.getDay_id());
         params.add(json);
         params.add(blPreciseExposureBean.getGm_nginx_timestamp());
         params.add(blPreciseExposureBean.getCreate_timestamp());

ml_c_et_pe_preciseexposure_dimen_d_rt/src/main/java/com/gmei/cache/MlPreciseExposureDao.java

@@ -37,7 +37,8 @@ public class MlPreciseExposureDao {
     */
     public void insertMlPreciseExposure(MlPreciseExposureBean mlPreciseExposureBean) throws Exception {
         sql = "insert into " + sinkTableName + "\n" +
-                "  (action,\n" +
+                "  (day_id,\n" +
+                "   action,\n" +
                 "   app_code,\n" +
                 "   page_code,\n" +
                 "   page_name,\n" +
@@ -109,11 +110,12 @@ public class MlPreciseExposureDao {
                 "   ?,\n" +
                 "   ?,\n" +
                 "   ?,\n" +
+                "   ?,\n" +
                 "   ?)";
 
         List<Object> params = new ArrayList<Object>();
+        params.add(mlPreciseExposureBean.getGm_nginx_time_day());
         params.add(mlPreciseExposureBean.getAction());
-//        params.add(mlPreciseExposureBean.getAction_name());
         params.add(mlPreciseExposureBean.getApp_code());
         params.add(mlPreciseExposureBean.getPage_code());
         params.add(mlPreciseExposureBean.getPage_name());

ml_c_et_pe_preciseexposure_dimen_d_rt/src/main/java/com/gmei/function/BlPreciseExposureMapFunction.java

@@ -11,6 +11,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.api.common.functions.MapFunction;
 
 import java.util.ArrayList;
+import java.util.Date;
 
 
 /**
@@ -63,6 +64,7 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
 
         String gm_nginx_time_date = null;
         String gm_nginx_time_day = null;
+        Date day_id = null;
         String create_time_date = null;
         String create_time_day = null;
 
@@ -193,6 +195,7 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
         //日志时间处理
         gm_nginx_time_date = DateUtil.timestampFormat(gm_nginx_timestamp, DateUtil.DateType.SECOND);
         gm_nginx_time_day = DateUtil.timestampFormat(gm_nginx_timestamp, DateUtil.DateType.DAY);
+        day_id = DateUtil.stringToDate(gm_nginx_timestamp);
         create_time_date = DateUtil.timestampFormat(create_timestamp, DateUtil.DateType.SECOND);
         create_time_day = DateUtil.timestampFormat(create_timestamp, DateUtil.DateType.DAY);
 
@@ -205,6 +208,7 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
             tab_name = null;
         }
 
+        blPreciseExposureBean.setDay_id(day_id);
         blPreciseExposureBean.setJson(jsonString);
         blPreciseExposureBean.setGm_nginx_timestamp(gm_nginx_timestamp);
         blPreciseExposureBean.setCreate_timestamp(create_timestamp);

ml_c_et_pe_preciseexposure_dimen_d_rt/src/main/java/com/gmei/function/MlpreciseExposureFlatMapFunction.java

@@ -74,16 +74,13 @@ public class MlpreciseExposureFlatMapFunction implements FlatMapFunction<BlPreci
                 card_type = "search_word";
             }
 
+            //1.14 百科卡片曝光新增参数wiki_type
+            //问题：从7.19.0版本开始，百科卡片曝光,新增了wiki_type参数
+
             //1.12 card_content_type的qa值应该被换成q_a
             if ("qa".equals(card_content_type)) {
                 card_content_type = card_content_type.replace("qa","q_a");
             }
-
-            //1.13 target_name与card_name的参数的统一
-            //问题描述：将target_name参数的值赋给card_name
-            //1.14 百科卡片曝光新增参数wiki_type
-            //问题：从7.19.0版本开始，百科卡片曝光,新增了wiki_type参数
-
             //card_type为card的标准码值为common_card,数据中为card
             if("card".equals(card_type)) {
                 card_type = "common_card";
@@ -92,8 +89,18 @@ public class MlpreciseExposureFlatMapFunction implements FlatMapFunction<BlPreci
             if("banner".equals(card_type)) {
                 card_type = "common_banner";
             }
+            //card_type为video的标准码值为video_card,数据中为video
+            if("video".equals(card_type)) {
+                card_type = "video_card";
+            }
+
+            //1.16 初期的搜索词曝光业务类型运营有promote,operating字段，需要修正为operation
+            if("promote".equals(transaction_type) || "operating".equals(transaction_type)) {
+                transaction_type = "operation";
+            }
 
             MlPreciseExposureBean mlPreciseExposureBean = new MlPreciseExposureBean();
+            mlPreciseExposureBean.setDay_id(blPreciseExposureBean.getDay_id());
             mlPreciseExposureBean.setUser_id(blPreciseExposureBean.getUser_id());
             mlPreciseExposureBean.setAction(blPreciseExposureBean.getAction());
 

ml_c_et_pe_preciseexposure_dimen_d_rt/src/main/java/com/gmei/utils/DateUtil.java

@@ -16,10 +16,12 @@ public class DateUtil {
 
     static SimpleDateFormat secDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     static SimpleDateFormat dayDate = new SimpleDateFormat("yyyyMMdd");
+    static SimpleDateFormat day_idDate = new SimpleDateFormat("yyyy-MM-dd");
 
     public enum DateType {
         DAY,
-        SECOND;
+        SECOND,
+        DAY_ID;
     }
 
     /**
@@ -41,8 +43,17 @@ public class DateUtil {
                 return dayDate.format(date);
             case SECOND:
                 return secDate.format(date);
+            case DAY_ID:
+                return day_idDate.format(date);
             default:
                 return null;
         }
     }
+
+    public static Date stringToDate(String timestamp) {
+        Double timestampDouble = Double.parseDouble(timestamp);
+        Long timestampLong = new Double(timestampDouble * 1000).longValue();
+        Date date= new Date(timestampLong);
+        return date;
+    }
 }