Skip to content

F
flink_warehouse_rt
Commit 0b60b567 authored by 刘喆's avatar 刘喆
Browse files
Options

update mlflatmap error

parent e15a2105
Hide whitespace changes
Inline Side-by-side
Showing with 695 additions and 101 deletions
pom.xml
View file @ 0b60b567
......@@ -94,8 +94,8 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.38</version>
<!--<version>8.0.12</version>-->
<!--<version>5.1.38</version>-->
<version>8.0.12</version>
</dependency>
......
src/main/java/com/gmei/bean/bl/BlPreciseExposureParamsExposureCardsBean.java
View file @ 0b60b567
package com.gmei.bean.bl;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
/**
* ClassName: BlPreciseExposureBean
* TableName: Bl_Et_Mg_PreciseExposure_Inc_D
......@@ -90,6 +88,8 @@ public class BlPreciseExposureParamsExposureCardsBean {
private String card_id;
private String card_content_type;
private String card_type;
private String card_name;
private String target_name;
private String transaction_type;
private String is_cpc;
private String cpc_referer;
......@@ -108,6 +108,22 @@ public class BlPreciseExposureParamsExposureCardsBean {
this.card_type = card_type;
}
public String getCard_name() {
return card_name;
}
public void setCard_name(String card_name) {
this.card_name = card_name;
}
public String getTarget_name() {
return target_name;
}
public void setTarget_name(String target_name) {
this.target_name = target_name;
}
public String getCpc_referer() {
return cpc_referer;
}
......@@ -164,10 +180,12 @@ public class BlPreciseExposureParamsExposureCardsBean {
this.relative_position = relative_position;
}
public BlPreciseExposureParamsExposureCardsBean(String card_id, String card_content_type, String card_type, String transaction_type, String is_cpc, String cpc_referer, String absolute_position, String relative_position) {
public BlPreciseExposureParamsExposureCardsBean(String card_id, String card_content_type, String card_type, String card_name, String target_name, String transaction_type, String is_cpc, String cpc_referer, String absolute_position, String relative_position) {
this.card_id = card_id;
this.card_content_type = card_content_type;
this.card_type = card_type;
this.card_name = card_name;
this.target_name = target_name;
this.transaction_type = transaction_type;
this.is_cpc = is_cpc;
this.cpc_referer = cpc_referer;
......@@ -181,6 +199,8 @@ public class BlPreciseExposureParamsExposureCardsBean {
"card_id='" + card_id + '\'' +
", card_content_type='" + card_content_type + '\'' +
", card_type='" + card_type + '\'' +
", card_name='" + card_name + '\'' +
", target_name='" + target_name + '\'' +
", transaction_type='" + transaction_type + '\'' +
", is_cpc='" + is_cpc + '\'' +
", cpc_referer='" + cpc_referer + '\'' +
......
src/main/java/com/gmei/bean/ml/MlPreciseExposureBean.java
View file @ 0b60b567
......@@ -53,6 +53,7 @@ public class MlPreciseExposureBean {
private String card_id;
private String card_name;
private String card_content_type;
private String card_content_type_name; //匹配码表DIM_CARD_CONTENT_TYPE
private String card_type;
......@@ -94,7 +95,7 @@ public class MlPreciseExposureBean {
public MlPreciseExposureBean() {
}
public MlPreciseExposureBean(String user_id, String action, String page_code, String page_name, String tab_name, String business_id, String referrer_code, String referrer_name, String referrer_id, String card_id, String card_content_type, String card_content_type_name, String card_type, String card_type_name, String is_cpc, String cpc_referer, String transaction_type, String transaction_type_name, String filter, String query, String app_version, String current_city_id, String current_city_name, String current_province_id, String current_province_name, String current_country_id, String current_country_name, String current_region_id, String current_region_name, String app_code, String device_os_type, String device_id, String create_time_day, String gm_nginx_time_day, Integer preciseexposure_num) {
public MlPreciseExposureBean(String user_id, String action, String page_code, String page_name, String tab_name, String business_id, String referrer_code, String referrer_name, String referrer_id, String card_id, String card_name, String card_content_type, String card_content_type_name, String card_type, String card_type_name, String is_cpc, String cpc_referer, String transaction_type, String transaction_type_name, String filter, String query, String app_version, String current_city_id, String current_city_name, String current_province_id, String current_province_name, String current_country_id, String current_country_name, String current_region_id, String current_region_name, String app_code, String device_os_type, String device_id, String create_time_day, String gm_nginx_time_day, Integer preciseexposure_num) {
this.user_id = user_id;
this.action = action;
// this.action_name = action_name;
......@@ -106,6 +107,7 @@ public class MlPreciseExposureBean {
this.referrer_name = referrer_name;
this.referrer_id = referrer_id;
this.card_id = card_id;
this.card_name = card_name;
this.card_content_type = card_content_type;
this.card_content_type_name = card_content_type_name;
this.card_type = card_type;
......@@ -222,6 +224,14 @@ public class MlPreciseExposureBean {
this.card_id = card_id;
}
public String getCard_name() {
return card_name;
}
public void setCard_name(String card_name) {
this.card_name = card_name;
}
public String getCard_content_type() {
return card_content_type;
}
......@@ -446,46 +456,6 @@ public class MlPreciseExposureBean {
this.preciseexposure_num = preciseexposure_num;
}
// @Override
//// public String toString() {
//// return user_id + '\001' +
//// action + '\001' +
////// action_name + '\001' +
//// page_code + '\001' +
//// page_name + '\001' +
//// tab_name + '\001' +
//// business_id + '\001' +
//// referrer_code + '\001' +
//// referrer_name + '\001' +
//// referrer_id + '\001' +
//// card_id + '\001' +
//// card_content_type + '\001' +
//// card_content_type_name + '\001' +
//// card_type + '\001' +
//// card_type_name + '\001' +
//// is_cpc + '\001' +
//// cpc_referer + '\001' +
//// transaction_type + '\001' +
//// transaction_type_name + '\001' +
//// filter + '\001' +
//// query + '\001' +
////// app_channel + '\001' +
//// app_version + '\001' +
//// current_city_id + '\001' +
//// current_city_name + '\001' +
//// current_province_id + '\001' +
//// current_province_name + '\001' +
//// current_country_id + '\001' +
//// current_country_name + '\001' +
//// current_region_id + '\001' +
//// current_region_name + '\001' +
//// app_code + '\001' +
//// device_os_type + '\001' +
//// device_id + '\001' +
//// create_time_day + '\001' +
//// gm_nginx_time_day + '\001' +
//// preciseexposure_num;
//// }
@Override
public String toString() {
return "MlPreciseExposureBean{" +
......@@ -499,6 +469,7 @@ public class MlPreciseExposureBean {
", referrer_name='" + referrer_name + '\'' +
", referrer_id='" + referrer_id + '\'' +
", card_id='" + card_id + '\'' +
", card_name='" + card_name + '\'' +
", card_content_type='" + card_content_type + '\'' +
", card_content_type_name='" + card_content_type_name + '\'' +
", card_type='" + card_type + '\'' +
......
src/main/java/com/gmei/function/BlPreciseExposureMapFunction.java
View file @ 0b60b567
......@@ -26,7 +26,6 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
@Override
public BlPreciseExposureBean map(String string) throws Exception {
BlPreciseExposureBean blPreciseExposureBean = new BlPreciseExposureBean();
JSONObject app = null;
String gm_nginx_timestamp = null;
String create_timestamp = null;
......@@ -67,8 +66,8 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
String create_time_date = null;
String create_time_day = null;
//去除掉json中的回车、换行、制表
String jsonString = string.replaceAll("(\\\\n|\\\\r|\\\\t)", "");
//去除掉json中的回车、换行、制表、空格
String jsonString = string.replaceAll("\\\\s+|\\\\\\\\\\\\\\\\n", "");
JSONObject jsonObject = JSON.parseObject(jsonString);
......@@ -122,7 +121,14 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
JSONObject jsonCard = jsonArrayExposureCards.getJSONObject(i);
String card_id = jsonCard.getString("card_id");
String transaction_type = jsonCard.getString("transaction_type");
String card_content_type = jsonCard.getString("card_content_type");
String card_content_type = jsonCard.getString("card_content_type ");
String card_type = jsonCard.getString("card_type");
String card_name = jsonCard.getString("card_name");
String target_name = jsonCard.getString("target_name");
String is_cpc = jsonCard.getString("is_cpc");
String cpc_referer = jsonCard.getString("cpc_referer");
String absolute_posttion = jsonCard.getString("absolute_position");
String relative_position = jsonCard.getString("relative_position");
// if("service".equals(card_content_type) && "7.7.60".equals(app_version)) {
if(StringUtils.startsWithIgnoreCase(card_id, "[") && StringUtils.endsWithIgnoreCase(card_id, "]")) {
JSONArray card_id_list = JSONArray.parseArray(card_id);
......@@ -139,15 +145,28 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
cardsBean.setTransaction_type(null);
}
cardsBean.setCard_content_type(card_content_type);
cardsBean.setCard_type(jsonCard.getString("card_type"));
cardsBean.setIs_cpc(jsonCard.getString("is_cpc"));
cardsBean.setCpc_referer(jsonCard.getString("cpc_referer"));
cardsBean.setAbsolute_position(jsonCard.getString("absolute_position"));
cardsBean.setRelative_position(jsonCard.getString("relative_position"));
cardsBean.setCard_type(card_type);
cardsBean.setCard_name(card_name);
cardsBean.setTarget_name(target_name);
cardsBean.setIs_cpc(is_cpc);
cardsBean.setCpc_referer(cpc_referer);
cardsBean.setAbsolute_position(absolute_posttion);
cardsBean.setRelative_position(relative_position);
exposure_cards.add(cardsBean);
}
} else {
BlPreciseExposureParamsExposureCardsBean cardsBean = JSON.toJavaObject(jsonCard, BlPreciseExposureParamsExposureCardsBean.class);
BlPreciseExposureParamsExposureCardsBean cardsBean = new BlPreciseExposureParamsExposureCardsBean();
cardsBean.setCard_id(card_id);
cardsBean.setTransaction_type(transaction_type);
cardsBean.setCard_content_type(card_content_type);
cardsBean.setCard_type(card_type);
cardsBean.setCard_name(card_name);
cardsBean.setTarget_name(target_name);
cardsBean.setIs_cpc(is_cpc);
cardsBean.setCpc_referer(cpc_referer);
cardsBean.setAbsolute_position(absolute_posttion);
cardsBean.setRelative_position(relative_position);
// BlPreciseExposureParamsExposureCardsBean cardsBean = JSON.toJavaObject(jsonCard, BlPreciseExposureParamsExposureCardsBean.class);
exposure_cards.add(cardsBean);
}
}
......@@ -155,7 +174,6 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
// System.out.println(JSON.parseObject(params.getString("exposure_cards")).toString());
}
//DEVICE
device = jsonObject.getJSONObject("device");
if (device != null) {
......@@ -219,7 +237,6 @@ public class BlPreciseExposureMapFunction implements MapFunction<String, BlPreci
blPreciseExposureBean.setGm_nginx_time_day(gm_nginx_time_day);
blPreciseExposureBean.setCreate_time_date(create_time_date);
blPreciseExposureBean.setCreate_time_day(create_time_day);
// System.out.println(blPreciseExposureBean);
return BeanReflectUtil.setNullValue(blPreciseExposureBean);
}
}
src/main/java/com/gmei/function/MlPreciseExposureKeySelector.java
View file @ 0b60b567
......@@ -39,6 +39,7 @@ public class MlPreciseExposureKeySelector implements KeySelector<MlPreciseExposu
groupByFields.add(mlPreciseExposureBean.getReferrer_code());
groupByFields.add(mlPreciseExposureBean.getReferrer_id());
groupByFields.add(mlPreciseExposureBean.getCard_id());
groupByFields.add(mlPreciseExposureBean.getCard_name());
groupByFields.add(mlPreciseExposureBean.getCard_content_type());
groupByFields.add(mlPreciseExposureBean.getCard_type());
groupByFields.add(mlPreciseExposureBean.getIs_cpc());
......
src/main/java/com/gmei/function/MlpreciseExposureFlatMapFunction.java
View file @ 0b60b567
......@@ -47,6 +47,8 @@ public class MlpreciseExposureFlatMapFunction implements FlatMapFunction<BlPreci
String card_id = exposureCardsBean.getCard_id();
String card_content_type = exposureCardsBean.getCard_content_type();
String card_type = exposureCardsBean.getCard_type();
String card_name = exposureCardsBean.getCard_name();
String target_name = exposureCardsBean.getTarget_name();
String is_cpc = exposureCardsBean.getIs_cpc();
String cpc_referer = exposureCardsBean.getCpc_referer();
String absolute_position = exposureCardsBean.getAbsolute_position();
......@@ -106,6 +108,9 @@ public class MlpreciseExposureFlatMapFunction implements FlatMapFunction<BlPreci
mlPreciseExposureBean.setCard_id(card_id);
mlPreciseExposureBean.setCard_content_type(card_content_type);
//1.13 target_name与card_name的参数的统一
//问题描述：将target_name参数的值赋给card_name
mlPreciseExposureBean.setCard_name(card_name == null ? target_name : card_name);
mlPreciseExposureBean.setCard_type(card_type);
mlPreciseExposureBean.setIs_cpc(is_cpc);
......@@ -114,9 +119,12 @@ public class MlpreciseExposureFlatMapFunction implements FlatMapFunction<BlPreci
// mlPreciseExposureBean.setRelative_position(relative_position);
mlPreciseExposureBean.setTransaction_type(transaction_type);
mlPreciseExposureBean.setFilter(blPreciseExposureBean.getFilter());
mlPreciseExposureBean.setQuery(blPreciseExposureBean.getQuery());
// mlPreciseExposureBean.setChannel_id(blPreciseExposureBean.getApp_channel());
mlPreciseExposureBean.setApp_version(blPreciseExposureBean.getApp_version());
mlPreciseExposureBean.setCurrent_city_id(blPreciseExposureBean.getApp_current_city_id());
mlPreciseExposureBean.setApp_code(blPreciseExposureBean.getApp_code());
mlPreciseExposureBean.setDevice_os_type(blPreciseExposureBean.getDevice_os_type());
mlPreciseExposureBean.setDevice_id(blPreciseExposureBean.getDevice_id());
......@@ -126,12 +134,6 @@ public class MlpreciseExposureFlatMapFunction implements FlatMapFunction<BlPreci
//1.11 transaction_type的空值需要被修正
//问题描述：transaction_type='' 需要替换为NULL
// List<String> card_content_type_list_ch = Arrays.asList("正常搜索","指定网页","美购详情页ID","医生主页ID","秒杀专场","交易专题ID","秒杀聚合","新专题聚合","医院主页ID","福利专题列表");
List<String> card_content_type_list_ch = Arrays.asList("指定网页");
if (card_content_type_list_ch.contains(card_content_type)) {
System.out.println(mlPreciseExposureBean);
}
mlPreciseExposureBeanList.add(BeanReflectUtil.setNullValue(mlPreciseExposureBean));
}
return mlPreciseExposureBeanList;
......
src/main/java/com/gmei/jdbc/MysqlJdbcDim.java
View file @ 0b60b567
......@@ -27,7 +27,7 @@ public class MysqlJdbcDim {
// private String password = "zJnxVEhyyxeC7ciqxdMITVyWqOFc2mew";
private String driver = "com.mysql.jdbc.Driver";
private String driver = "com.mysql.cj.jdbc.Driver";
private String url;
private Connection conn;
private PreparedStatement ps;
......
src/main/java/com/gmei/jdbc/MysqlJdbcSink.java
View file @ 0b60b567
......@@ -29,7 +29,7 @@ public class MysqlJdbcSink {
// private String username = "work";
// private String password = "zJnxVEhyyxeC7ciqxdMITVyWqOFc2mew";
private String driver = "com.mysql.jdbc.Driver";
private String driver = "com.mysql.cj.jdbc.Driver";
private String url;
private Connection conn;
......
src/main/java/com/gmei/source/BlMaiDianKafkaSource.java
View file @ 0b60b567
......@@ -4,6 +4,8 @@ import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import java.nio.charset.Charset;
import java.util.Properties;
......@@ -49,7 +51,7 @@ public class BlMaiDianKafkaSource {
// props.put("auto.offset.reset", "earliest");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
FlinkKafkaConsumer011<String> myConsumer = new FlinkKafkaConsumer011<String>(topic, new SimpleStringSchema(), props);
FlinkKafkaConsumer011<String> myConsumer = new FlinkKafkaConsumer011<String>(topic, new SimpleStringSchema(Charset.forName("UTF-8")), props);
myConsumer.setStartFromGroupOffsets();//默认消费策略
// myConsumer.setStartFromEarliest();
return myConsumer;
......
src/main/java/com/gmei/sql/bl_et_mg_preciseexposure_inc_d.sql
View file @ 0b60b567
......@@ -53,19 +53,19 @@ INSERT OVERWRITE TABLE BL_ET_MG_PRECISEEXPOSURE_INC_D PARTITION (PARTITION_DAY =
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.version')) = '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.version'))) VERSION,
CASE
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.params')) = '' OR GET_JSON_OBJECT(LV.RAW, '$.params') IS NULL THEN
JSON_MAP('{}', 'string,string')
JSON_MAP('{}', 'string,string')
WHEN IS_JSON(GET_JSON_OBJECT(LV.RAW, '$.params')) = 'true' THEN
JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.params')), 'string,string')
JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.params')), 'string,string')
ELSE JSON_MAP(REGEXP_REPLACE(REGEXP_REPLACE(REGEXP_REPLACE(TRIM(GET_JSON_OBJECT(LV.RAW,'$.params')),'("){2}','"'),':",',':"",'),':"}',':""}'),'string,string')
END PARAMS,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app_session_id')) = '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.app_session_id'))) APP_SESSION_ID,
CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp')) AS DOUBLE ) AS BIGINT) GM_NGINX_TIMESTAMP,
CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at')) AS DOUBLE ) AS BIGINT) CREATE_TIMESTAMP,
CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp')) AS DOUBLE) AS BIGINT) GM_NGINX_TIMESTAMP,
CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at')) AS DOUBLE) AS BIGINT) CREATE_TIMESTAMP,
CASE
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.app')) = '' OR GET_JSON_OBJECT(LV.RAW, '$.app') IS NULL THEN
JSON_MAP('{}', 'string,string')
JSON_MAP('{}', 'string,string')
ELSE
JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app')), 'string,string')
JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app')), 'string,string')
END AS APP,
CASE
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.device')) = '' OR GET_JSON_OBJECT(LV.RAW, '$.device') IS NULL THEN
......@@ -99,17 +99,17 @@ INSERT OVERWRITE TABLE BL_ET_MG_PRECISEEXPOSURE_INC_D PARTITION (PARTITION_DAY =
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.params.query'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.params.query'))) PARAMS_QUERY,
CASE
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.grey_type')) IN ('', 'unknown') OR GET_JSON_OBJECT(LV.RAW, '$.app.grey_type') IS NULL THEN
JSON_MAP('{}', 'string,string')
JSON_MAP('{}', 'string,string')
ELSE
JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.grey_type')), 'string,string')
JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.grey_type')), 'string,string')
END APP_GREY_TYPE,
CASE
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.channel'))= '' THEN
NULL
NULL
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.channel'))= 'AppStore' THEN
'App Store'
'App Store'
ELSE
TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.channel'))
TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.channel'))
END AS APP_CHANNEL,
-- IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.channel'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.channel'))) APP_CHANNEL,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.version'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.version'))) APP_VERSION,
......@@ -117,12 +117,10 @@ INSERT OVERWRITE TABLE BL_ET_MG_PRECISEEXPOSURE_INC_D PARTITION (PARTITION_DAY =
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.current_city_id'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.current_city_id'))) APP_CURRENT_CITY_ID,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.name'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.name'))) APP_NAME,
CASE
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.user_type')) = '' OR GET_JSON_OBJECT(LV.RAW, '$.app.user_type') IS NULL THEN
JSON_MAP('{}', 'string,string')
ELSE
JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.user_type')), 'string,string')
END AS APP_USER_TYPE,
WHEN TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.user_type')) = ''
OR GET_JSON_OBJECT(LV.RAW, '$.app.user_type') IS NULL THEN JSON_MAP('{}', 'string,string')
ELSE JSON_MAP(TRIM(GET_JSON_OBJECT(LV.RAW, '$.app.user_type')), 'string,string')
END AS APP_USER_TYPE,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.is_WiFi'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.is_WiFi'))) DEVICE_IS_WIFI,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.device_type'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.device_type'))) DEVICE_TYPE,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.model'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.model'))) DEVICE_MODEL,
......@@ -144,17 +142,25 @@ INSERT OVERWRITE TABLE BL_ET_MG_PRECISEEXPOSURE_INC_D PARTITION (PARTITION_DAY =
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.manufacturer'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.manufacturer'))) DEVICE_MANUFACTURER,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.idfv'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.idfv'))) DEVICE_IDFV,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.idfa'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.device.idfa'))) DEVICE_IDFA,
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp')) AS DOUBLE ) AS BIGINT),'yyyy-MM-dd HH:mm:ss')) GM_NGINX_TIME_DATE, --服务端接收时间
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp')) AS DOUBLE ) AS BIGINT),'yyyyMMdd')) GM_NGINX_TIME_DAY, --服务端接收日期
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at')) AS DOUBLE ) AS BIGINT),'yyyy-MM-dd HH:mm:ss')) TIME_DATE, --时间戳格式化后的时间
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at')) AS DOUBLE ) AS BIGINT),'yyyyMMdd')) TIME_DAY --时间戳格式化后的日期
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp')) AS DOUBLE) AS BIGINT),'yyyy-MM-dd HH:mm:ss')) GM_NGINX_TIME_DATE, --服务端接收时间
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.gm_nginx_timestamp')) AS DOUBLE) AS BIGINT),'yyyyMMdd')) GM_NGINX_TIME_DAY, --服务端接收日期
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at')) AS DOUBLE) AS BIGINT),'yyyy-MM-dd HH:mm:ss')) TIME_DATE, --时间戳格式化后的时间
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at'))= '', NULL, FROM_UNIXTIME(CAST(CAST(TRIM(GET_JSON_OBJECT(LV.RAW, '$.create_at')) AS DOUBLE) AS BIGINT),'yyyyMMdd')) TIME_DAY, --时间戳格式化后的日期
ROW_NUMBER() OVER(PARTITION BY T.JSON) AS RANK_NUM
FROM TL.TL_LOG_PRECISE_EXPOSURE T
LATERAL VIEW JSON_TUPLE(REGEXP_REPLACE(REGEXP_REPLACE(T.JSON, '\\\\\\\\n', ''), '\\s+', ''), '@raw') LV AS RAW
WHERE T.PARTITION_DAY = '${partition_day}') T1
WHERE T.PARTITION_DAY = '${partition_day}'
AND (CASE
WHEN IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.params.page_name'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.params.page_name')))='home' AND
IF(TRIM(GET_JSON_OBJECT(LV.RAW, '$.params.is_popup'))= '', NULL, TRIM(GET_JSON_OBJECT(LV.RAW, '$.params.is_popup'))) IN (1, '1') THEN
0
ELSE
1
END) = 1) T1
LEFT JOIN (SELECT T.ID AS CITY_ID,
T.TAG_ID AS TAG_ID
FROM TL.TL_ZX_API_CITY T
WHERE T.PARTITION_DAY = '${partition_day}'
AND T.IS_ONLINE = 'true') T2
ON T1.APP_CURRENT_CITY_ID = T2.TAG_ID;
\ No newline at end of file
ON T1.APP_CURRENT_CITY_ID = T2.TAG_ID
WHERE RANK_NUM=1;
\ No newline at end of file
src/main/java/com/gmei/sql/ml_c_et_pe_preciseexposure_dimen_d.sql 0 → 100644
View file @ 0b60b567
INSERT OVERWRITE TABLE ML_C_ET_PE_PRECISEEXPOSURE_DIMEN_D PARTITION (PARTITION_DAY = '${partition_day}')
SELECT T8.JSON, --原始日志
T8.APP, --APP字段（MAP）
T8.PARAMS, --埋点参数字段(MAP)
T8.DEVICE, --埋点device字段(MAP)
T8.GM_NGINX_KEY,
T8.CREATE_TIMESTAMP, --时间戳
T8.GM_NGINX_TIMESTAMP, --服务端接收时间戳
T8.VERSION, --版本信息
T8.APP_SESSION_ID, --客户端回话id
T8.APP_SERIAL_ID, --埋点序列id
T8.APP_USER_TYPE, --app用户类型
T8.CHANNEL_ID, --渠道id
T8.CURRENT_CITY_ID, --当前城市id
T8.APP_CODE, --app名称
T8.APP_VERSION, --app版本
T8.APP_GREY_TYPE, --灰度组
T8.USER_ID, --用户id
T8.DEVICE_ID, --设备id
T8.DEVICE_MANUFACTURER, --设备制造商
T8.DEVICE_OS_TYPE, --设备类型
T8.DEVICE_MODEL, --设备型号
T8.DEVICE_SYS_VERSION, --设备系统版本
T8.DEVICE_NET_TYPE, --设备网络类型
T8.DEVICE_ANDROID_ID, --设备安卓ID
T8.DEVICE_IP, --设备IP
T8.DEVICE_IDFV, --设备IDFV
T8.DEVICE_IDFA, --设备IDFA
T8.LOCATION, --位置
T8.ACTION, --事件名称
T8.PAGE_CODE, --页面名称
T8.BUSINESS_ID, --业务ID
T8.TAB_CODE, --tab编码
T8.REFERRER_CODE, --上级页面编码
T8.REFERRER_ID, --上级页面ID
T8.UP_LOADING_TIMES, --上拉加载次数
T8.DOWN_LOADING_TIMES, --下拉加载次数
T8.UP_SLIDE_TIMES, --上拉滑动次数
T8.DOWN_SLIDE_TIMES, --下拉滑动次数
T8.IS_EXPOSURE, --是否曝光
T8.EXPOSURE_CARDS, --曝光所有卡片
T8.MODULE_ID, --模块id
T8.MODULE_TYPE, --模块类型，豆腐块才有该参数
T8.CARD_ID, --卡片ID
T8.CARD_TYPE, --卡片类型
NVL(T8.CARD_NAME,T8.QUERY) CARD_NAME, --卡片名称
T8.CARD_CONTENT_TYPE, --卡片内容类型
T8.TRANSACTION_TYPE,
T8.IS_CPC, --是否cpc计费
T8.CPC_REFERER, --cpc_referer值
T8.ABSOLUTE_POSITION, --绝对位置
T8.RELATIVE_POSITION, --相对位置
T8.FILTER, --筛选条件
NVL(T12.NEW_CODE,T8.GRID) AS PAGE_POSITION, --卡片上的业务区域
T8.RESULT_STATUS, --结果状态
T8.URL, --跳转链接
T8.CURRENT_POSITION, --点击icon在第几屏
T8.TOTAL_POSITION, --总共有几屏
T8.WIKI_WORD, --wiki卡片上的关键字
T8.GM_NGINX_TIME_DATE, --服务端接收时间
T8.GM_NGINX_TIME_DAY, --服务端接收日期
T8.CREATE_TIME_DATE, --时间戳格式化后的时间
T8.CREATE_TIME_DAY, --时间戳格式化后的日期
T8.DOCTOR_ID, --医生ID
T9.HOSPITAL_ID, --医院ID
T9.BUSINESS_PARTNER_ID, --商务ID
T10.MERCHANT_ID, --商户ID
T11.MERCHANT_DOCTOR_ID, --机构管理者ID
'pe' AS EVENT_TYPE,--事件类型
1 AS PRECISEEXPOSURE_NUM
FROM (SELECT T3.*,
CASE
WHEN T3.CARD_CONTENT_TYPE = 'service' THEN
T4.DOCTOR_ID
WHEN T3.CARD_CONTENT_TYPE = 'doctor' THEN
T3.CARD_ID
WHEN T3.CARD_CONTENT_TYPE = 'hospital' THEN
T5.DOCTOR_ID
WHEN T3.CARD_CONTENT_TYPE = 'diary' THEN
T6.DOCTOR_ID
WHEN T3.CARD_CONTENT_TYPE = 'topic' THEN
T7.DOCTOR_ID
ELSE
NULL
END AS DOCTOR_ID
FROM (SELECT T2.JSON AS JSON, --原始日志
T2.APP AS APP, --APP字段（MAP）
T2.PARAMS AS PARAMS, --埋点参数字段(MAP)
T2.DEVICE AS DEVICE, --埋点device字段(MAP)
T2.GM_NGINX_KEY AS GM_NGINX_KEY,
T2.CREATE_TIMESTAMP AS CREATE_TIMESTAMP, --时间戳
T2.GM_NGINX_TIMESTAMP AS GM_NGINX_TIMESTAMP, --服务端接收时间戳
T2.VERSION AS VERSION, --版本信息
T2.APP_SESSION_ID AS APP_SESSION_ID, --客户端回话id
T2.APP_SERIAL_ID AS APP_SERIAL_ID, --埋点序列id
T2.APP_USER_TYPE AS APP_USER_TYPE, --app用户类型
T2.APP_CHANNEL AS CHANNEL_ID, --渠道id
T2.APP_CURRENT_CITY_ID AS CURRENT_CITY_ID, --当前城市id
T2.APP_NAME AS APP_CODE, --app名称
T2.APP_VERSION AS APP_VERSION, --app版本
T2.APP_GREY_TYPE AS APP_GREY_TYPE, --灰度组
T2.USER_ID AS USER_ID, --用户id
T2.DEVICE_ID AS DEVICE_ID, --设备id
T2.DEVICE_MANUFACTURER AS DEVICE_MANUFACTURER, --设备制造商
T2.DEVICE_TYPE AS DEVICE_OS_TYPE, --设备类型
T2.DEVICE_MODEL AS DEVICE_MODEL, --设备型号
T2.DEVICE_SYS_VERSION AS DEVICE_SYS_VERSION, --设备系统版本
T2.DEVICE_NET_TYPE AS DEVICE_NET_TYPE, --设备网络类型
T2.DEVICE_ANDROID_ID AS DEVICE_ANDROID_ID, --设备安卓ID
T2.DEVICE_IP AS DEVICE_IP, --设备IP
T2.DEVICE_IDFV AS DEVICE_IDFV, --设备IDFV
T2.DEVICE_IDFA AS DEVICE_IDFA, --设备IDFA
ARRAY(T2.DEVICE_LAT, T2.DEVICE_LNG) AS LOCATION, --位置
T2.TYPE AS ACTION, --事件名称
T2.PARAMS_PAGE_NAME AS PAGE_CODE, --页面名称
T2.PARAMS_BUSINESS_ID AS BUSINESS_ID, --业务ID
T2.PARAMS_TAB_NAME AS TAB_CODE, --tab编码
T2.PARAMS_REFERRER AS REFERRER_CODE, --上级页面编码
T2.PARAMS_REFERRER_ID AS REFERRER_ID, --上级页面ID
T2.PARAMS_UP_LOADING_TIMES AS UP_LOADING_TIMES, --上拉加载次数
T2.PARAMS_DOWN_LOADING_TIMES AS DOWN_LOADING_TIMES, --下拉加载次数
T2.PARAMS_UP_SLIDE_TIMES AS UP_SLIDE_TIMES, --上拉滑动次数
T2.PARAMS_DOWN_SLIDE_TIMES AS DOWN_SLIDE_TIMES, --下拉滑动次数
T2.PARAMS_IS_EXPOSURE AS IS_EXPOSURE, --是否曝光
T2.PARAMS_EXPOSURE_CARDS AS EXPOSURE_CARDS, --曝光所有卡片
T2.MODULE_ID AS MODULE_ID, --模块id
T2.MODULE_TYPE AS MODULE_TYPE, --模块类型，豆腐块才有该参数
STRING(CARD_ID) AS CARD_ID, --卡片ID
T2.CARD_TYPE AS CARD_TYPE, --卡片类型
T2.CARD_NAME AS CARD_NAME, --卡片名称
T2.CARD_CONTENT_TYPE AS CARD_CONTENT_TYPE, --卡片内容类型
T2.TRANSACTION_TYPE_LIST [ POS ] AS TRANSACTION_TYPE,
T2.PARAMS_QUERY AS QUERY, --搜索词
T2.IS_CPC AS IS_CPC, --是否cpc计费
T2.CPC_REFERER AS CPC_REFERER, --cpc_referer值
T2.ABSOLUTE_POSITION AS ABSOLUTE_POSITION, --绝对位置
T2.RELATIVE_POSITION AS RELATIVE_POSITION, --相对位置
T2.FILTER AS FILTER, --筛选条件
T2.IN_PAGE_POS AS PAGE_POSITION, --卡片上的业务区域
T2.RESULT_STATUS AS RESULT_STATUS, --结果状态
T2.URL AS URL, --跳转链接
T2.GRID AS GRID, --那个区域（a、b、c、d，目前线上最多四张图片）
T2.CURRENT_POSITION AS CURRENT_POSITION, --点击icon在第几屏
T2.TOTAL_POSITION AS TOTAL_POSITION, --总共有几屏
T2.WIKI_WORD AS WIKI_WORD, --wiki卡片上的关键字
T2.GM_NGINX_TIME_DATE AS GM_NGINX_TIME_DATE, --服务端接收时间
T2.GM_NGINX_TIME_DAY AS GM_NGINX_TIME_DAY, --服务端接收日期
T2.CREATE_TIME_DATE AS CREATE_TIME_DATE, --时间戳格式化后的时间
T2.CREATE_TIME_DAY AS CREATE_TIME_DAY --时间戳格式化后的日期
FROM (SELECT T1.JSON,
T1.GM_NGINX_KEY,
T1.VERSION,
T1.PARAMS,
T1.APP_SESSION_ID,
T1.GM_NGINX_TIMESTAMP,
T1.CREATE_TIMESTAMP,
T1.APP,
T1.DEVICE,
T1.USER_ID,
T1.TYPE,
T1.PARAMS_DOWN_LOADING_TIMES,
T1.PARAMS_UP_LOADING_TIMES,
T1.PARAMS_DOWN_SLIDE_TIMES,
T1.PARAMS_UP_SLIDE_TIMES,
T1.PARAMS_PAGE_NAME,
T1.PARAMS_TAB_NAME,
T1.PARAMS_BUSINESS_ID,
T1.PARAMS_REFERRER,
T1.PARAMS_REFERRER_ID,
T1.PARAMS_EXPOSURE_CARDS,
T1.PARAMS_IS_EXPOSURE,
T1.PARAMS_QUERY,
T1.APP_GREY_TYPE,
T1.APP_CHANNEL,
T1.APP_VERSION,
T1.APP_SERIAL_ID,
T1.APP_CURRENT_CITY_ID,
T1.APP_NAME,
T1.APP_USER_TYPE,
T1.DEVICE_TYPE,
T1.DEVICE_MODEL,
T1.DEVICE_SYS_VERSION,
T1.DEVICE_NET_TYPE,
T1.DEVICE_ID,
T1.DEVICE_ANDROID_ID,
T1.DEVICE_LNG,
T1.DEVICE_LAT,
T1.DEVICE_IP,
T1.DEVICE_MANUFACTURER,
T1.DEVICE_IDFV,
T1.DEVICE_IDFA,
T1.GM_NGINX_TIME_DATE,
T1.GM_NGINX_TIME_DAY,
T1.CREATE_TIME_DATE,
T1.CREATE_TIME_DAY,
RESULT_CARD, --将card_id拼成数组形式['1001']
CASE
WHEN INSTR(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_id")),
'[') = 1 AND
INSTR(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_id")),
']') =
LENGTH(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_id"))) THEN
JSON_SPLIT(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_id")))
ELSE
ARRAY(IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_id")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_id"))))
END AS CARD_ID_LIST, --将transaction_type拼成数组的形式['-1']
CASE
WHEN INSTR(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.transaction_type")),
'[') = 1 AND
INSTR(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.transaction_type")),
']') =
LENGTH(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.transaction_type"))) THEN
JSON_SPLIT(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.transaction_type")))
ELSE
ARRAY(IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.transaction_type")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.transaction_type"))))
END AS TRANSACTION_TYPE_LIST, --card_content_type为'banner','icon','gadget','search_query','function_entrance'置空，切为qa替换称q_a
CASE
WHEN TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")) IN
('banner',
'static_banner' 'icon',
'gadget',
'search_query',
'function_entrance') THEN
NULL
WHEN TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")) = 'qa' THEN
'q_a'
ELSE
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")))
END AS CARD_CONTENT_TYPE, --card_content_type为'banner','icon','gadget','search_query','function_entrance' card_type取值为card_content_type
--切当card_content_type为search_query时card_type为search_word
CASE
WHEN TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")) IN
('static_banner',
'icon',
'gadget',
'function_entrance') THEN
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")))
WHEN TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")) =
'search_query' THEN
'search_word'
WHEN TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_content_type")) =
'banner' THEN
'common_banner'
WHEN TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_type")) =
'card' THEN
'common_card'
WHEN TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_type")) =
'video' THEN
'video_card'
ELSE
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_type")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_type")))
END AS CARD_TYPE, --当版本为7.7.35和7.7.36的时候相对位置转换成绝对位置
CASE
WHEN T1.DEVICE_TYPE = 'android' AND
T1.APP_VERSION IN ('7.7.35', '7.7.36') THEN
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.relative_position")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.relative_position")))
ELSE
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.absolute_position")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.absolute_position")))
END AS ABSOLUTE_POSITION, --当版本为7.7.35和7.7.36的时候绝对位置转换成相对位置
CASE
WHEN T1.DEVICE_TYPE = 'android' AND
T1.APP_VERSION IN ('7.7.35', '7.7.36') THEN
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.absolute_position")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.absolute_position")))
ELSE
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.relative_position")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.relative_position")))
END AS RELATIVE_POSITION, --result_status先取exposure_cards 再选params
CASE
WHEN IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.result_status")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.result_status"))) IS NULL AND
IF(TRIM(GET_JSON_OBJECT(TO_JSON(T1.PARAMS),
"$.result_status")) = '',
NULL,
TRIM(GET_JSON_OBJECT(TO_JSON(T1.PARAMS),
"$.result_status"))) IS NOT NULL THEN
IF(TRIM(GET_JSON_OBJECT(TO_JSON(T1.PARAMS),
"$.result_status")) = '',
NULL,
TRIM(GET_JSON_OBJECT(TO_JSON(T1.PARAMS),
"$.result_status")))
ELSE
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.result_status")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.result_status")))
END AS RESULT_STATUS, --filter和filer_f的参数统一
CASE
WHEN T1.PARAMS_FILTER IS NOT NULL AND
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.filter_f")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.filter_f"))) IS NULL THEN
T1.PARAMS_FILTER
WHEN T1.PARAMS_FILTER IS NULL AND
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.filter_f")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.filter_f"))) IS NULL THEN
NULL
ELSE
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.filter_f")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.filter_f")))
END AS FILTER, --if(trim(get_json_object(result_card,"$.filter_f"))='',NULL,trim(get_json_object(result_card,"$.filter_f"))) as filter_f,
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD, "$.is_cpc")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD, "$.is_cpc"))) AS IS_CPC,
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.cpc_referer")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.cpc_referer"))) AS CPC_REFERER,
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.in_page_pos")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.in_page_pos"))) AS IN_PAGE_POS,
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.sec_tab_name")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.sec_tab_name"))) AS SEC_TAB_NAME, --二级页面名称
--非正常卡片通用参数（banner 豆腐块 icon）
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.module_id")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.module_id"))) AS MODULE_ID, --模块id
--target_name与card_name的参数统一
CASE
WHEN IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_name ")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_name "))) IS NULL AND
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.target_name")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.target_name"))) IS NOT NULL THEN
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.target_name")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.target_name")))
WHEN IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_name ")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_name "))) IS NULL AND
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.target_name")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.target_name"))) IS NULL THEN
NULL
ELSE
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_name ")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.card_name ")))
END AS CARD_NAME,
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD, "$.url")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD, "$.url"))) AS URL, --跳转链接
--豆腐块额外参数
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.module_type")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.module_type"))) AS MODULE_TYPE, --模块类型
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD, "$.grid")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD, "$.grid"))) AS GRID, --区域 （a、b、c、d，目前线上最多四张图片）
--icon的额外参数
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.total_position")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.total_position"))) AS TOTAL_POSITION, --总共有几屏
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.current_position")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.current_position"))) AS CURRENT_POSITION, --icon在第几屏
--百科卡片独有的参数
IF(TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.wiki_word")) = '',
NULL,
TRIM(GET_JSON_OBJECT(RESULT_CARD,
"$.wiki_word"))) AS WIKI_WORD --wiki关键字
FROM (SELECT T.JSON,
T.GM_NGINX_KEY,
T.VERSION,
T.PARAMS,
T.APP_SESSION_ID,
T.GM_NGINX_TIMESTAMP,
T.CREATE_TIMESTAMP,
T.APP,
T.DEVICE,
T.USER_ID,
T.TYPE,
T.PARAMS_DOWN_LOADING_TIMES,
T.PARAMS_DOWN_SLIDE_TIMES,
T.PARAMS_UP_LOADING_TIMES,
T.PARAMS_UP_SLIDE_TIMES,
T.PARAMS_PAGE_NAME,
IF(T.PARAMS_PAGE_NAME RLIKE
'search_result_',
NULL,
T.PARAMS_TAB_NAME) AS PARAMS_TAB_NAME,
T.PARAMS_BUSINESS_ID,
T.PARAMS_REFERRER,
T.PARAMS_REFERRER_ID,
T.PARAMS_EXPOSURE_CARDS,
STRING(T.PARAMS_IS_EXPOSURE) PARAMS_IS_EXPOSURE,
T.PARAMS_FILTER,
T.PARAMS_QUERY,
T.APP_GREY_TYPE,
T.APP_CHANNEL,
T.APP_VERSION,
T.APP_SERIAL_ID,
T.APP_CURRENT_CITY_ID,
T.APP_NAME,
T.APP_USER_TYPE,
T.DEVICE_IS_WIFI,
T.DEVICE_TYPE,
T.DEVICE_MODEL,
T.DEVICE_SYS_VERSION,
T.DEVICE_NET_TYPE,
T.DEVICE_ID,
T.DEVICE_ANDROID_ID,
T.DEVICE_LNG,
T.DEVICE_LAT,
T.DEVICE_IP,
T.DEVICE_MANUFACTURER,
T.DEVICE_IDFV,
T.DEVICE_IDFA,
T.GM_NGINX_TIME_DATE,
T.GM_NGINX_TIME_DAY,
T.CREATE_TIME_DATE,
T.CREATE_TIME_DAY
FROM BL.BL_ET_MG_PRECISEEXPOSURE_INC_D T
WHERE T.PARTITION_DAY = '${partition_day}'
AND STRING(T.PARAMS_IS_EXPOSURE) = '1'
AND (CASE
WHEN ((INT(SPLIT(T.APP_VERSION, '\\.') [ 0 ]) = 7 AND
INT(SPLIT(T.APP_VERSION, '\\.') [ 1 ]) = 7 AND
INT(SPLIT(T.APP_VERSION, '\\.') [ 2 ]) >= 70) OR
(INT(SPLIT(T.APP_VERSION, '\\.') [ 0 ]) = 7 AND
INT(SPLIT(T.APP_VERSION, '\\.') [ 1 ]) = 8)) AND
T.PARAMS_PAGE_NAME = 'home' THEN
T.PARAMS_PAGE_NAME <> 'home'
WHEN T.PARAMS_PAGE_NAME = 'category' AND
(INT(SPLIT(T.APP_VERSION, '\\.') [ 0 ]) = 7 AND
INT(SPLIT(T.APP_VERSION, '\\.') [ 1 ]) < 14) THEN
T.PARAMS_PAGE_NAME <> 'category'
ELSE
1 = 1
END)) T1 LATERAL VIEW EXPLODE(T1.PARAMS_EXPOSURE_CARDS) CARD AS RESULT_CARD) T2 LATERAL VIEW POSEXPLODE(T2.CARD_ID_LIST) CARD1 AS POS,
CARD_ID) T3 --美购 card_id = service_id
LEFT JOIN (SELECT T.SERVICE_ID, T.DOCTOR_ID
FROM BL.BL_PT_SV_SERVICEINFO_D T
WHERE T.PARTITION_DAY = '${partition_day}') T4
ON T3.CARD_ID = T4.SERVICE_ID --医院 card_id=hospital_id
LEFT JOIN (SELECT T.DOCTOR_ID,
T.DOCTOR_TYPE,
T.HOSPITAL_ID,
T.BUSINESS_PARTNER_ID
FROM BL.BL_CT_DT_DOCTORINFO_D T
WHERE T.PARTITION_DAY = '${partition_day}'
AND DOCTOR_TYPE = '1') T5
ON T3.CARD_ID = T5.HOSPITAL_ID --日记本 card_id=diary_id
LEFT JOIN (SELECT T.DIARY_ID, T.ORIGINAL_DOCTOR_ID AS DOCTOR_ID
FROM BL.BL_OP_CO_DIARYINFO_D T
WHERE T.PARTITION_DAY = '${partition_day}') T6
ON T3.CARD_ID = T6.DIARY_ID --日记贴 card_id=topic_id
LEFT JOIN (SELECT T7_1.TOPIC_ID, T7_2.DOCTOR_ID
FROM (SELECT T.TOPIC_ID, T.DIARY_ID
FROM BL.BL_OP_CO_TOPICINFO_D T
WHERE T.PARTITION_DAY = '${partition_day}'
AND T.TABLE_TYPE IN ('topic', 'article')) T7_1
LEFT JOIN (SELECT T.DIARY_ID,
T.ORIGINAL_DOCTOR_ID AS DOCTOR_ID
FROM BL.BL_OP_CO_DIARYINFO_D T
WHERE T.PARTITION_DAY = '${partition_day}') T7_2
ON T7_1.DIARY_ID = T7_2.DIARY_ID) T7
ON T3.CARD_ID = T7.TOPIC_ID) T8
LEFT JOIN (SELECT T.DOCTOR_ID AS DOCTOR_ID,
T.DOCTOR_TYPE AS DOCTOR_TYPE,
T.HOSPITAL_ID AS HOSPITAL_ID,
T.BUSINESS_PARTNER_ID AS BUSINESS_PARTNER_ID
FROM BL.BL_CT_DT_DOCTORINFO_D T
WHERE T.PARTITION_DAY = '${partition_day}') T9
ON T8.DOCTOR_ID = T9.DOCTOR_ID
LEFT JOIN (SELECT T.DOCTOR_ID AS DOCTOR_ID, T.MERCHANT_ID AS MERCHANT_ID
FROM BL.BL_CT_MC_MERCHANTDOCTORRELATION_D T
WHERE T.PARTITION_DAY = '${partition_day}') T10
ON T9.DOCTOR_ID = T10.DOCTOR_ID
LEFT JOIN (SELECT T.MERCHANT_ID AS MERCHANT_ID,
T.DOCTOR_ID AS MERCHANT_DOCTOR_ID
FROM BL.BL_CT_MC_MERCHANTINFO_D T
WHERE T.PARTITION_DAY = '${partition_day}') T11
ON T10.MERCHANT_ID = T11.MERCHANT_ID
LEFT JOIN (SELECT T.CODE, T.NEW_CODE
FROM DIM.DIM_PAGE_POSITION_TRANSFORM T) T12
ON T8.PAGE_POSITION = T12.CODE;
\ No newline at end of file
src/main/java/com/gmei/streaming/PreciseExposureStreaming.java
View file @ 0b60b567
......@@ -109,9 +109,9 @@ public class PreciseExposureStreaming {
// .forRowFormat(new Path(outputPath), new SimpleStringEncoder())
// .withBucketAssigner(dayBucketAssigner)
// .build();
// mlPreciseExposureJsonDimStream.print();
mlPreciseExposureJsonDimStream.print();
// mlPreciseExposureJsonDimStream.addSink(new PrintSinkFunction<>());
mlPreciseExposureJsonDimStream.addSink(new MlPreciseExposureMysqlSink(sinkJdbcUrl, sinkTableName)).name("mlPreciseExposureSink");
// mlPreciseExposureJsonDimStream.addSink(new MlPreciseExposureMysqlSink(sinkJdbcUrl, sinkTableName)).name("mlPreciseExposureSink");
env.execute("ml_c_et_pe_preciseexposure_dimen_d_rt_test");
}
......
src/test/java/com/gmei/function/BlPreciseExposureMapFunctionTest.java
View file @ 0b60b567
......@@ -10,7 +10,9 @@ public class BlPreciseExposureMapFunctionTest {
@Test
public void testMap() {
String json = "{\"create_at\":\"1576355745\",\"gm_nginx_timestamp\":1577283386.396,\"user_id\":\"30374524\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":\"0\",\"exposure_cards\":[{\"absolute_position\":\"0\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"0\",\"card_id\":2},{\"absolute_position\":\"0\",\"target_name\":\"\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"gadget\",\"relative_position\":\"\",\"card_id\":916},{\"absolute_position\":\"1\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"1\",\"card_id\":2},{\"target_name\":\"双12大促-主会场\",\"absolute_position\":\"1\",\"card_content_type\":\"指定网页\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"relative_position\":\"\",\"in_page_pos\":\"top\",\"card_id\":null},{\"absolute_position\":\"2\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"2\",\"card_id\":2},{\"absolute_position\":\"3\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"3\",\"card_id\":2},{\"absolute_position\":\"4\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"4\",\"card_id\":2}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"is_exposure\":\"1\",\"referrer\":\"\",\"business_id\":\"\",\"referrer_id\":\"\"},\"app_session_id\":\"386FA2C1-F140-408A-BC05-7085446D6A18\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\\"report_result\\\":0,\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"post_detail\\\":\\\"0\\\",\\\"video_steep\\\":1,\\\"face_scan\\\":0}\",\"current_city_id\":\"weifang\",\"version\":\"7.19.1\",\"user_type\":{\"config_type\":1},\"serial_id\":1,\"channel\":\"AppStore\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"D4BC83F7-82DE-42A2-8B51-61841D3572B4\",\"sys_version\":\"12.4.1\",\"lng\":0,\"is_WiFi\":\"1\",\"lat\":0,\"device_id\":\"D4BC83F7-82DE-42A2-8B51-61841D3572B4\",\"idfa\":\"00000000-0000-0000-0000-000000000000\",\"ip\":\"192.168.0.101\",\"device_type\":\"ios\",\"manufacturer\":\"Apple\",\"model\":\"iPhone10,2\"},\"type\":\"page_precise_exposure\"}";
String json = "{\"create_at\":\"1577447031\",\"gm_nginx_timestamp\":1577447032.059,\"user_id\":\"\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":0,\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type \":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type \":\"新专题聚合\",\"card_id\":46},{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type \":\"指定网页\",\"absolute_position\":3},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":4,\"card_content_type \":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type \":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type \":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"齿科-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type \":\"新专题聚合\",\"card_id\":41}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"referrer_id\":\"\",\"referrer\":\"\",\"is_exposure\":\"1\",\"business_id\":\"\"},\"app_session_id\":\"168871fb-1471-43ac-b4da-e07009d88be7\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"{\\\"video_steep\\\": 1, \\\"post_detail\\\": \\\"0\\\", \\\"face_detect_result\\\": \\\"B\\\", \\\"home\\\": \\\"0\\\", \\\"face_scan\\\": 0, \\\"report_result\\\": 0}\",\"current_city_id\":\"\",\"user_type\":{\"config_type\":\"1\"},\"name\":\"gengmei_user\",\"serial_id\":31,\"channel\":\"yiba5202\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_80a2a60d59729bda\",\"sys_version\":\"8.0.0\",\"lng\":\"0.0\",\"is_WiFi\":\"0\",\"lat\":\"0.0\",\"device_id\":\"869712034003715\",\"manufacturer\":\"HUAWEI\",\"net_type\":\"4G\",\"device_type\":\"android\",\"ip\":\"10.138.25.16\",\"model\":\"LDN-AL20\"},\"type\":\"page_precise_exposure\"}";
// String json = "{\"create_at\":\"1576355745\",\"gm_nginx_timestamp\":1577283386.396,\"user_id\":\"30374524\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":\"0\",\"exposure_cards\":[{\"absolute_position\":\"0\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"0\",\"card_id\":2},{\"absolute_position\":\"0\",\"target_name\":\"\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"gadget\",\"relative_position\":\"\",\"card_id\":916},{\"absolute_position\":\"1\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"1\",\"card_id\":2},{\"target_name\":\"双12大促-主会场\",\"absolute_position\":\"1\",\"card_content_type\":\"指定网页\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"relative_position\":\"\",\"in_page_pos\":\"top\",\"card_id\":null},{\"absolute_position\":\"2\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"2\",\"card_id\":2},{\"absolute_position\":\"3\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"3\",\"card_id\":2},{\"absolute_position\":\"4\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"4\",\"card_id\":2}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"is_exposure\":\"1\",\"referrer\":\"\",\"business_id\":\"\",\"referrer_id\":\"\"},\"app_session_id\":\"386FA2C1-F140-408A-BC05-7085446D6A18\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\\"report_result\\\":0,\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"post_detail\\\":\\\"0\\\",\\\"video_steep\\\":1,\\\"face_scan\\\":0}\",\"current_city_id\":\"weifang\",\"version\":\"7.19.1\",\"user_type\":{\"config_type\":1},\"serial_id\":1,\"channel\":\"AppStore\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"D4BC83F7-82DE-42A2-8B51-61841D3572B4\",\"sys_version\":\"12.4.1\",\"lng\":0,\"is_WiFi\":\"1\",\"lat\":0,\"device_id\":\"D4BC83F7-82DE-42A2-8B51-61841D3572B4\",\"idfa\":\"00000000-0000-0000-0000-000000000000\",\"ip\":\"192.168.0.101\",\"device_type\":\"ios\",\"manufacturer\":\"Apple\",\"model\":\"iPhone10,2\"},\"type\":\"page_precise_exposure\"}";
// String json = "{\"create_at\":\"1576347089\",\"gm_nginx_timestamp\":1576347089.299,\"user_id\":\"31961883\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":5,\"exposure_cards\":{},\"tab_name\":\"service\",\"down_slide_times\":1,\"page_name\":\"category\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":\"home\",\"business_id\":\"9\",\"filter\":\"智能排序\",\"referrer_id\":\"\"},\"app_session_id\":\"0092AC31-EF72-44FF-AE1E-8BB6F8448A08\",\"app\":{\"version\":\"7.19.1\",\"grey_type\":\"{\\n \\\"report_result\\\" : 0,\\n \\\"face_detect_result\\\" : \\\"B\\\",\\n \\\"home\\\" : \\\"0\\\",\\n \\\"post_detail\\\" : \\\"0\\\",\\n \\\"video_steep\\\" : 0,\\n \\\"face_scan\\\" : 0\\n}\",\"current_city_id\":\"liuan\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":28,\"channel\":\"App Store\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"51E1C5FD-B710-4820-91A6-E54C77AA4C60\",\"sys_version\":\"12.4.1\",\"lng\":116.48697900811,\"is_WiFi\":\"1\",\"manufacturer\":\"Apple\",\"device_id\":\"CD1187A7-091C-4497-96A3-A716BE510905\",\"device_type\":\"ios\",\"ip\":\"192.168.0.101\",\"idfa\":\"CD1187A7-091C-4497-96A3-A716BE510905\",\"lat\":31.757989657248,\"model\":\"iPhone10,1\"},\"type\":\"page_precise_exposure\"}";
// String json = "{\"app\":{\"user_type\":{\"config_type\":1},\"serial_id\":5,\"name\":\"gengmei_user\",\"channel\":\"App Store\",\"version\":\"7.7.40\",\"current_city_id\":\"beijin\n" +
// "g\"},\"app_session_id\":\"805E7CE4-0FB1-44E1-B32F-837B62D11D91\",\"gm_nginx_timestamp\":1576347092.686,\"user_id\":\"29105318\",\"gm_nginx_key\":1,\"creat\n" +
......@@ -35,10 +37,10 @@ public class BlPreciseExposureMapFunctionTest {
@Test
public void testReplace() {
String json = "{\"{\\n}\"}";
String json = "{\"{\n} \t\" }";
System.out.println(json);
// String jsonOnly = json.replace("\n", "").replace("\r","");
String jsonAll = json.replaceAll("[\\n\\r\\t]", "");
String jsonAll = json.replaceAll("\\s+", "");
// System.out.println(jsonOnly);
System.out.println("------------");
System.out.println(jsonAll);
......
src/test/java/com/gmei/kafka/MyProducter.java
View file @ 0b60b567
......@@ -52,7 +52,8 @@ public class MyProducter {
// ProducerConfig config = new ProducerConfig(props);
//创建生产这对象
Producer<String, String> producer = new KafkaProducer<String, String>(props);
String json = "{\"create_at\":\"1577364648\",\"gm_nginx_timestamp\":1577364646.542,\"user_id\":\"\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":0,\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":3,\"card_content_type\":\"新专题聚合\",\"card_id\":46},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type\":\"指定网页\",\"absolute_position\":4},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type\":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type\":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type\":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":3,\"card_content_type\":\"新专题聚合\",\"card_id\":46},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type\":\"指定网页\",\"absolute_position\":4},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type\":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type\":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type\":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"referrer_id\":\"\",\"referrer\":\"\",\"is_exposure\":\"1\",\"business_id\":\"\"},\"app_session_id\":\"6d167db8-147d-4ded-b7a8-0d9f176f91a1\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"unknown\",\"current_city_id\":\"chongqing\",\"user_type\":{},\"name\":\"gengmei_user\",\"serial_id\":1,\"channel\":\"huawei\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_99a0301553dc4f12\",\"sys_version\":\"10\",\"lng\":\"106.56958\",\"is_WiFi\":\"1\",\"lat\":\"29.680635\",\"device_id\":\"864674045613095\",\"manufacturer\":\"HUAWEI\",\"net_type\":\"wifi\",\"device_type\":\"android\",\"ip\":\"192.168.0.100\",\"model\":\"VOG-AL00\"},\"type\":\"page_precise_exposure\"}";
String json = "{\"create_at\":\"1577447031\",\"gm_nginx_timestamp\":1577447032.059,\"user_id\":\"\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":0,\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type \":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type \":\"新专题聚合\",\"card_id\":46},{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":1},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type \":\"指定网页\",\"absolute_position\":3},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":4,\"card_content_type \":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type \":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type \":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"齿科-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type \":\"新专题聚合\",\"card_id\":41}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"referrer_id\":\"\",\"referrer\":\"\",\"is_exposure\":\"1\",\"business_id\":\"\"},\"app_session_id\":\"168871fb-1471-43ac-b4da-e07009d88be7\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"{\\\"video_steep\\\": 1, \\\"post_detail\\\": \\\"0\\\", \\\"face_detect_result\\\": \\\"B\\\", \\\"home\\\": \\\"0\\\", \\\"face_scan\\\": 0, \\\"report_result\\\": 0}\",\"current_city_id\":\"\",\"user_type\":{\"config_type\":\"1\"},\"name\":\"gengmei_user\",\"serial_id\":31,\"channel\":\"yiba5202\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_80a2a60d59729bda\",\"sys_version\":\"8.0.0\",\"lng\":\"0.0\",\"is_WiFi\":\"0\",\"lat\":\"0.0\",\"device_id\":\"869712034003715\",\"manufacturer\":\"HUAWEI\",\"net_type\":\"4G\",\"device_type\":\"android\",\"ip\":\"10.138.25.16\",\"model\":\"LDN-AL20\"},\"type\":\"page_precise_exposure\"}";
// String json = "{\"create_at\":\"1577364648\",\"gm_nginx_timestamp\":1577364646.542,\"user_id\":\"\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":0,\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":1,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":2,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":3,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"absolute_position\":4,\"transaction_type\":\"\",\"relative_position\":0,\"card_content_type\":\"function_entrance\",\"card_id\":2},{\"target_name\":\"\",\"relative_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"gadget\",\"card_id\":923},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":3,\"card_content_type\":\"新专题聚合\",\"card_id\":46},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type\":\"指定网页\",\"absolute_position\":4},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type\":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type\":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type\":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38},{\"target_name\":\"鼻部-双旦大促-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":3,\"card_content_type\":\"新专题聚合\",\"card_id\":46},{\"target_name\":\"12.26社区\",\"relative_position\":0,\"transaction_type\":\"operation\",\"card_type\":\"card\",\"in_page_pos\":\"top\",\"card_content_type\":\"指定网页\",\"absolute_position\":4},{\"target_name\":\"吸脂填充-双旦\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":5,\"card_content_type\":\"新专题聚合\",\"card_id\":42},{\"target_name\":\"美肤三针双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":6,\"card_content_type\":\"新专题聚合\",\"card_id\":39},{\"target_name\":\"轮廓双旦banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":7,\"card_content_type\":\"新专题聚合\",\"card_id\":36},{\"target_name\":\"12月新人专场\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"福利专题列表\",\"card_id\":6377},{\"target_name\":\"双旦大促-主会场-华西\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"福利专题列表\",\"card_id\":6565},{\"target_name\":\"眼部双旦-banner\",\"relative_position\":0,\"in_page_pos\":\"top\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"新专题聚合\",\"card_id\":38}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"referrer_id\":\"\",\"referrer\":\"\",\"is_exposure\":\"1\",\"business_id\":\"\"},\"app_session_id\":\"6d167db8-147d-4ded-b7a8-0d9f176f91a1\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"unknown\",\"current_city_id\":\"chongqing\",\"user_type\":{},\"name\":\"gengmei_user\",\"serial_id\":1,\"channel\":\"huawei\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_99a0301553dc4f12\",\"sys_version\":\"10\",\"lng\":\"106.56958\",\"is_WiFi\":\"1\",\"lat\":\"29.680635\",\"device_id\":\"864674045613095\",\"manufacturer\":\"HUAWEI\",\"net_type\":\"wifi\",\"device_type\":\"android\",\"ip\":\"192.168.0.100\",\"model\":\"VOG-AL00\"},\"type\":\"page_precise_exposure\"}";
// String json = "{\"create_at\":\"1577324083\",\"gm_nginx_timestamp\":1577324083.74,\"user_id\":\"32339790\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":13,\"exposure_cards\":[{\"absolute_position\":\"23\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"0\",\"card_id\":\"689089\"},{\"absolute_position\":\"26\",\"transaction_type\":\"\",\"relative_position\":\"3\",\"card_content_type\":\"search_query\",\"in_page_pos\":\"related_search\",\"card_id\":[\"注射美容\",\"硅胶丰额头\",\"祛颊脂垫\",\"胶原蛋白填充垫下巴\",\"面部不对称改善\",\"膨体丰额头\"]},{\"absolute_position\":\"27\",\"cpc_referer\":1,\"relative_position\":\"1\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"is_cpc\":0,\"card_content_type\":\"diary\",\"card_id\":15804523},{\"absolute_position\":\"28\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"article\",\"relative_position\":\"2\",\"card_id\":\"19514087\"},{\"absolute_position\":\"29\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"3\",\"card_id\":\"696977\"},{\"absolute_position\":\"30\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"4\",\"card_id\":\"686606\"},{\"absolute_position\":\"33\",\"cpc_referer\":1,\"relative_position\":\"4\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"is_cpc\":0,\"card_content_type\":\"diary\",\"card_id\":15965971},{\"absolute_position\":\"34\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"article\",\"relative_position\":\"2\",\"card_id\":\"19515896\"},{\"absolute_position\":\"35\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"3\",\"card_id\":\"700587\"},{\"absolute_position\":\"36\",\"transaction_type\":\"\",\"card_type\":\"card\",\"card_content_type\":\"answer\",\"relative_position\":\"4\",\"card_id\":\"700403\"},{\"absolute_position\":\"39\",\"transaction_type\":\"\",\"relative_position\":\"3\",\"card_content_type\":\"search_query\",\"in_page_pos\":\"related_search\",\"card_id\":[\"长下巴矫正\",\"激光溶脂瘦脸\",\"额头\",\"人工骨垫下巴\",\"下巴截骨前移\",\"玻尿酸丰耳垂\"]},{\"absolute_position\":\"40\",\"cpc_referer\":1,\"relative_position\":\"4\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"is_cpc\":0,\"card_content_type\":\"diary\",\"card_id\":15953555}],\"tab_name\":\"综合\",\"down_slide_times\":1,\"query\":\"瘦脸针\",\"up_loading_times\":1,\"is_exposure\":\"1\",\"referrer\":\"search_home\",\"business_id\":\"\",\"page_name\":\"search_result_more\",\"referrer_id\":\"\"},\"app_session_id\":\"5A18488E-0965-45EE-B35A-BE82266B785B\",\"app\":{\"version\":\"7.19.1\",\"grey_type\":\"{\\\"report_result\\\":0,\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"post_detail\\\":\\\"0\\\",\\\"video_steep\\\":1,\\\"face_scan\\\":0}\",\"current_city_id\":\"qingdao\",\"name\":\"gengmei_user\",\"user_type\":{\"config_type\":1},\"serial_id\":14,\"channel\":\"AppStore\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"F7B0BC42-DCDD-4AE8-A0A1-C205233A94A4\",\"sys_version\":\"12.4\",\"lng\":120.38120711884,\"is_WiFi\":\"0\",\"manufacturer\":\"Apple\",\"device_id\":\"BD96C788-434E-46EB-898C-DF2941882425\",\"device_type\":\"ios\",\"ip\":\"10.171.24.14\",\"idfa\":\"BD96C788-434E-46EB-898C-DF2941882425\",\"lat\":36.08928448005,\"model\":\"iPhone11,6\"},\"type\":\"page_precise_exposure\"}";
// String json = "{\"create_at\":\"1577275915\",\"gm_nginx_timestamp\":1577275916.509,\"user_id\":\"32368787\",\"version\":\"147\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_exposure\":\"1\",\"exposure_cards\":[{\"absolute_position\":0,\"transaction_type\":\"\",\"card_type\":\"card\",\"relative_position\":0,\"card_content_type\":\"banner\",\"card_id\":\"\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5785839\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":1,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5785492\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":2,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5742979\"},{\"cpc_referer\":\"2\",\"relative_position\":0,\"absolute_position\":3,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"service\",\"result_status\":\"1\",\"card_id\":\"5750757\"},{\"cpc_referer\":\"1\",\"relative_position\":0,\"transaction_type\":\"\",\"card_type\":\"card\",\"absolute_position\":0,\"card_content_type\":\"answer\",\"card_id\":\"332510\"},{\"cpc_referer\":\"1\",\"relative_position\":1,\"transaction_type\":\"\",\"card_type\":\"card\",\"absolute_position\":1,\"card_content_type\":\"answer\",\"card_id\":\"307094\"},{\"cpc_referer\":\"1\",\"relative_position\":2,\"is_cpc\":0,\"transaction_type\":\"-1\",\"card_type\":\"card\",\"absolute_position\":2,\"card_content_type\":\"user_post\",\"card_id\":70366}],\"tab_name\":\"综合\",\"down_slide_times\":0,\"query\":\"魔镜\",\"filter\":\"\",\"referrer_id\":\"\",\"referrer\":\"search_home\",\"up_loading_times\":0,\"business_id\":\"\",\"page_name\":\"search_result_more\"},\"app_session_id\":\"7c55ad0f-a0a8-46c8-ad21-bd9f3f12449f\",\"app\":{\"version\":\"7.19.2\",\"grey_type\":\"{\\\"video_steep\\\":0,\\\"post_detail\\\":\\\"0\\\",\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"face_scan\\\":0,\\\"report_result\\\":0}\",\"current_city_id\":\"xining\",\"user_type\":{\"config_type\":\"1\"},\"name\":\"gengmei_user\",\"serial_id\":32,\"channel\":\"guangdiantongand05\"},\"gm_nginx_key\":1,\"device\":{\"android_device_id\":\"androidid_2117151825754036\",\"sys_version\":\"9\",\"lng\":\"101.689229\",\"is_WiFi\":\"1\",\"lat\":\"36.944768\",\"device_id\":\"865787042243564\",\"manufacturer\":\"Xiaomi\",\"net_type\":\"wifi\",\"device_type\":\"android\",\"ip\":\"192.168.0.10\",\"model\":\"RedmiNote7Pro\"},\"type\":\"page_precise_exposure\"}";
// String json = "{\"create_at\":\"1576355745\",\"gm_nginx_timestamp\":1577283386.396,\"user_id\":\"30374524\",\"version\":\"110\",\"params\":{\"down_loading_times\":0,\"up_slide_times\":0,\"is_popup\":\"0\",\"exposure_cards\":[{\"absolute_position\":\"0\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"0\",\"card_id\":2},{\"absolute_position\":\"0\",\"target_name\":\"\",\"transaction_type\":\"-1\",\"card_type\":\"card\",\"card_content_type\":\"gadget\",\"relative_position\":\"\",\"card_id\":916},{\"absolute_position\":\"1\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"1\",\"card_id\":2},{\"target_name\":\"双12大促-主会场\",\"absolute_position\":\"1\",\"card_content_type\":\"指定网页\",\"transaction_type\":\"operation\",\"card_type\":\"card\",\"relative_position\":\"\",\"in_page_pos\":\"top\",\"card_id\":null},{\"absolute_position\":\"2\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"2\",\"card_id\":2},{\"absolute_position\":\"3\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"3\",\"card_id\":2},{\"absolute_position\":\"4\",\"transaction_type\":\"\",\"card_content_type\":\"function_entrance\",\"relative_position\":\"4\",\"card_id\":2}],\"tab_name\":\"\",\"down_slide_times\":0,\"page_name\":\"home\",\"up_loading_times\":0,\"is_exposure\":\"1\",\"referrer\":\"\",\"business_id\":\"\",\"referrer_id\":\"\"},\"app_session_id\":\"386FA2C1-F140-408A-BC05-7085446D6A18\",\"app\":{\"name\":\"gengmei_user\",\"grey_type\":\"{\\\"report_result\\\":0,\\\"face_detect_result\\\":\\\"B\\\",\\\"home\\\":\\\"0\\\",\\\"post_detail\\\":\\\"0\\\",\\\"video_steep\\\":1,\\\"face_scan\\\":0}\",\"current_city_id\":\"weifang\",\"version\":\"7.19.1\",\"user_type\":{\"config_type\":1},\"serial_id\":1,\"channel\":\"AppStore\"},\"gm_nginx_key\":1,\"device\":{\"idfv\":\"D4BC83F7-82DE-42A2-8B51-61841D3572B4\",\"sys_version\":\"12.4.1\",\"lng\":0,\"is_WiFi\":\"1\",\"lat\":0,\"device_id\":\"D4BC83F7-82DE-42A2-8B51-61841D3572B4\",\"idfa\":\"00000000-0000-0000-0000-000000000000\",\"ip\":\"192.168.0.101\",\"device_type\":\"ios\",\"manufacturer\":\"Apple\",\"model\":\"iPhone10,2\"},\"type\":\"page_precise_exposure\"}";
......
src/test/java/com/gmei/utils/BeanReflectUtilTest.java
View file @ 0b60b567
......@@ -8,7 +8,7 @@ public class BeanReflectUtilTest {
@Test
public void setNullValueTest() {
BlPreciseExposureParamsExposureCardsBean cardBean = new BlPreciseExposureParamsExposureCardsBean(
"123","diary","card","",null,null,"1","2"
"123","diary","card",null,null,"",null,null,"1","2"
);
try {
BeanReflectUtil.setNullValue(cardBean);
......@@ -22,7 +22,7 @@ public class BeanReflectUtilTest {
@Test
public void setFieldValueTest() {
BlPreciseExposureParamsExposureCardsBean cardBean = new BlPreciseExposureParamsExposureCardsBean(
"123","diary","card","",null,null,"1","2"
"123","diary","card",null,null,"",null,null,"1","2"
);
BeanReflectUtil.setFieldValue(cardBean, "transaction_type", "-1");
......@@ -32,7 +32,7 @@ public class BeanReflectUtilTest {
@Test
public void getFieldValueTest() throws Exception {
BlPreciseExposureParamsExposureCardsBean cardBean = new BlPreciseExposureParamsExposureCardsBean(
"123","diary","card","",null,null,"1","2"
"123","diary","card",null,null,"",null,null,"1","2"
);
String transaction_type = (String) BeanReflectUtil.getFieldValue(cardBean, "card_type");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment