Skip to content

F
flink_warehouse_rt
Commit e15a2105 authored by 刘喆's avatar 刘喆
Browse files
Options

init project flink_warehouse_rt

parent 0f62fca8
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 3588 additions and 0 deletions
pom.xml 0 → 100644
View file @ e15a2105
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.gmei.flink</groupId>
<artifactId>flink_warehouse_rt_test</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>1.9.0</flink.version>
<slf4j.version>1.7.7</slf4j.version>
<log4j.version>1.2.17</log4j.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka-0.11_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.11</artifactId>
<version>${flink.version}</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
<!-- provided在这表示此依赖只在代码编译的时候使用，运行和打包的时候不使用 -->
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-statebackend-rocksdb_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-json</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.msgpack</groupId>
<artifactId>msgpack</artifactId>
<version>0.6.11</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>1.8.8</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.62</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>24.0-jre</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.38</version>
<!--<version>8.0.12</version>-->
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.8</source> <!-- If you want to use Java 8, change this to "1.8" -->
<target>1.8</target> <!-- If you want to use Java 8, change this to "1.8" -->
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.3</version>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>com.gmei.streaming.PreciseExposureStreaming</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>assembly</goal>
</goals>
</execution>
</executions>
</plugin>
<!--使单元测试不影响项目的编译-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skip>true</skip><!--跳过单元测试-->
<!--<testFailureIgnore>true</testFailureIgnore>--><!--这个网上很多的解决方式是这个,其实这个,其实这个配置后打包还是会编译单元测试类的,只是忽略编译单元测试类的错误.-->
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
src/main/java/com/gmei/bean/bl/BlPreciseExposureAppBean.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.bl;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
/**
* ClassName: BlPreciseExposureBean
* TableName: Bl_Et_Mg_PreciseExposure_Inc_D
* Function: TODO ADD FUNCTION.
* Reason: TODO ADD REASON.
* Date: 2019/12/3 下午5:07
*
* @author liuzhe
* @since JDK 1.8
*/
/*
{
"gm_nginx_key":2,
"version":"110",
"params":{
"up_slide_times":0,
"down_loading_times":0,
"exposure_cards":{
},
"up_loading_times":0,
"is_exposure":1,
"tab_name":"精选",
"referrer_id":"",
"down_slide_times":0,
"referrer":"",
"page_name":"home",
"business_id":""
},
"app_session_id":"C3280044-5C8E-459D-AE6C-8E26BBACD6C9",
"gm_nginx_timestamp":1546307958.631,
"create_at":"1546307958",
"app":{
"channel":"AppStore",
"version":"7.7.35",
"serial_id":42,
"current_city_id":"worldwide",
"name":"gengmei_user",
"user_type":{
}
},
"device":{
"is_WiFi":"0",
"device_type":"ios",
"device_id":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"lng":0,
"lat":0,
"ip":"10.156.100.97",
"manufacturer":"Apple",
"idfa":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"idfv":"352A6D64-17CA-4520-831A-2CE9507631D8"
},
"user_id":"30864538",
"type":"home_choiceness_card_exposure"
}
*/
public class BlPreciseExposureAppBean {
@JsonProperty(value = "channel")
private String app_channel;
@JsonProperty(value = "version")
private String app_version;
@JsonProperty(value = "serial_id")
private String app_serial_id;
@JsonProperty(value = "current_city_id")
private String app_current_city_id;
@JsonProperty(value = "name")
private String app_code;
@JsonProperty(value = "user_type")
private BlPreciseExposureAppUserTypeBean app_user_type;
public BlPreciseExposureAppBean() {
}
public String getApp_channel() {
return app_channel;
}
public void setApp_channel(String app_channel) {
this.app_channel = app_channel;
}
public String getApp_version() {
return app_version;
}
public void setApp_version(String app_version) {
this.app_version = app_version;
}
public String getApp_serial_id() {
return app_serial_id;
}
public void setApp_serial_id(String app_serial_id) {
this.app_serial_id = app_serial_id;
}
public String getApp_current_city_id() {
return app_current_city_id;
}
public void setApp_current_city_id(String app_current_city_id) {
this.app_current_city_id = app_current_city_id;
}
public String getApp_code() {
return app_code;
}
public void setApp_code(String app_code) {
this.app_code = app_code;
}
public BlPreciseExposureAppUserTypeBean getApp_user_type() {
return app_user_type;
}
public void setApp_user_type(BlPreciseExposureAppUserTypeBean app_user_type) {
this.app_user_type = app_user_type;
}
public BlPreciseExposureAppBean(String app_channel, String app_version, String app_serial_id, String app_current_city_id, String app_code, BlPreciseExposureAppUserTypeBean app_user_type) {
this.app_channel = app_channel;
this.app_version = app_version;
this.app_serial_id = app_serial_id;
this.app_current_city_id = app_current_city_id;
this.app_code = app_code;
this.app_user_type = app_user_type;
}
@Override
public String toString() {
return "BlPreciseExposureAppBean{" +
"app_channel='" + app_channel + '\'' +
", app_version='" + app_version + '\'' +
", app_serial_id='" + app_serial_id + '\'' +
", app_current_city_id='" + app_current_city_id + '\'' +
", app_code='" + app_code + '\'' +
", app_user_type=" + app_user_type +
'}';
}
}
src/main/java/com/gmei/bean/bl/BlPreciseExposureAppUserTypeBean.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.bl;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
/**
* ClassName: BlPreciseExposureBean
* TableName: Bl_Et_Mg_PreciseExposure_Inc_D
* Function: TODO ADD FUNCTION.
* Reason: TODO ADD REASON.
* Date: 2019/12/3 下午5:07
*
* @author liuzhe
* @since JDK 1.8
*/
/*
{
"gm_nginx_key":2,
"version":"110",
"params":{
"up_slide_times":0,
"down_loading_times":0,
"exposure_cards":{
},
"up_loading_times":0,
"is_exposure":1,
"tab_name":"精选",
"referrer_id":"",
"down_slide_times":0,
"referrer":"",
"page_name":"home",
"business_id":""
},
"app_session_id":"C3280044-5C8E-459D-AE6C-8E26BBACD6C9",
"gm_nginx_timestamp":1546307958.631,
"create_at":"1546307958",
"app":{
"channel":"AppStore",
"version":"7.7.35",
"serial_id":42,
"current_city_id":"worldwide",
"name":"gengmei_user",
"user_type":{
}
},
"device":{
"is_WiFi":"0",
"device_type":"ios",
"device_id":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"lng":0,
"lat":0,
"ip":"10.156.100.97",
"manufacturer":"Apple",
"idfa":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"idfv":"352A6D64-17CA-4520-831A-2CE9507631D8"
},
"user_id":"30864538",
"type":"home_choiceness_card_exposure"
}
*/
public class BlPreciseExposureAppUserTypeBean {
private String config_type;
public BlPreciseExposureAppUserTypeBean() {
}
public String getConfig_type() {
return config_type;
}
public void setConfig_type(String config_type) {
this.config_type = config_type;
}
public BlPreciseExposureAppUserTypeBean(String config_type) {
this.config_type = config_type;
}
@Override
public String toString() {
return "BlPreciseExposureAppUserTypeBean{" +
"config_type='" + config_type + '\'' +
'}';
}
}
src/main/java/com/gmei/bean/bl/BlPreciseExposureBean.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/main/java/com/gmei/bean/bl/BlPreciseExposureDeviceBean.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.bl;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
import java.util.ArrayList;
import java.util.HashMap;
/**
* ClassName: BlPreciseExposureBean
* TableName: Bl_Et_Mg_PreciseExposure_Inc_D
* Function: TODO ADD FUNCTION.
* Reason: TODO ADD REASON.
* Date: 2019/12/3 下午5:07
*
* @author liuzhe
* @since JDK 1.8
*/
/*
{
"gm_nginx_key":2,
"version":"110",
"params":{
"up_slide_times":0,
"down_loading_times":0,
"exposure_cards":{
},
"up_loading_times":0,
"is_exposure":1,
"tab_name":"精选",
"referrer_id":"",
"down_slide_times":0,
"referrer":"",
"page_name":"home",
"business_id":""
},
"app_session_id":"C3280044-5C8E-459D-AE6C-8E26BBACD6C9",
"gm_nginx_timestamp":1546307958.631,
"create_at":"1546307958",
"app":{
"channel":"AppStore",
"version":"7.7.35",
"serial_id":42,
"current_city_id":"worldwide",
"name":"gengmei_user",
"user_type":{
}
},
"device":{
"is_WiFi":"0",
"device_type":"ios",
"device_id":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"lng":0,
"lat":0,
"ip":"10.156.100.97",
"manufacturer":"Apple",
"idfa":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"idfv":"352A6D64-17CA-4520-831A-2CE9507631D8"
},
"user_id":"30864538",
"type":"home_choiceness_card_exposure"
}
*/
public class BlPreciseExposureDeviceBean {
@JsonProperty(value = "is_WiFi")
private String device_is_WiFi;
@JsonProperty(value = "device_type")
private String device_os_type;
@JsonProperty(value = "model")
private String device_model;
@JsonProperty(value = "sys_version")
private String device_sys_version;
@JsonProperty(value = "net_type")
private String device_net_type;
@JsonProperty(value = "device_id")
private String device_id;
@JsonProperty(value = "android_device_id")
private String device_android_id;
@JsonProperty(value = "lng")
private String device_lng;
@JsonProperty(value = "lat")
private String device_lat;
@JsonProperty(value = "ip")
private String device_ip;
@JsonProperty(value = "manufacturer")
private String device_manufacturer;
@JsonProperty(value = "idfv")
private String device_idfv;
@JsonProperty(value = "idfa")
private String device_idfa;
public BlPreciseExposureDeviceBean() {
}
public BlPreciseExposureDeviceBean(String device_is_WiFi, String device_os_type, String device_model, String device_sys_version, String device_net_type, String device_id, String device_android_id, String device_lng, String device_lat, String device_ip, String device_manufacturer, String device_idfv, String device_idfa) {
this.device_is_WiFi = device_is_WiFi;
this.device_os_type = device_os_type;
this.device_model = device_model;
this.device_sys_version = device_sys_version;
this.device_net_type = device_net_type;
this.device_id = device_id;
this.device_android_id = device_android_id;
this.device_lng = device_lng;
this.device_lat = device_lat;
this.device_ip = device_ip;
this.device_manufacturer = device_manufacturer;
this.device_idfv = device_idfv;
this.device_idfa = device_idfa;
}
public String getDevice_is_WiFi() {
return device_is_WiFi;
}
public void setDevice_is_WiFi(String device_is_WiFi) {
this.device_is_WiFi = device_is_WiFi;
}
public String getDevice_os_type() {
return device_os_type;
}
public void setDevice_os_type(String device_os_type) {
this.device_os_type = device_os_type;
}
public String getDevice_model() {
return device_model;
}
public void setDevice_model(String device_model) {
this.device_model = device_model;
}
public String getDevice_sys_version() {
return device_sys_version;
}
public void setDevice_sys_version(String device_sys_version) {
this.device_sys_version = device_sys_version;
}
public String getDevice_net_type() {
return device_net_type;
}
public void setDevice_net_type(String device_net_type) {
this.device_net_type = device_net_type;
}
public String getDevice_id() {
return device_id;
}
public void setDevice_id(String device_id) {
this.device_id = device_id;
}
public String getDevice_android_id() {
return device_android_id;
}
public void setDevice_android_id(String device_android_id) {
this.device_android_id = device_android_id;
}
public String getDevice_lng() {
return device_lng;
}
public void setDevice_lng(String device_lng) {
this.device_lng = device_lng;
}
public String getDevice_lat() {
return device_lat;
}
public void setDevice_lat(String device_lat) {
this.device_lat = device_lat;
}
public String getDevice_ip() {
return device_ip;
}
public void setDevice_ip(String device_ip) {
this.device_ip = device_ip;
}
public String getDevice_manufacturer() {
return device_manufacturer;
}
public void setDevice_manufacturer(String device_manufacturer) {
this.device_manufacturer = device_manufacturer;
}
public String getDevice_idfv() {
return device_idfv;
}
public void setDevice_idfv(String device_idfv) {
this.device_idfv = device_idfv;
}
public String getDevice_idfa() {
return device_idfa;
}
public void setDevice_idfa(String device_idfa) {
this.device_idfa = device_idfa;
}
@Override
public String toString() {
return "BlPreciseExposureDeviceBean{" +
"device_is_WiFi='" + device_is_WiFi + '\'' +
", device_os_type='" + device_os_type + '\'' +
", device_model='" + device_model + '\'' +
", device_sys_version='" + device_sys_version + '\'' +
", device_net_type='" + device_net_type + '\'' +
", device_id='" + device_id + '\'' +
", device_android_id='" + device_android_id + '\'' +
", device_lng='" + device_lng + '\'' +
", device_lat='" + device_lat + '\'' +
", device_ip='" + device_ip + '\'' +
", device_manufacturer='" + device_manufacturer + '\'' +
", device_idfv='" + device_idfv + '\'' +
", device_idfa='" + device_idfa + '\'' +
'}';
}
}
src/main/java/com/gmei/bean/bl/BlPreciseExposureParamsBean.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.bl;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
/**
* ClassName: BlPreciseExposureBean
* TableName: Bl_Et_Mg_PreciseExposure_Inc_D
* Function: TODO ADD FUNCTION.
* Reason: TODO ADD REASON.
* Date: 2019/12/3 下午5:07
*
* @author liuzhe
* @since JDK 1.8
*/
/*
{
"gm_nginx_key":2,
"version":"110",
"params":{
"up_slide_times":0,
"down_loading_times":0,
"exposure_cards":{
},
"up_loading_times":0,
"is_exposure":1,
"tab_name":"精选",
"referrer_id":"",
"down_slide_times":0,
"referrer":"",
"page_name":"home",
"business_id":""
},
"app_session_id":"C3280044-5C8E-459D-AE6C-8E26BBACD6C9",
"gm_nginx_timestamp":1546307958.631,
"create_at":"1546307958",
"app":{
"channel":"AppStore",
"version":"7.7.35",
"serial_id":42,
"current_city_id":"worldwide",
"name":"gengmei_user",
"user_type":{
}
},
"device":{
"is_WiFi":"0",
"device_type":"ios",
"device_id":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"lng":0,
"lat":0,
"ip":"10.156.100.97",
"manufacturer":"Apple",
"idfa":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"idfv":"352A6D64-17CA-4520-831A-2CE9507631D8"
},
"user_id":"30864538",
"type":"home_choiceness_card_exposure"
}
*/
public class BlPreciseExposureParamsBean {
private Integer down_loading_times;
private Integer down_slide_times;
private Integer up_loading_times;
private Integer up_slide_times;
@JsonProperty(value = "page_name")
private String page_code;
private String tab_name;
private String business_id;
@JsonProperty(value = "referrer")
private String referrer_code;
private String referrer_id;
private BlPreciseExposureParamsExposureCardsBean exposure_cards;
private String is_exposure;
private String filter;
private String query;
public BlPreciseExposureParamsBean() {
}
public Integer getDown_loading_times() {
return down_loading_times;
}
public void setDown_loading_times(Integer down_loading_times) {
this.down_loading_times = down_loading_times;
}
public Integer getDown_slide_times() {
return down_slide_times;
}
public void setDown_slide_times(Integer down_slide_times) {
this.down_slide_times = down_slide_times;
}
public Integer getUp_loading_times() {
return up_loading_times;
}
public void setUp_loading_times(Integer up_loading_times) {
this.up_loading_times = up_loading_times;
}
public Integer getUp_slide_times() {
return up_slide_times;
}
public void setUp_slide_times(Integer up_slide_times) {
this.up_slide_times = up_slide_times;
}
public String getPage_code() {
return page_code;
}
public void setPage_code(String page_code) {
this.page_code = page_code;
}
public String getTab_name() {
return tab_name;
}
public void setTab_name(String tab_name) {
this.tab_name = tab_name;
}
public String getBusiness_id() {
return business_id;
}
public void setBusiness_id(String business_id) {
this.business_id = business_id;
}
public String getReferrer_code() {
return referrer_code;
}
public void setReferrer_code(String referrer_code) {
this.referrer_code = referrer_code;
}
public String getReferrer_id() {
return referrer_id;
}
public void setReferrer_id(String referrer_id) {
this.referrer_id = referrer_id;
}
public BlPreciseExposureParamsExposureCardsBean getExposure_cards() {
return exposure_cards;
}
public void setExposure_cards(BlPreciseExposureParamsExposureCardsBean exposure_cards) {
this.exposure_cards = exposure_cards;
}
public String getIs_exposure() {
return is_exposure;
}
public void setIs_exposure(String is_exposure) {
this.is_exposure = is_exposure;
}
public String getFilter() {
return filter;
}
public void setFilter(String filter) {
this.filter = filter;
}
public String getQuery() {
return query;
}
public void setQuery(String query) {
this.query = query;
}
public BlPreciseExposureParamsBean(Integer down_loading_times, Integer down_slide_times, Integer up_loading_times, Integer up_slide_times, String page_code, String tab_name, String business_id, String referrer_code, String referrer_id, BlPreciseExposureParamsExposureCardsBean exposure_cards, String is_exposure, String filter, String query) {
this.down_loading_times = down_loading_times;
this.down_slide_times = down_slide_times;
this.up_loading_times = up_loading_times;
this.up_slide_times = up_slide_times;
this.page_code = page_code;
this.tab_name = tab_name;
this.business_id = business_id;
this.referrer_code = referrer_code;
this.referrer_id = referrer_id;
this.exposure_cards = exposure_cards;
this.is_exposure = is_exposure;
this.filter = filter;
this.query = query;
}
@Override
public String toString() {
return "BlPreciseExposureParamsBean{" +
"down_loading_times=" + down_loading_times +
", down_slide_times=" + down_slide_times +
", up_loading_times=" + up_loading_times +
", up_slide_times=" + up_slide_times +
", page_code='" + page_code + '\'' +
", tab_name='" + tab_name + '\'' +
", business_id='" + business_id + '\'' +
", referrer_code='" + referrer_code + '\'' +
", referrer_id='" + referrer_id + '\'' +
", exposure_cards=" + exposure_cards +
", is_exposure='" + is_exposure + '\'' +
", filter='" + filter + '\'' +
", query='" + query + '\'' +
'}';
}
}
src/main/java/com/gmei/bean/bl/BlPreciseExposureParamsExposureCardsBean.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.bl;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
/**
* ClassName: BlPreciseExposureBean
* TableName: Bl_Et_Mg_PreciseExposure_Inc_D
* Function: TODO ADD FUNCTION.
* Reason: TODO ADD REASON.
* Date: 2019/12/3 下午5:07
*
* @author liuzhe
* @since JDK 1.8
*/
/*
{
"gm_nginx_key":2,
"version":"110",
"params":{
"up_slide_times":0,
"down_loading_times":0,
"exposure_cards":{
},
"up_loading_times":0,
"is_exposure":1,
"tab_name":"精选",
"referrer_id":"",
"down_slide_times":0,
"referrer":"",
"page_name":"home",
"business_id":""
},
"app_session_id":"C3280044-5C8E-459D-AE6C-8E26BBACD6C9",
"gm_nginx_timestamp":1546307958.631,
"create_at":"1546307958",
"app":{
"channel":"AppStore",
"version":"7.7.35",
"serial_id":42,
"current_city_id":"worldwide",
"name":"gengmei_user",
"user_type":{
}
},
"device":{
"is_WiFi":"0",
"device_type":"ios",
"device_id":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"lng":0,
"lat":0,
"ip":"10.156.100.97",
"manufacturer":"Apple",
"idfa":"DE8EA66A-BDE9-47CD-9795-24E444F5BC17",
"idfv":"352A6D64-17CA-4520-831A-2CE9507631D8"
},
"user_id":"30864538",
"type":"home_choiceness_card_exposure"
}
*/
/*
get_json_object(result_card,"$.card_id") as card_id, --卡片id
get_json_object(result_card,"$.card_content_type") as card_content_type, --卡片内容类型
get_json_object(result_card,"$.card_type") as card_type, --卡片类型
get_json_object(result_card,"$.transaction_type") as transaction_type, --卡片业务类型
get_json_object(result_card,"$.absolute_position") as absolute_position, --绝对位置
get_json_object(result_card,"$.relative_position") as relative_position, --相对位置
get_json_object(result_card,"$.filter_f") as filter_f,
get_json_object(result_card,"$.is_cpc") as is_cpc, --是否为cpc收费卡片
get_json_object(result_card,"$.cpc_referer") as cpc_referer, --cpc_referer
get_json_object(result_card,"$.in_page_pose ") as in_page_pose,
get_json_object(result_card,"$.result_status") as result_status, --结果状态
get_json_object(result_card,"$.sec_tab_name") as sec_tab_name, --二级页面名称
--非正常卡片通用参数（banner 豆腐块 icon）
get_json_object(result_card,"$.module_id") as module_id, --模块id
get_json_object(result_card,"$.card_name ") as card_name, --卡片名字
get_json_object(result_card,"$.url") as url, --跳转链接
--豆腐块额外参数
get_json_object(result_card,"$.module_type") as module_type, --模块类型
get_json_object(result_card,"$.grid") as grid, --区域 （a、b、c、d，目前线上最多四张图片）
--icon的额外参数
get_json_object(result_card,"$.total_position") as total_position,--总共有几屏
get_json_object(result_card,"$.current_position") as current_position --icon在第几屏
*/
public class BlPreciseExposureParamsExposureCardsBean {
private String card_id;
private String card_content_type;
private String card_type;
private String transaction_type;
private String is_cpc;
private String cpc_referer;
private String absolute_position;
private String relative_position;
public BlPreciseExposureParamsExposureCardsBean() {
}
public String getCard_type() {
return card_type;
}
public void setCard_type(String card_type) {
this.card_type = card_type;
}
public String getCpc_referer() {
return cpc_referer;
}
public void setCpc_referer(String cpc_referer) {
this.cpc_referer = cpc_referer;
}
public String getCard_id() {
return card_id;
}
public void setCard_id(String card_id) {
this.card_id = card_id;
}
public String getCard_content_type() {
return card_content_type;
}
public void setCard_content_type(String card_content_type) {
this.card_content_type = card_content_type;
}
public String getIs_cpc() {
return is_cpc;
}
public void setIs_cpc(String is_cpc) {
this.is_cpc = is_cpc;
}
public String getAbsolute_position() {
return absolute_position;
}
public void setAbsolute_position(String absolute_position) {
this.absolute_position = absolute_position;
}
public String getTransaction_type() {
return transaction_type;
}
public void setTransaction_type(String transaction_type) {
this.transaction_type = transaction_type;
}
public String getRelative_position() {
return relative_position;
}
public void setRelative_position(String relative_position) {
this.relative_position = relative_position;
}
public BlPreciseExposureParamsExposureCardsBean(String card_id, String card_content_type, String card_type, String transaction_type, String is_cpc, String cpc_referer, String absolute_position, String relative_position) {
this.card_id = card_id;
this.card_content_type = card_content_type;
this.card_type = card_type;
this.transaction_type = transaction_type;
this.is_cpc = is_cpc;
this.cpc_referer = cpc_referer;
this.absolute_position = absolute_position;
this.relative_position = relative_position;
}
@Override
public String toString() {
return "BlPreciseExposureParamsExposureCardsBean{" +
"card_id='" + card_id + '\'' +
", card_content_type='" + card_content_type + '\'' +
", card_type='" + card_type + '\'' +
", transaction_type='" + transaction_type + '\'' +
", is_cpc='" + is_cpc + '\'' +
", cpc_referer='" + cpc_referer + '\'' +
", absolute_position='" + absolute_position + '\'' +
", relative_position='" + relative_position + '\'' +
'}';
}
}
src/main/java/com/gmei/bean/dim/DimCardContentType.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.dim;
/**
* ClassName: DimCardContentType
* Function:
* Reason: 卡片内容类型DIM_CARD_CONTENT_TYPE
* Date: 2019/12/6 下午8:20
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimCardContentType {
private String code; //编码
private String pk; //主键
private String name; //名称
private String memo; //说明
private String new_code; //转换后编码
private Integer oid; //排序
public DimCardContentType() {
}
public DimCardContentType(String code, String pk, String name, String memo, String new_code, Integer oid) {
this.code = code;
this.pk = pk;
this.name = name;
this.memo = memo;
this.new_code = new_code;
this.oid = oid;
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getPk() {
return pk;
}
public void setPk(String pk) {
this.pk = pk;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getMemo() {
return memo;
}
public void setMemo(String memo) {
this.memo = memo;
}
public String getNew_code() {
return new_code;
}
public void setNew_code(String new_code) {
this.new_code = new_code;
}
public Integer getOid() {
return oid;
}
public void setOid(Integer oid) {
this.oid = oid;
}
@Override
public String toString() {
return "DimCity{" +
"code='" + code + '\'' +
", pk='" + pk + '\'' +
", name='" + name + '\'' +
", memo='" + memo + '\'' +
", new_code='" + new_code + '\'' +
", oid=" + oid +
'}';
}
}
src/main/java/com/gmei/bean/dim/DimCity.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.dim;
/**
* ClassName: DimCity
* Function:
* Reason: 城市码表DIM_CITY
* Date: 2019/12/6 下午8:20
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimCity {
private String code; //城市编码
private String pk; //城市TAGID
private String name; //城市名称
private String memo; //城市说明
private String parent_province_code; //省份编码
private String parent_province_pk; //省份TAGID
private String parent_province_name; //省份名称
private String parent_province_memo; //省份说明
private String parent_country_code; //国家编码
private String parent_country_pk; //国家TAGID
private String parent_country_name; //国家名称
private String parent_country_memo; //国家说明
private String parent_region_code; //区域编码
private String parent_region_pk; //区域ID
private String parent_region_name; //区域名称
private String parent_region_memo; //区域说明
private Integer oid; //排序
public DimCity() {
}
public DimCity(String code, String pk, String name, String memo, String parent_province_code, String parent_province_pk, String parent_province_name, String parent_province_memo, String parent_country_code, String parent_country_pk, String parent_country_name, String parent_country_memo, String parent_region_code, String parent_region_pk, String parent_region_name, String parent_region_memo, Integer oid) {
this.code = code;
this.pk = pk;
this.name = name;
this.memo = memo;
this.parent_province_code = parent_province_code;
this.parent_province_pk = parent_province_pk;
this.parent_province_name = parent_province_name;
this.parent_province_memo = parent_province_memo;
this.parent_country_code = parent_country_code;
this.parent_country_pk = parent_country_pk;
this.parent_country_name = parent_country_name;
this.parent_country_memo = parent_country_memo;
this.parent_region_code = parent_region_code;
this.parent_region_pk = parent_region_pk;
this.parent_region_name = parent_region_name;
this.parent_region_memo = parent_region_memo;
this.oid = oid;
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getPk() {
return pk;
}
public void setPk(String pk) {
this.pk = pk;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getMemo() {
return memo;
}
public void setMemo(String memo) {
this.memo = memo;
}
public String getParent_province_code() {
return parent_province_code;
}
public void setParent_province_code(String parent_province_code) {
this.parent_province_code = parent_province_code;
}
public String getParent_province_pk() {
return parent_province_pk;
}
public void setParent_province_pk(String parent_province_pk) {
this.parent_province_pk = parent_province_pk;
}
public String getParent_province_name() {
return parent_province_name;
}
public void setParent_province_name(String parent_province_name) {
this.parent_province_name = parent_province_name;
}
public String getParent_province_memo() {
return parent_province_memo;
}
public void setParent_province_memo(String parent_province_memo) {
this.parent_province_memo = parent_province_memo;
}
public String getParent_country_code() {
return parent_country_code;
}
public void setParent_country_code(String parent_country_code) {
this.parent_country_code = parent_country_code;
}
public String getParent_country_pk() {
return parent_country_pk;
}
public void setParent_country_pk(String parent_country_pk) {
this.parent_country_pk = parent_country_pk;
}
public String getParent_country_name() {
return parent_country_name;
}
public void setParent_country_name(String parent_country_name) {
this.parent_country_name = parent_country_name;
}
public String getParent_country_memo() {
return parent_country_memo;
}
public void setParent_country_memo(String parent_country_memo) {
this.parent_country_memo = parent_country_memo;
}
public String getParent_region_code() {
return parent_region_code;
}
public void setParent_region_code(String parent_region_code) {
this.parent_region_code = parent_region_code;
}
public String getParent_region_pk() {
return parent_region_pk;
}
public String getParent_region_name() {
return parent_region_name;
}
public void setParent_region_name(String parent_region_name) {
this.parent_region_name = parent_region_name;
}
public String getParent_region_memo() {
return parent_region_memo;
}
public void setParent_region_memo(String parent_region_memo) {
this.parent_region_memo = parent_region_memo;
}
@Override
public String toString() {
return "DimCity{" +
"code='" + code + '\'' +
", pk='" + pk + '\'' +
", name='" + name + '\'' +
", memo='" + memo + '\'' +
", parent_province_code='" + parent_province_code + '\'' +
", parent_province_pk='" + parent_province_pk + '\'' +
", parent_province_name='" + parent_province_name + '\'' +
", parent_province_memo='" + parent_province_memo + '\'' +
", parent_country_code='" + parent_country_code + '\'' +
", parent_country_pk='" + parent_country_pk + '\'' +
", parent_country_name='" + parent_country_name + '\'' +
", parent_country_memo='" + parent_country_memo + '\'' +
", parent_region_code='" + parent_region_code + '\'' +
", parent_region_pk='" + parent_region_pk + '\'' +
", parent_region_name='" + parent_region_name + '\'' +
", parent_region_memo='" + parent_region_memo + '\'' +
", oid=" + oid +
'}';
}
public void setParent_region_pk(String parent_region_pk) {
this.parent_region_pk = parent_region_pk;
}
public Integer getOid() {
return oid;
}
public void setOid(Integer oid) {
this.oid = oid;
}
}
src/main/java/com/gmei/bean/dim/DimPageType.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.dim;
/**
* ClassName: DimPageType
* Function:
* Reason: 页面类型码表DIM_PAGE_TYPE
* Date: 2019/12/6 下午8:20
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimPageType {
private String code; //页面编码
private String pk; //页面主键
private String name; //页面名称
private String memo; //页面说明
private String parent_app_code; //上级APP编码
private String parent_app_pk; //上级APP主键
private String parent_app_name; //上级APP名称
private String parent_app_memo; //上级APP说明
private String type_s_code; //页面分类小类编码
private String type_s_name; //页面分类小类名称
private String type_m_code; //页面分类中类编码
private String type_m_name; //页面分类中类名称
private String type_l_code; //页面分类大类编码
private String type_l_name; //页面分类大类名称
private Integer oid; //排序
public DimPageType() {
}
public DimPageType(String code, String pk, String name, String memo, String parent_app_code, String parent_app_pk, String parent_app_name, String parent_app_memo, String type_s_code, String type_s_name, String type_m_code, String type_m_name, String type_l_code, String type_l_name, Integer oid) {
this.code = code;
this.pk = pk;
this.name = name;
this.memo = memo;
this.parent_app_code = parent_app_code;
this.parent_app_pk = parent_app_pk;
this.parent_app_name = parent_app_name;
this.parent_app_memo = parent_app_memo;
this.type_s_code = type_s_code;
this.type_s_name = type_s_name;
this.type_m_code = type_m_code;
this.type_m_name = type_m_name;
this.type_l_code = type_l_code;
this.type_l_name = type_l_name;
this.oid = oid;
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getPk() {
return pk;
}
public void setPk(String pk) {
this.pk = pk;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getMemo() {
return memo;
}
public void setMemo(String memo) {
this.memo = memo;
}
public String getParent_app_code() {
return parent_app_code;
}
public void setParent_app_code(String parent_app_code) {
this.parent_app_code = parent_app_code;
}
public String getParent_app_pk() {
return parent_app_pk;
}
public void setParent_app_pk(String parent_app_pk) {
this.parent_app_pk = parent_app_pk;
}
public String getParent_app_name() {
return parent_app_name;
}
public void setParent_app_name(String parent_app_name) {
this.parent_app_name = parent_app_name;
}
public String getParent_app_memo() {
return parent_app_memo;
}
public void setParent_app_memo(String parent_app_memo) {
this.parent_app_memo = parent_app_memo;
}
public String getType_s_code() {
return type_s_code;
}
public void setType_s_code(String type_s_code) {
this.type_s_code = type_s_code;
}
public String getType_s_name() {
return type_s_name;
}
public void setType_s_name(String type_s_name) {
this.type_s_name = type_s_name;
}
public String getType_m_code() {
return type_m_code;
}
public void setType_m_code(String type_m_code) {
this.type_m_code = type_m_code;
}
public String getType_m_name() {
return type_m_name;
}
public void setType_m_name(String type_m_name) {
this.type_m_name = type_m_name;
}
public String getType_l_code() {
return type_l_code;
}
public void setType_l_code(String type_l_code) {
this.type_l_code = type_l_code;
}
public String getType_l_name() {
return type_l_name;
}
public void setType_l_name(String type_l_name) {
this.type_l_name = type_l_name;
}
public Integer getOid() {
return oid;
}
public void setOid(Integer oid) {
this.oid = oid;
}
@Override
public String toString() {
return "DimPageType{" +
"code='" + code + '\'' +
", pk='" + pk + '\'' +
", name='" + name + '\'' +
", memo='" + memo + '\'' +
", parent_app_code='" + parent_app_code + '\'' +
", parent_app_pk='" + parent_app_pk + '\'' +
", parent_app_name='" + parent_app_name + '\'' +
", parent_app_memo='" + parent_app_memo + '\'' +
", type_s_code='" + type_s_code + '\'' +
", type_s_name='" + type_s_name + '\'' +
", type_m_code='" + type_m_code + '\'' +
", type_m_name='" + type_m_name + '\'' +
", type_l_code='" + type_l_code + '\'' +
", type_l_name='" + type_l_name + '\'' +
", oid=" + oid +
'}';
}
}
src/main/java/com/gmei/bean/dim/DimTable.java 0 → 100644
View file @ e15a2105
package com.gmei.bean.dim;
/**
* ClassName: DimTable
* Function:
* Reason: 标准码表共用
* include DIM_CARD_TYPE、DIM_TRANSACTION_TYPE、DIM_CARD_CONTENT_TYPE
* Date: 2019/12/6 下午8:20
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimTable {
private String code; //编码
private String pk; //主键
private String name; //名称
private String memo; //说明
private Integer oid; //排序
public DimTable() {
}
public DimTable(String code, String pk, String name, String memo, Integer oid) {
this.code = code;
this.pk = pk;
this.name = name;
this.memo = memo;
this.oid = oid;
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getPk() {
return pk;
}
public void setPk(String pk) {
this.pk = pk;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getMemo() {
return memo;
}
public void setMemo(String memo) {
this.memo = memo;
}
public Integer getOid() {
return oid;
}
public void setOid(Integer oid) {
this.oid = oid;
}
@Override
public String toString() {
return "DimCity{" +
"code='" + code + '\'' +
", pk='" + pk + '\'' +
", name='" + name + '\'' +
", memo='" + memo + '\'' +
", oid=" + oid +
'}';
}
}
src/main/java/com/gmei/bean/ml/MlPreciseExposureBean.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/main/java/com/gmei/cache/CacheServiceAbstract.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import java.util.Map;
import java.util.concurrent.Callable;
/**
* ClassName: CacheServiceAbstract
* Function:
* Reason: 缓存抽象类
* Date: 2019/12/10 下午7:01
*
* @author liuzhe
* @since JDK 1.8
*/
public abstract class CacheServiceAbstract<K, V> {
abstract V getValue(K key, Callable<V> callable);
// abstract V getValue(K key, Callable<Optional<V>> callable);
abstract void clearCache();
abstract Long cacheSize();
abstract void putValue(K key, V value);
abstract void invalidate(Object key);
// abstract void putValues(ArrayList<V> ts);
abstract void putValues(Map<K, V> map);
}
src/main/java/com/gmei/cache/DimCardContentTypeDao.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import com.gmei.bean.dim.DimCardContentType;
import com.gmei.jdbc.MysqlJdbcDim;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
/**
* ClassName: DimCardContentTypeDao
* Function:
* Reason: dim_card_content_type卡片内容类型数据库操作类
* Date: 2019/12/10 下午7:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimCardContentTypeDao {
private MysqlJdbcDim mysqlJdbcDim;
private ResultSet resultSet;
private String sql;
private String dimJdbcUrl;
public DimCardContentTypeDao() {
}
public DimCardContentTypeDao(String dimJdbcUrl) {
this.dimJdbcUrl = dimJdbcUrl;
this.mysqlJdbcDim = MysqlJdbcDim.getInstance(dimJdbcUrl);
}
/**
* Function: findAllDimCardContentType
* Reason: 查询所有卡片内容类型信息
* Date: 2019/12/25 下午5:32
*
* @author liuzhe
* @since JDK 1.8
*/
public ArrayList<DimCardContentType> findAllDimCardContentType(String tableName) {
sql = String.format(
"select code, pk, name, memo, new_code" +
" from %s", tableName);
ArrayList<DimCardContentType> dimCardContentTypeList = new ArrayList<>();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
DimCardContentType dimCardContentType = new DimCardContentType();
dimCardContentType.setCode(resultSet.getString(1));
dimCardContentType.setPk(resultSet.getString(2));
dimCardContentType.setName(resultSet.getString(3));
dimCardContentType.setMemo(resultSet.getString(4));
dimCardContentType.setNew_code(resultSet.getString(5));
dimCardContentTypeList.add(dimCardContentType);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return dimCardContentTypeList;
}
/**
* Function: findDimCardContentTypeByCode
* Reason: 通过卡片内容类型编码查询信息
* Date: 2019/12/25 下午5:34
*
* @author liuzhe
* @since JDK 1.8
*/
public DimCardContentType findDimCardContentTypeByCode(String tableName, String code){
if (code == null || code.trim().length() == 0) {
return null;
} else {
code = code.trim();
}
String sql = String.format(
"select code, pk, name, memo, new_code " +
" from %s " +
" where code = '%s'", tableName, code);
DimCardContentType dimCardContentType = new DimCardContentType();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
dimCardContentType.setCode(code);
dimCardContentType.setPk(resultSet.getString(2));
dimCardContentType.setName(resultSet.getString(3));
dimCardContentType.setMemo(resultSet.getString(4));
dimCardContentType.setNew_code(resultSet.getString(5));
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
mysqlJdbcDim.close(null, null, resultSet);
}
return dimCardContentType;
}
}
\ No newline at end of file
src/main/java/com/gmei/cache/DimCityDao.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import com.gmei.bean.dim.DimCity;
import com.gmei.jdbc.MysqlJdbcDim;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
/**
* ClassName: DimCityDao
* Function:
* Reason: dim_city城市码表数据库操作类
* Date: 2019/12/10 下午7:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimCityDao {
private MysqlJdbcDim mysqlJdbcDim;
private ResultSet resultSet;
private String sql;
private Connection conn;
private String dimJdbcUrl;
public DimCityDao() {
}
public DimCityDao(String dimJdbcUrl) {
this.dimJdbcUrl = dimJdbcUrl;
this.mysqlJdbcDim = MysqlJdbcDim.getInstance(dimJdbcUrl);
}
/**
* Function: findDimCityByCode
* Reason: 根据城市ID查询城市信息
* Date: 2019/12/7 下午7:56
*
* @author liuzhe
* @since JDK 1.8
*/
public DimCity findDimCityByCode(String tableName, String code){
if (code == null || code.trim().length() == 0) {
return null;
} else {
code = code.trim();
}
mysqlJdbcDim.getConnection();
String sql = String.format(
"select code, pk, name, memo, " +
" parent_province_code, parent_province_pk, parent_province_memo," +
" parent_country_code, parent_country_pk, parent_country_name," +
" parent_region_code, parent_region_pk, parent_region_name " +
" from %s " +
" where code = '%s'", tableName, code);
// ArrayList<DimCity> dimCityList = new ArrayList<>();
DimCity dimCity = new DimCity();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
dimCity.setCode(code);
dimCity.setPk(resultSet.getString(2));
dimCity.setName(resultSet.getString(3));
dimCity.setMemo(resultSet.getString(4));
dimCity.setParent_province_code(resultSet.getString(5));
dimCity.setParent_province_pk(resultSet.getString(6));
dimCity.setParent_province_name(resultSet.getString(7));
dimCity.setParent_country_code(resultSet.getString(8));
dimCity.setParent_country_pk(resultSet.getString(9));
dimCity.setParent_country_name(resultSet.getString(10));
dimCity.setParent_region_code(resultSet.getString(11));
dimCity.setParent_region_pk(resultSet.getString(12));
dimCity.setParent_region_name(resultSet.getString(13));
// dimCityList.add(dimCity);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
mysqlJdbcDim.close(null, null, resultSet);
}
return dimCity;
}
/**
* Function: findDimCityByPk
* Reason: 根据城市TAGID查询城市信息
* Date: 2019/12/7 下午7:56
*
* @author liuzhe
* @since JDK 1.8
*/
public DimCity findDimCityByPk(String tableName, String pk){
if (pk == null || pk.trim().length() == 0) {
return null;
}
pk = pk.trim();
mysqlJdbcDim.getConnection();
String sql = String.format(
"select code, pk, name, memo, " +
" parent_province_code, parent_province_pk, parent_province_memo," +
" parent_country_code, parent_country_pk, parent_country_name," +
" parent_region_code, parent_region_pk, parent_region_name " +
" from %s " +
" where pk = '%s'", tableName, pk);
// ArrayList<DimCity> dimCityList = new ArrayList<>();
DimCity dimCity = new DimCity();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
dimCity.setCode(resultSet.getString(1));
dimCity.setPk(pk);
dimCity.setName(resultSet.getString(3));
dimCity.setMemo(resultSet.getString(4));
dimCity.setParent_province_code(resultSet.getString(5));
dimCity.setParent_province_pk(resultSet.getString(6));
dimCity.setParent_province_name(resultSet.getString(7));
dimCity.setParent_country_code(resultSet.getString(8));
dimCity.setParent_country_pk(resultSet.getString(9));
dimCity.setParent_country_name(resultSet.getString(10));
dimCity.setParent_region_code(resultSet.getString(11));
dimCity.setParent_region_pk(resultSet.getString(12));
dimCity.setParent_region_name(resultSet.getString(13));
// dimCityList.add(dimCity);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
mysqlJdbcDim.close(conn, null, resultSet);
}
return dimCity;
}
/**
* Function: findAllDimCity
* Reason: 查询所有城市信息
* Date: 2019/12/25 下午5:21
*
* @author liuzhe
* @since JDK 1.8
*/
public ArrayList<DimCity> findAllDimCity(String tableName) {
mysqlJdbcDim.getConnection();
sql = String.format(
"select code, pk, name, memo, " +
" parent_province_code, parent_province_pk, parent_province_memo," +
" parent_country_code, parent_country_pk, parent_country_name," +
" parent_region_code, parent_region_pk, parent_region_name " +
" from %s", tableName);
ArrayList<DimCity> dimCityList = new ArrayList<>();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
DimCity dimCity = new DimCity();
dimCity.setCode(resultSet.getString(1));
dimCity.setPk(resultSet.getString(2));
dimCity.setName(resultSet.getString(3));
dimCity.setMemo(resultSet.getString(4));
dimCity.setParent_province_code(resultSet.getString(5));
dimCity.setParent_province_pk(resultSet.getString(6));
dimCity.setParent_province_name(resultSet.getString(7));
dimCity.setParent_country_code(resultSet.getString(8));
dimCity.setParent_country_pk(resultSet.getString(9));
dimCity.setParent_country_name(resultSet.getString(10));
dimCity.setParent_region_code(resultSet.getString(11));
dimCity.setParent_region_pk(resultSet.getString(12));
dimCity.setParent_region_name(resultSet.getString(13));
dimCityList.add(dimCity);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return dimCityList;
}
}
src/main/java/com/gmei/cache/DimPageTypeDao.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import com.gmei.bean.dim.DimPageType;
import com.gmei.jdbc.MysqlJdbcDim;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
/**
* ClassName: DimPageTypeDao
* Function:
* Reason: dim_page_type页面名称标准化就三级分类码表数据库操作类
* Date: 2019/12/10 下午7:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimPageTypeDao {
private MysqlJdbcDim mysqlJdbcDim;
private ResultSet resultSet;
private String sql;
private Connection conn;
private String dimJdbcUrl;
public DimPageTypeDao() {
}
public DimPageTypeDao(String dimJdbcUrl) {
this.dimJdbcUrl = dimJdbcUrl;
this.mysqlJdbcDim = MysqlJdbcDim.getInstance(dimJdbcUrl);
}
/**
* Function: findAllDimPageType
* Reason: 查询所有页面信息（gengmei_user）
* Date: 2019/12/25 下午5:23
*
* @author liuzhe
* @since JDK 1.8
*/
public ArrayList<DimPageType> findAllDimPageType(String tableName) {
mysqlJdbcDim.getConnection();
sql = String.format(
"select code, pk, name, memo, " +
" parent_app_code, parent_app_pk, parent_app_name, " +
" type_s_code, type_s_name, " +
" type_m_code, type_m_name, " +
" type_l_code, type_l_name " +
" from %s" +
" where parent_app_code = 'gengmei_user'", tableName);
ArrayList<DimPageType> dimPageTypeList = new ArrayList<>();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
DimPageType dimPageType = new DimPageType();
dimPageType.setCode(resultSet.getString(1));
dimPageType.setPk(resultSet.getString(2));
dimPageType.setName(resultSet.getString(3));
dimPageType.setMemo(resultSet.getString(4));
dimPageType.setParent_app_code(resultSet.getString(5));
dimPageType.setParent_app_pk(resultSet.getString(6));
dimPageType.setParent_app_name(resultSet.getString(7));
dimPageType.setType_s_code(resultSet.getString(8));
dimPageType.setType_s_name(resultSet.getString(9));
dimPageType.setType_m_code(resultSet.getString(10));
dimPageType.setType_m_name(resultSet.getString(11));
dimPageType.setType_l_code(resultSet.getString(12));
dimPageType.setType_l_name(resultSet.getString(13));
dimPageTypeList.add(dimPageType);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
mysqlJdbcDim.close(null, null, resultSet);
}
return dimPageTypeList;
}
/**
* Function: findDimPageTypeByCode
* Reason: 根据页面编码查询页面信息（gengmei_user）
* Date: 2019/12/25 下午5:23
*
* @author liuzhe
* @since JDK 1.8
*/
public DimPageType findDimPageTypeByCode(String tableName, String code){
if (code == null || code.trim().length() == 0) {
return null;
} else {
code = code.trim();
}
mysqlJdbcDim.getConnection();
String sql = String.format(
"select code, pk, name, memo, " +
" parent_app_code, parent_app_pk, parent_app_name, " +
" type_s_code, type_s_name, " +
" type_m_code, type_m_name, " +
" type_l_code, type_l_name " +
" from %s" +
" where parent_app_code = 'gengmei_user'" +
" and code = '%s'", tableName, code);
// ArrayList<DimPageType> dimPageTypeList = new ArrayList<>();
DimPageType dimPageType = new DimPageType();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
dimPageType.setCode(code);
dimPageType.setPk(resultSet.getString(2));
dimPageType.setName(resultSet.getString(3));
dimPageType.setMemo(resultSet.getString(4));
dimPageType.setParent_app_code(resultSet.getString(5));
dimPageType.setParent_app_pk(resultSet.getString(6));
dimPageType.setParent_app_name(resultSet.getString(7));
dimPageType.setType_s_code(resultSet.getString(8));
dimPageType.setType_s_name(resultSet.getString(9));
dimPageType.setType_m_code(resultSet.getString(10));
dimPageType.setType_m_name(resultSet.getString(11));
dimPageType.setType_l_code(resultSet.getString(12));
dimPageType.setType_l_name(resultSet.getString(13));
// dimPageTypeList.add(dimPageType);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
mysqlJdbcDim.close(null, null, resultSet);
}
return dimPageType;
}
}
src/main/java/com/gmei/cache/DimTableCallable.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import com.gmei.bean.dim.DimCity;
import com.gmei.utils.DimTableList;
import java.util.concurrent.Callable;
/**
* ClassName: DimTableCallable
* Function:
* Reason: 通用Dim码表多线程调用类
* Date: 2019/12/6 下午9:05t
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimTableCallable<V> implements Callable<V> {
private String code;
private String tableName;
private String dimJdbcUrl;
public DimTableCallable(String code, String tableName, String dimJdbcUrl) {
this.code = code;
this.tableName = tableName;
this.dimJdbcUrl = dimJdbcUrl;
}
public DimTableCallable() {}
@Override
public V call() {
// try {
if(code == null) {
switch(DimTableList.getDimTableByName(tableName)) {
case DIM_CITY:
DimCityDao dimCityDao = new DimCityDao(dimJdbcUrl);
return (V) dimCityDao.findAllDimCity(tableName);
case DIM_CARD_CONTENT_TYPE:
DimCardContentTypeDao dimCardContentTypeDao = new DimCardContentTypeDao(dimJdbcUrl);
return (V) dimCardContentTypeDao.findAllDimCardContentType(tableName);
case DIM_TRANSACTION_TYPE:
DimTableDao dimTransactionTypeDao = new DimTableDao(dimJdbcUrl);
return (V) dimTransactionTypeDao.findAllDimTable(tableName);
case DIM_PAGE_TYPE:
DimPageTypeDao dimPageTypeDao = new DimPageTypeDao(dimJdbcUrl);
return (V) dimPageTypeDao.findAllDimPageType(tableName);
case DIM_CARD_TYPE:
DimTableDao dimCardTypeDao = new DimTableDao(dimJdbcUrl);
return (V) dimCardTypeDao.findAllDimTable(tableName);
default:
return null;
}
} else {
switch(DimTableList.getDimTableByName(tableName)) {
case DIM_CITY:
DimCityDao dimCityDao = new DimCityDao(dimJdbcUrl);
DimCity dimCity = dimCityDao.findDimCityByCode(tableName, code);
if (dimCity == null) {
dimCity = dimCityDao.findDimCityByPk(tableName, code);
}
return (V) dimCity;
case DIM_CARD_CONTENT_TYPE:
DimCardContentTypeDao dimCardContentTypeDao = new DimCardContentTypeDao(dimJdbcUrl);
return (V) dimCardContentTypeDao.findDimCardContentTypeByCode(tableName, code);
case DIM_TRANSACTION_TYPE:
DimTableDao dimTransactionTypeDao = new DimTableDao(dimJdbcUrl);
return (V) dimTransactionTypeDao.findDimTableByCode(tableName, code);
case DIM_PAGE_TYPE:
DimPageTypeDao dimPageTypeDao = new DimPageTypeDao(dimJdbcUrl);
return (V) dimPageTypeDao.findDimPageTypeByCode(tableName, code);
case DIM_CARD_TYPE:
DimTableDao dimCardTypeDao = new DimTableDao(dimJdbcUrl);
return (V) dimCardTypeDao.findDimTableByCode(tableName, code);
default:
return null;
}
}
// } catch (Exception e) {
// e.printStackTrace();
// return null;
// }
}
}
src/main/java/com/gmei/cache/DimTableDao.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import com.gmei.bean.dim.DimTable;
import com.gmei.jdbc.MysqlJdbcDim;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
/**
* ClassName: DimTableDao
* Function:
* Reason: 通用Dim码表
* Date: 2019/12/10 下午7:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class DimTableDao {
private MysqlJdbcDim mysqlJdbcDim;
private ResultSet resultSet;
private String sql;
private Connection conn;
private String dimJdbcUrl;
public DimTableDao() {
}
public DimTableDao(String dimJdbcUrl) {
this.dimJdbcUrl = dimJdbcUrl;
this.mysqlJdbcDim = MysqlJdbcDim.getInstance(dimJdbcUrl);
}
/**
* Function: findAllDimTable
* Reason: 查询所有码表信息
* Date: 2019/12/25 下午5:26
*
* @author liuzhe
* @since JDK 1.8
*/
public ArrayList<DimTable> findAllDimTable(String tableName) {
mysqlJdbcDim.getConnection();
sql = String.format("select code, pk, name, memo from %s", tableName);
ArrayList<DimTable> dimTableList = new ArrayList<>();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
DimTable dim_table = new DimTable();
dim_table.setCode(resultSet.getString(1));
dim_table.setPk(resultSet.getString(2));
dim_table.setName(resultSet.getString(3));
dim_table.setMemo(resultSet.getString(4));
dimTableList.add(dim_table);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
mysqlJdbcDim.close(null, null, resultSet);
}
return dimTableList;
}
/**
* Function: findDimTableByCode
* Reason: 根据编码查询码表信息
* Date: 2019/12/25 下午5:26
*
* @author liuzhe
* @since JDK 1.8
*/
public DimTable findDimTableByCode(String tableName, String code){
if (code == null || code.trim().length() == 0) {
return null;
} else {
code = code.trim();
}
mysqlJdbcDim.getConnection();
String sql = String.format("select code, pk, name, memo from %s where code = '%s'", tableName, code);
DimTable dimTable = new DimTable();
resultSet = mysqlJdbcDim.query(sql);
try {
while (resultSet.next()) {
dimTable.setCode(code);
dimTable.setPk(resultSet.getString(2));
dimTable.setName(resultSet.getString(3));
dimTable.setMemo(resultSet.getString(4));
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
mysqlJdbcDim.close(null, null, resultSet);
}
return dimTable;
}
}
src/main/java/com/gmei/cache/MlPreciseExposureDao.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import com.gmei.bean.ml.MlPreciseExposureBean;
import com.gmei.jdbc.MysqlJdbcSink;
import java.util.ArrayList;
import java.util.List;
/**
* ClassName: MlPreciseExposureDao
* Function:
* Reason: ml_c_et_pe_preciseexposure_dimen_d_rt数据下发操作类
* Date: 2019/12/19 上午11:35
*
* @author liuzhe
* @since JDK 1.8
*/
public class MlPreciseExposureDao {
private MysqlJdbcSink mysqlJdbcSink;
private String sql;
private String sinkJdbcUrl;
private String sinkTableName;
public MlPreciseExposureDao(String sinkJdbcUrl, String sinkTableName) {
this.sinkJdbcUrl = sinkJdbcUrl;
this.sinkTableName = sinkTableName;
this.mysqlJdbcSink = MysqlJdbcSink.getInstance(sinkJdbcUrl);
}
/**
* Function: insertMlPreciseExposure
* Reason: 向表中插入数据
* Date: 2019/12/25 下午5:38
*
* @author liuzhe
* @since JDK 1.8
*/
public void insertMlPreciseExposure(MlPreciseExposureBean mlPreciseExposureBean) throws Exception {
sql = "insert into " + sinkTableName + "\n" +
" (action,\n" +
" app_code,\n" +
" page_code,\n" +
" page_name,\n" +
" tab_name,\n" +
" business_id,\n" +
" referrer_code,\n" +
" referrer_name,\n" +
" referrer_id,\n" +
" card_id,\n" +
" card_content_type,\n" +
" card_content_type_name,\n" +
" card_type,\n" +
" card_type_name,\n" +
" is_cpc,\n" +
" cpc_referer,\n" +
" transaction_type,\n" +
" transaction_type_name,\n" +
" filter,\n" +
" query,\n" +
" app_version,\n" +
" user_id,\n" +
" device_id,\n" +
" device_os_type,\n" +
" current_city_id,\n" +
" current_city_name,\n" +
" current_province_id,\n" +
" current_province_name,\n" +
" current_country_id,\n" +
" current_country_name,\n" +
" current_region_id,\n" +
" current_region_name,\n" +
" create_time_day,\n" +
" gm_nginx_time_day,\n" +
" preciseexposure_num)\n" +
"values\n" +
" (?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?,\n" +
" ?)";
List<Object> params = new ArrayList<Object>();
params.add(mlPreciseExposureBean.getAction());
// params.add(mlPreciseExposureBean.getAction_name());
params.add(mlPreciseExposureBean.getApp_code());
params.add(mlPreciseExposureBean.getPage_code());
params.add(mlPreciseExposureBean.getPage_name());
params.add(mlPreciseExposureBean.getTab_name());
params.add(mlPreciseExposureBean.getBusiness_id());
params.add(mlPreciseExposureBean.getReferrer_code());
params.add(mlPreciseExposureBean.getReferrer_name());
params.add(mlPreciseExposureBean.getReferrer_id());
params.add(mlPreciseExposureBean.getCard_id());
params.add(mlPreciseExposureBean.getCard_content_type());
params.add(mlPreciseExposureBean.getCard_content_type_name());
params.add(mlPreciseExposureBean.getCard_type());
params.add(mlPreciseExposureBean.getCard_type_name());
params.add(mlPreciseExposureBean.getIs_cpc());
params.add(mlPreciseExposureBean.getCpc_referer());
params.add(mlPreciseExposureBean.getTransaction_type());
params.add(mlPreciseExposureBean.getTransaction_type_name());
params.add(mlPreciseExposureBean.getFilter());
params.add(mlPreciseExposureBean.getQuery());
params.add(mlPreciseExposureBean.getApp_version());
params.add(mlPreciseExposureBean.getUser_id());
params.add(mlPreciseExposureBean.getDevice_id());
params.add(mlPreciseExposureBean.getDevice_os_type());
params.add(mlPreciseExposureBean.getCurrent_city_id());
params.add(mlPreciseExposureBean.getCurrent_city_name());
params.add(mlPreciseExposureBean.getCurrent_province_id());
params.add(mlPreciseExposureBean.getCurrent_province_name());
params.add(mlPreciseExposureBean.getCurrent_country_id());
params.add(mlPreciseExposureBean.getCurrent_country_name());
params.add(mlPreciseExposureBean.getCurrent_region_id());
params.add(mlPreciseExposureBean.getCurrent_region_name());
params.add(mlPreciseExposureBean.getCreate_time_day());
params.add(mlPreciseExposureBean.getGm_nginx_time_day());
params.add(mlPreciseExposureBean.getPreciseexposure_num());
// System.out.println(params.toString());
mysqlJdbcSink.update(sql, params);
}
}
src/main/java/com/gmei/cache/SimpleCacheService.java 0 → 100644
View file @ e15a2105
package com.gmei.cache;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
/**
* Created by allen on 2017/4/6.
*/
public class SimpleCacheService<K, V> extends CacheServiceAbstract<K, V> implements Serializable {
private Cache<K, V> cache = null;
private int maximumSize = 1000;
private int expireAfterWrite = 1;
public SimpleCacheService() {
createCache();
}
public SimpleCacheService(int maximumSize, int expireAfterWrite) {
this.maximumSize = maximumSize;
this.expireAfterWrite = expireAfterWrite;
createCache();
}
public void createCache() {
cache = CacheBuilder
.newBuilder()
.maximumSize(maximumSize)
.expireAfterWrite(expireAfterWrite, TimeUnit.HOURS)
.build();
}
@Override
public V getValue(final K key, final Callable<V> callable) {
if(key == null) {
return null;
}
try {
return cache.get(key, callable);
} catch (ExecutionException e) {
e.printStackTrace();
return null;
}
}
public Map<K, V> asMap(Cache cache) {
Map cacheMap = cache.asMap();
return cacheMap;
}
@Override
public void putValue(K key, V value) {
cache.put(key, value);
}
@Override
public void invalidate(Object key) {
cache.invalidate(key);
}
@Override
public void clearCache(){
this.cache.invalidateAll();
}
@Override
public Long cacheSize() {
return this.cache.size();
}
@Override
public void putValues(Map<K, V> map) {
cache.putAll(map);
// cache = new CacheLoader() {
// @Override
// public Object load(Object o) throws Exception {
// for (Object o : objects) {
// cache.put(o.getcode(), value);
// }
// return o;
// }
// };
// new CacheLoader<Long, Optional<Object>>() {
//
// @Override
// public Optional<Object> load(Long storeId) throws Exception {
// Object s = null;
//
// List<Object> results = null;
// if (results != null && results.size() > 0) {
// s = results.get(0);
// }
//
// return Optional.ofNullable(s);
// }
//
// }
}
public Cache<K, V> getCache() {
return cache;
}
// public void putValues(Map<K, V> map) {
//
// }
}
src/main/java/com/gmei/function/BlMaiDianOutputSelector.java 0 → 100644
View file @ e15a2105
package com.gmei.function;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import java.util.ArrayList;
import java.util.List;
/**
* ClassName: BlMaiDianOutputSelector
* Function:
* Reason: BL层数据选择器
* Date: 2019/12/7 上午11:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class BlMaiDianOutputSelector implements OutputSelector<String> {
@Override
public Iterable<String> select(String s) {
List<String> output = new ArrayList<String>();
JSONObject jsonObject = JSON.parseObject(s);
String action = jsonObject.getString("type");
// System.out.println(s);
if ("home_choiceness_card_exposure".equals(action) || "page_precise_exposure".equals(action)) {
output.add("et_pe");
} else if ("page_view".equals(action)) {
output.add("et_le");
} else if (StringUtils.containsIgnoreCase(action,"click")) {
output.add("et_ck");
} else {
output.add("et_ot");
}
return output;
}
}
src/main/java/com/gmei/function/BlPreciseExposureFilterFunction.java 0 → 100644
View file @ e15a2105
package com.gmei.function;
import com.gmei.bean.bl.BlPreciseExposureBean;
import org.apache.flink.api.common.functions.FilterFunction;
import java.util.Arrays;
import java.util.List;
/**
* ClassName: BlPreciseExposureFilterFunction
* Function:
* Reason: BL层数据筛选器
* Date: 2019/12/7 上午11:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class BlPreciseExposureFilterFunction implements FilterFunction<BlPreciseExposureBean> {
@Override
public boolean filter(BlPreciseExposureBean blPreciseExposureBean) throws Exception {
String app_code = blPreciseExposureBean.getApp_code();
String is_exposure = blPreciseExposureBean.getIs_exposure();
String is_popup = blPreciseExposureBean.getIs_popup();
String app_version = blPreciseExposureBean.getApp_version();
String page_code = blPreciseExposureBean.getPage_code();
Boolean flag = true;
// System.out.println(blPreciseExposureBean);
//清洗掉app_code不等于gengmei_user与is_exposure不等于1的
//1.5 Home页精准曝光触发两次
//问题描述：在7.7.70','7.7.71','7.7.72','7.7.75','7.7.76','7.8.0','7.8.1' (7.7.70--7.9.0)版本中的首页精准曝光会触发两次，经确认，需要去除page_name='home'
//1.6 品类聚合页的精准曝光问题
//问题描述：品类聚合页的精准曝光没做，但是数据中却有精准曝光、而且有问题，因此过滤掉，目前还不能确定什么版本可以解决，先暂时根据page_name过滤掉page_name='category'
//1.15 当首页精准曝光是通过弹窗触发或者点击push的时候，会有额外的精准曝光数据，需要去掉这部分数据
List<String> app_version_list = Arrays.asList("7.7.70","7.7.71","7.7.72","7.7.75","7.7.76","7.8.0","7.8.1");
String[] version = app_version.split("\\.");
if(version.length != 3) {
flag = false;
// return false;
} else if(!"gengmei_user".equals(app_code) || !"1".equals(is_exposure)) {
flag = false;
// return false;
} else if(app_version_list.contains(app_version) && "home".equals(page_code)) {
flag = false;
// return false;
} else if("category".equals(page_code) && "7".equals(version[0]) && Integer.parseInt(version[1]) < 14 ) {
flag = false;
// return false;
} else if("home".equals(page_code) && "1".equals(is_popup)) {
flag = false;
}
// System.out.println(flag);
return flag;
}
}
src/main/java/com/gmei/function/DimRichAsyncFunction.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/main/java/com/gmei/function/MlPreciseExposureKeySelector.java 0 → 100644
View file @ e15a2105
package com.gmei.function;
import com.gmei.bean.ml.MlPreciseExposureBean;
import org.apache.flink.api.java.functions.KeySelector;
import java.util.ArrayList;
/**
* ClassName: MlPreciseExposureKeySelector
* Function:
* Reason: ML层数据聚合分组器
* Date: 2019/12/7 上午11:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class MlPreciseExposureKeySelector implements KeySelector<MlPreciseExposureBean, Object> {
@Override
public Object getKey(MlPreciseExposureBean mlPreciseExposureBean) throws Exception {
ArrayList list = groupByFields(mlPreciseExposureBean);
return list;
}
/**
* Function: groupByFields
* Reason: 分组字段（GROUP BY）
* Date: 2019/12/25 下午5:17
*
* @author liuzhe
* @since JDK 1.8
*/
public ArrayList groupByFields(MlPreciseExposureBean mlPreciseExposureBean) {
ArrayList groupByFields = new ArrayList();
groupByFields.add(mlPreciseExposureBean.getUser_id());
groupByFields.add(mlPreciseExposureBean.getAction());
groupByFields.add(mlPreciseExposureBean.getApp_code());
groupByFields.add(mlPreciseExposureBean.getTab_name());
groupByFields.add(mlPreciseExposureBean.getBusiness_id());
groupByFields.add(mlPreciseExposureBean.getReferrer_code());
groupByFields.add(mlPreciseExposureBean.getReferrer_id());
groupByFields.add(mlPreciseExposureBean.getCard_id());
groupByFields.add(mlPreciseExposureBean.getCard_content_type());
groupByFields.add(mlPreciseExposureBean.getCard_type());
groupByFields.add(mlPreciseExposureBean.getIs_cpc());
groupByFields.add(mlPreciseExposureBean.getCpc_referer());
// groupByFields.add(mlPreciseExposureBean.getAbsolute_position());
// groupByFields.add(mlPreciseExposureBean.getRelative_position());
groupByFields.add(mlPreciseExposureBean.getTransaction_type());
groupByFields.add(mlPreciseExposureBean.getFilter());
groupByFields.add(mlPreciseExposureBean.getQuery());
// groupByFields.add(mlPreciseExposureBean.getChannel_id());
groupByFields.add(mlPreciseExposureBean.getApp_version());
groupByFields.add(mlPreciseExposureBean.getCurrent_city_id());
groupByFields.add(mlPreciseExposureBean.getApp_code());
groupByFields.add(mlPreciseExposureBean.getDevice_os_type());
groupByFields.add(mlPreciseExposureBean.getDevice_id());
groupByFields.add(mlPreciseExposureBean.getCreate_time_day());
groupByFields.add(mlPreciseExposureBean.getGm_nginx_time_day());
return groupByFields;
}
}
src/main/java/com/gmei/function/MlpreciseExposureFlatMapFunction.java 0 → 100644
View file @ e15a2105
package com.gmei.function;
import com.gmei.bean.bl.BlPreciseExposureBean;
import com.gmei.bean.bl.BlPreciseExposureParamsExposureCardsBean;
import com.gmei.bean.ml.MlPreciseExposureBean;
import com.gmei.utils.BeanReflectUtil;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.util.Collector;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* ClassName: MlpreciseExposureFlatMapFunction
* Function:
* Reason: ML层数据清洗转换器（一对多）
* Date: 2019/12/7 上午11:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class MlpreciseExposureFlatMapFunction implements FlatMapFunction<BlPreciseExposureBean, MlPreciseExposureBean> {
@Override
public void flatMap(BlPreciseExposureBean blPreciseExposureBean, Collector<MlPreciseExposureBean> collector) throws Exception {
ArrayList<MlPreciseExposureBean> mlPreciseExposureBeanList = flatmapBlPreciseExposure(blPreciseExposureBean);
// System.out.println("MlpreciseExposureFlatMapFunction" + blPreciseExposureBean);
for (MlPreciseExposureBean mlPreciseExposureBean : mlPreciseExposureBeanList) {
collector.collect(mlPreciseExposureBean);
}
}
/**
* Function: flatmapBlPreciseExposure
* Reason: 数据爆炸（LATERAL VIEW EXPLODE）
* Date: 2019/12/25 下午5:14
*
* @author liuzhe
* @since JDK 1.8
*/
public ArrayList<MlPreciseExposureBean> flatmapBlPreciseExposure(BlPreciseExposureBean blPreciseExposureBean) throws Exception {
ArrayList<BlPreciseExposureParamsExposureCardsBean> exposure_cards = blPreciseExposureBean.getExposure_cards();
String app_version = blPreciseExposureBean.getApp_version();
String device_os_type = blPreciseExposureBean.getDevice_os_type();
ArrayList<MlPreciseExposureBean> mlPreciseExposureBeanList = new ArrayList<>();
for(BlPreciseExposureParamsExposureCardsBean exposureCardsBean: exposure_cards) {
String card_id = exposureCardsBean.getCard_id();
String card_content_type = exposureCardsBean.getCard_content_type();
String card_type = exposureCardsBean.getCard_type();
String is_cpc = exposureCardsBean.getIs_cpc();
String cpc_referer = exposureCardsBean.getCpc_referer();
String absolute_position = exposureCardsBean.getAbsolute_position();
String relative_position = exposureCardsBean.getRelative_position();
String transaction_type = exposureCardsBean.getTransaction_type();
//1.3 绝对位置和相对位置位置错误
//问题描述：在'7.7.35','7.7.36' android的绝对位置和相对位置写反
if("android".equals(device_os_type) && ("7.7.35".equals(app_version) || "7.7.36".equals(app_version))) {
String change = absolute_position;
absolute_position = relative_position;
relative_position = change;
}
//1.10 banner、豆腐块、icon、搜索词和功能区入口的card_type参数问题
//问题描述：当卡片类型为豆腐块、icon、banner、搜索词和功能区入口的时候的时候正确的card_type的参数值被赋给了card_content_type（banner、icon、gadget、search_query、function_entrance）
List<String> card_content_type_list = Arrays.asList("banner","icon","gadget","search_query","function_entrance");
if (card_content_type_list.contains(card_content_type)) {
card_type = card_content_type;
card_content_type = null;
}
if ("search_query".equals(card_type)) {
card_type = "search_word";
}
//1.12 card_content_type的qa值应该被换成q_a
if ("qa".equals(card_content_type)) {
card_content_type = card_content_type.replace("qa","q_a");
}
//1.13 target_name与card_name的参数的统一
//问题描述：将target_name参数的值赋给card_name
//1.14 百科卡片曝光新增参数wiki_type
//问题：从7.19.0版本开始，百科卡片曝光,新增了wiki_type参数
//card_type为card的标准码值为common_card,数据中为card
if("card".equals(card_type)) {
card_type = "common_card";
}
//card_type为banner的标准码值为common_banner,数据中为banner
if("banner".equals(card_type)) {
card_type = "common_banner";
}
MlPreciseExposureBean mlPreciseExposureBean = new MlPreciseExposureBean();
mlPreciseExposureBean.setUser_id(blPreciseExposureBean.getUser_id());
mlPreciseExposureBean.setAction(blPreciseExposureBean.getAction());
mlPreciseExposureBean.setPage_code(blPreciseExposureBean.getPage_code());
mlPreciseExposureBean.setTab_name(blPreciseExposureBean.getTab_name());
mlPreciseExposureBean.setBusiness_id(blPreciseExposureBean.getBusiness_id());
mlPreciseExposureBean.setReferrer_code(blPreciseExposureBean.getReferrer_code());
mlPreciseExposureBean.setReferrer_id(blPreciseExposureBean.getReferrer_id());
mlPreciseExposureBean.setCard_id(card_id);
mlPreciseExposureBean.setCard_content_type(card_content_type);
mlPreciseExposureBean.setCard_type(card_type);
mlPreciseExposureBean.setIs_cpc(is_cpc);
mlPreciseExposureBean.setCpc_referer(cpc_referer);
// mlPreciseExposureBean.setAbsolute_position(absolute_position);
// mlPreciseExposureBean.setRelative_position(relative_position);
mlPreciseExposureBean.setTransaction_type(transaction_type);
// mlPreciseExposureBean.setChannel_id(blPreciseExposureBean.getApp_channel());
mlPreciseExposureBean.setApp_version(blPreciseExposureBean.getApp_version());
mlPreciseExposureBean.setCurrent_city_id(blPreciseExposureBean.getApp_current_city_id());
mlPreciseExposureBean.setApp_code(blPreciseExposureBean.getApp_code());
mlPreciseExposureBean.setDevice_os_type(blPreciseExposureBean.getDevice_os_type());
mlPreciseExposureBean.setDevice_id(blPreciseExposureBean.getDevice_id());
mlPreciseExposureBean.setCreate_time_day(blPreciseExposureBean.getCreate_time_day());
mlPreciseExposureBean.setGm_nginx_time_day(blPreciseExposureBean.getGm_nginx_time_day());
mlPreciseExposureBean.setPreciseexposure_num(1);
//1.11 transaction_type的空值需要被修正
//问题描述：transaction_type='' 需要替换为NULL
// List<String> card_content_type_list_ch = Arrays.asList("正常搜索","指定网页","美购详情页ID","医生主页ID","秒杀专场","交易专题ID","秒杀聚合","新专题聚合","医院主页ID","福利专题列表");
List<String> card_content_type_list_ch = Arrays.asList("指定网页");
if (card_content_type_list_ch.contains(card_content_type)) {
System.out.println(mlPreciseExposureBean);
}
mlPreciseExposureBeanList.add(BeanReflectUtil.setNullValue(mlPreciseExposureBean));
}
return mlPreciseExposureBeanList;
}
}
src/main/java/com/gmei/jdbc/MysqlJdbcDim.java 0 → 100644
View file @ e15a2105
package com.gmei.jdbc;
import java.sql.*;
import java.util.List;
/**
* ClassName: MysqlJdbcDim
* Function:
* Reason: 单例模式的码表层jdbc工具
* Date: 2019/12/7 上午11:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class MysqlJdbcDim {
private volatile static MysqlJdbcDim mysqlJdbcDim = null;
// public static String driver = "com.mysql.cj.jdbc.Driver";
// public static String url = "jdbc:mysql://localhost:3306/zhengxing?useSSL=false&serverTimezone=UTC";
// //?useUnicode=true&characterEncoding=utf8&serverTimezone=GMT&allowMultiQueries=true&useSSL=false
// public static String username = "jayden";
// public static String password = "jayden548493";
// private String driver = "com.mysql.jdbc.Driver";
// private String url = "jdbc:mysql://172.22.30.12:3506/test?useUnicode=true&characterEncoding=UTF-8";//设置连接字符串
// private String username = "work";
// private String password = "zJnxVEhyyxeC7ciqxdMITVyWqOFc2mew";
private String driver = "com.mysql.jdbc.Driver";
private String url;
private Connection conn;
private PreparedStatement ps;
private MysqlJdbcDim(String url) {
this.url = url;
}
public static MysqlJdbcDim getInstance(String url) {
if (mysqlJdbcDim == null) {
synchronized (MysqlJdbcDim.class) {
if(mysqlJdbcDim == null) {
mysqlJdbcDim = new MysqlJdbcDim(url);
}
}
}
return mysqlJdbcDim;
}
public Connection getConnection() {
try {
Class.forName(driver);
if(conn == null || conn.isClosed()) {
// conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/zhengxing?useSSL=false&serverTimezone=UTC", "jayden", "jayden548943");
// conn = DriverManager.getConnection(url, username, password);
conn = DriverManager.getConnection(url);
}
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
return conn;
}
public void insert(String sql, List<Object> param) throws SQLException {
ps = conn.prepareStatement(sql);
if (param != null && param.size() > 0) {
for (int i = 0; i < param.size(); i++) {
ps.setObject(i + 1, param.get(i));
}
}
ps.executeUpdate();
}
public void insert(String sql) throws SQLException {
Statement st = null;
st = conn.createStatement();
st.executeUpdate(sql);
}
public ResultSet query(String sql, List<Object> param) {
Statement st = null;
PreparedStatement ps = null;
ResultSet rs = null;
try {
ps = conn.prepareStatement(sql);
if (param != null && param.size() > 0) {
for (int i = 0; i < param.size(); i++) {
ps.setObject(i + 1, param.get(i));
}
}
if(!ps.isClosed())
System.out.println("Succeeded connecting to the prepareStatement!");
rs = ps.executeQuery();
} catch (SQLException e) {
e.printStackTrace();
}
return rs;
}
public ResultSet query(String sql) {
if(conn == null){
conn = getConnection();
}
ResultSet rs = null;
Statement st = null;
try {
st = conn.createStatement();
rs = st.executeQuery(sql);
} catch (SQLException e) {
e.printStackTrace();
}
return rs;
}
public static void close(Connection con, Statement st, ResultSet rs) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
if (st != null) {
try {
st.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (con != null) {
try {
con.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
}
src/main/java/com/gmei/jdbc/MysqlJdbcSink.java 0 → 100644
View file @ e15a2105
package com.gmei.jdbc;
import java.sql.*;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* ClassName: MysqlJdbcSink
* Function:
* Reason: 单例模式的数据下发jdbc工具
* Date: 2019/12/7 上午11:01
*
* @author liuzhe
* @since JDK 1.8
*/
public class MysqlJdbcSink {
private volatile static MysqlJdbcSink mysqlJdbcSink = null;
// private static String driver = "com.mysql.cj.jdbc.Driver";
// public static String url = "jdbc:mysql://localhost:3306/zhengxing?useSSL=false&serverTimezone=UTC";
// //?useUnicode=true&characterEncoding=utf8&serverTimezone=GMT&allowMultiQueries=true&useSSL=false
// public static String username = "jayden";
// public static String password = "jayden548493";
// private String driver = "com.mysql.jdbc.Driver";
// private String url = "jdbc:mysql://172.22.30.12:3506/test?useUnicode=true&characterEncoding=UTF-8";//设置连接字符串
// //rewriteBatchedStatements=true
// private String username = "work";
// private String password = "zJnxVEhyyxeC7ciqxdMITVyWqOFc2mew";
private String driver = "com.mysql.jdbc.Driver";
private String url;
private Connection conn;
private MysqlJdbcSink(String url) {
this.url = url;
}
public static MysqlJdbcSink getInstance(String url) {
if (mysqlJdbcSink == null) {
synchronized (MysqlJdbcSink.class) {
if(mysqlJdbcSink == null) {
mysqlJdbcSink = new MysqlJdbcSink(url);
}
}
}
return mysqlJdbcSink;
}
public Connection getConnection() {
try {
Class.forName(driver);
if(conn == null || conn.isClosed()) {
// conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/zhengxing?useSSL=false&serverTimezone=UTC", "jayden", "jayden548943");
// conn = DriverManager.getConnection(url, username, password);
conn = DriverManager.getConnection(url);
}
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
return conn;
}
public void update(String sql, List<Object> params) throws SQLException {
PreparedStatement ps = null;
try {
ps = conn.prepareStatement(sql);
if (params != null && params.size() > 0) {
for (int i = 0; i < params.size(); i++) {
ps.setObject(i + 1, params.get(i));
}
}
int resultNum = ps.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
} finally {
ps.close();
}
// System.out.println(printRealSql(sql, params).replaceAll("(\\s+|\\\\n)", ""));
}
public void insert(String sql) throws SQLException {
Statement st = null;
try {
st = conn.createStatement();
st.executeUpdate(sql);
} catch (SQLException e) {
e.printStackTrace();
} finally {
st.close();
}
}
public String printRealSql(String sql, List<Object> params) {
if(params == null || params.size() == 0) {
// System.out.println("The SQL is------------>\n" + sql);
return sql;
}
if (!match(sql, params)) {
System.out.println("SQL 语句中的占位符与参数个数不匹配。SQL：" + sql);
return null;
}
int cols = params.size();
// values = params.toArray(index, count);
// Object[] values = new Object[cols];
// System.arraycopy(params, 0, values, 0, cols);
Object[] values = params.toArray(new Object[cols]);
for (int i = 0; i < cols; i++) {
Object value = values[i];
if (value instanceof Date) {
values[i] = "'" + value + "'";
} else if (value instanceof String) {
values[i] = "'" + value + "'";
} else if (value instanceof Boolean) {
values[i] = (Boolean) value ? 1 : 0;
}
}
String statement = String.format(sql.replaceAll("\\?", "%s"), values);
// System.out.println("The SQL is------------>\n" + statement);
return statement;
}
private static boolean match(String sql, List<Object> params) {
if(params == null || params.size() == 0) return true; // 没有参数，完整输出
Matcher m = Pattern.compile("(\\?)").matcher(sql);
int count = 0;
while (m.find()) {
count++;
}
return count == params.size();
}
// public ResultSet query(String sql, List<Object> param) {
// try {
// ps = conn.prepareStatement(sql);
//
// if (param != null && param.size() > 0) {
// for (int i = 0; i < param.size(); i++) {
// ps.setObject(i + 1, param.get(i));
// }
// }
// if(!ps.isClosed())
// System.out.println("Succeeded connecting to the prepareStatement!");
// rs = ps.executeQuery();
// } catch (SQLException e) {
// e.printStackTrace();
// }
// return rs;
// }
public ResultSet query(String sql) {
// System.out.println(sql);
if(conn == null){
conn = getConnection();
}
ResultSet rs = null;
Statement st = null;
try {
st = conn.createStatement();
rs = st.executeQuery(sql);
} catch (SQLException e) {
e.printStackTrace();
}
return rs;
}
public void close(Connection con, Statement st, ResultSet rs) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
if (st != null) {
try {
st.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (con != null) {
try {
con.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
}
src/main/java/com/gmei/schema/BlPreciseExposureSchema.java 0 → 100644
View file @ e15a2105
package com.gmei.schema;
import com.gmei.bean.bl.BlPreciseExposureBean;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
import org.msgpack.MessagePack;
import org.msgpack.type.Value;
import org.msgpack.type.ValueFactory;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* ClassName: BlPreciseExposureSchema
* Function:
* Reason: 暂时未用到
* Date: 2019/12/17 下午4:51
*
* @author liuzhe
* @since JDK 1.8
*/
public class BlPreciseExposureSchema implements DeserializationSchema<String>, SerializationSchema<String> {
private static final long serialVersionUID = 1L;
private transient Charset charset;
public BlPreciseExposureSchema() {
this(StandardCharsets.UTF_8);
}
public BlPreciseExposureSchema(Charset charset) {
this.charset = checkNotNull(charset);
}
public Charset getCharset() {
return charset;
}
private void writeObject (ObjectOutputStream out) throws IOException {
out.defaultWriteObject();
out.writeUTF(charset.name());
}
private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException {
in.defaultReadObject();
String charsetName = in.readUTF();
this.charset = Charset.forName(charsetName);
}
@Override
public String deserialize(byte[] bytes) throws IOException {
String msg = new String(bytes, StandardCharsets.UTF_8);
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(msg);
BlPreciseExposureBean blPreciseExposureBean = mapper.readValue(msg, BlPreciseExposureBean.class);
MessagePack msgpack = new MessagePack();
Value MSGPACK_STRING_VALUE_CONTENT = ValueFactory.createRawValue("content".getBytes(StandardCharsets.US_ASCII));
String contentString = "";
try {
contentString = msgpack.read(bytes).asMapValue().get(MSGPACK_STRING_VALUE_CONTENT).asRawValue().getString();
} catch (IOException e) {
e.printStackTrace();
}
return contentString;
}
@Override
public boolean isEndOfStream(String s) {
return false;
}
@Override
public byte[] serialize(String s) {
return s.getBytes(charset);
}
@Override
public TypeInformation<String> getProducedType() {
return BasicTypeInfo.STRING_TYPE_INFO;
}
}
src/main/java/com/gmei/sink/DayBucketAssigner.java 0 → 100644
View file @ e15a2105
package com.gmei.sink;
import com.gmei.bean.ml.MlPreciseExposureBean;
import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.streaming.api.functions.sink.filesystem.BucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.SimpleVersionedStringSerializer;
/**
* ClassName: DayBucketAssigner
* Function:
* Reason: 按照日期分区
* Date: 2019/12/17 下午7:49
*
* @author liuzhe
* @since JDK 1.8
*/
public class DayBucketAssigner implements BucketAssigner<MlPreciseExposureBean, String> {
@Override
public String getBucketId(MlPreciseExposureBean mlPreciseExposureBean, Context context) {
String gm_nginx_time_day = mlPreciseExposureBean.getGm_nginx_time_day();
return gm_nginx_time_day;
}
@Override
public SimpleVersionedSerializer<String> getSerializer() {
return SimpleVersionedStringSerializer.INSTANCE;
}
}
src/main/java/com/gmei/sink/MlPreciseExposureMysqlSink.java 0 → 100644
View file @ e15a2105
package com.gmei.sink;
import com.gmei.bean.ml.MlPreciseExposureBean;
import com.gmei.cache.MlPreciseExposureDao;
import com.gmei.jdbc.MysqlJdbcSink;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import java.sql.Connection;
/**
* ClassName: MlPreciseExposureMysqlSink
* Function:
* Reason: 数据下发mysql
* Date: 2019/12/16 下午6:45
*
* @author liuzhe
* @since JDK 1.8
*/
public class MlPreciseExposureMysqlSink extends RichSinkFunction<MlPreciseExposureBean> {
private MysqlJdbcSink mysqlJdbcSink;
private Connection conn;
private MlPreciseExposureDao mlPreciseExposureDao;
private int maxRetry = 1;
private long retryTime = 3000;
private String sinkJdbcUrl;
private String sinkTableName;
public MlPreciseExposureMysqlSink(String sinkJdbcUrl, String sinkTableName) {
this.sinkJdbcUrl = sinkJdbcUrl;
this.sinkTableName = sinkTableName;
}
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
mysqlJdbcSink = MysqlJdbcSink.getInstance(sinkJdbcUrl);
mlPreciseExposureDao = new MlPreciseExposureDao(sinkJdbcUrl, sinkTableName);
conn = mysqlJdbcSink.getConnection();
}
@Override
public void invoke(MlPreciseExposureBean value, Context context) throws Exception {
try {
conn.setAutoCommit(false);
mlPreciseExposureDao.insertMlPreciseExposure(value);
conn.commit();
} catch (Exception e) {
conn.rollback();
int numRetry = 1;
Exception lastException = e;
while (numRetry <= maxRetry) {
try {
numRetry ++;
Thread.sleep(retryTime);
mysqlJdbcSink.close(conn, null, null);
conn = mysqlJdbcSink.getConnection();
conn.setAutoCommit(false);
mlPreciseExposureDao.insertMlPreciseExposure(value);
conn.commit();
} catch (Exception e1) {
conn.rollback();
lastException = e1;
continue;
}
return;
}
throw lastException;
}
}
@Override
public void close() throws Exception {
super.close();
mysqlJdbcSink.close(conn, null, null);
}
}
src/main/java/com/gmei/source/BlMaiDianKafkaSource.java 0 → 100644
View file @ e15a2105
package com.gmei.source;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import java.util.Properties;
/**
* ClassName: BlMaiDianKafkaSource
* Function:
* Reason: 获取kafka资源
* Date: 2019/12/5 下午3:54
*
* @author liuzhe
* @since JDK 1.8
*/
public class BlMaiDianKafkaSource {
private String brokers;
private String topic;
private String groupId;
public BlMaiDianKafkaSource() {
}
public BlMaiDianKafkaSource(String brokers, String topic, String groupId) {
this.brokers = brokers;
this.topic = topic;
this.groupId = groupId;
}
/**
* Function: addSource
* Reason: 获取kafka资源
* Date: 2019/12/25 下午4:49
*
* @author liuzhe
* @since JDK 1.8
*/
public FlinkKafkaConsumer011<String> addSource() {
Properties props = new Properties();
props.put("bootstrap.servers", brokers);
props.put("group.id", groupId);
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
// props.put("auto.offset.reset", "earliest");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
FlinkKafkaConsumer011<String> myConsumer = new FlinkKafkaConsumer011<String>(topic, new SimpleStringSchema(), props);
myConsumer.setStartFromGroupOffsets();//默认消费策略
// myConsumer.setStartFromEarliest();
return myConsumer;
}
public DataStreamSource<String> addSource(StreamExecutionEnvironment streamExecutionEnvironment) {
StreamExecutionEnvironment env = streamExecutionEnvironment;
String topicName = "test";
String groupId = "group1";
Properties props = new Properties();
props.put("bootstrap.servers", "localhost:9092");
props.put("group.id", groupId);
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("auto.offset.reset", "earliest");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
FlinkKafkaConsumer011<String> myConsumer = new FlinkKafkaConsumer011<String>(topicName, new SimpleStringSchema(), props);
myConsumer.setStartFromGroupOffsets();//默认消费策略
// myConsumer.setStartFromEarliest();
DataStreamSource<String> dataStreamSource = env.addSource(myConsumer);
return dataStreamSource;
}
}
src/main/java/com/gmei/sql/bl_et_mg_preciseexposure_inc_d.sql 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/main/java/com/gmei/sql/dim_table_ddl.sql 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/main/java/com/gmei/sql/dim_table_dml.sql 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/main/java/com/gmei/streaming/PreciseExposureStreaming.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/main/java/com/gmei/utils/BeanReflectUtil.java 0 → 100644
View file @ e15a2105
package com.gmei.utils;
import java.lang.reflect.Field;
/**
* ClassName: BeanReflectUtil
* Function:
* Reason: 类反射工具类
* Date: 2019/12/5 下午3:54
*
* @author liuzhe
* @since JDK 1.8
*/
public class BeanReflectUtil {
/**
* Function: getFieldValue
* Reason: 通过类反射获取属性值
* Date: 2019/12/25 下午4:41
*
* @author liuzhe
* @since JDK 1.8
*/
public static <T> Object getFieldValue(T obj, String field) throws Exception {
Class<?> claz = obj.getClass();
Field f = null;
Object fieldValue = null;
Field[] fields = claz.getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
if (fields[i].getName().equals(field)) {
f = claz.getDeclaredField(field);
f.setAccessible(true);
fieldValue = f.get(obj);
}
}
return fieldValue;
}
/**
* Function: setFieldValue
* Reason: 通过类反射设置属性值
* Date: 2019/12/25 下午4:41
*
* @author liuzhe
* @since JDK 1.8
*/
public static <T> void setFieldValue(T obj, String field, Object value) {
Class<?> claz = obj.getClass();
Field f = null;
try {
Field[] fields = claz.getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
if (fields[i].getName().equals(field)) {
f = claz.getDeclaredField(field);
f.setAccessible(true);
f.set(obj, value);
}
}
} catch (Exception e) {
e.getStackTrace();
}
}
/**
* Function: setNullValue
* Reason: 通过类反射设置所有空字符串为NULL
* Date: 2019/12/25 下午4:42
*
* @author liuzhe
* @since JDK 1.8
*/
public static <T> T setNullValue(T source) throws IllegalArgumentException, IllegalAccessException, SecurityException {
Field[] fields = source.getClass().getDeclaredFields();
for (Field field : fields) {
if (field.getGenericType().toString().equals("class java.lang.String")) {
field.setAccessible(true);
Object obj = field.get(source);
if (obj != null && obj.equals("")) {
field.set(source, null);
}
}
}
return source;
}
}
src/main/java/com/gmei/utils/DateUtil.java 0 → 100644
View file @ e15a2105
package com.gmei.utils;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* ClassName: DateUtil
* Function:
* Reason: 日期格式工具类
* Date: 2019/12/17 下午4:51
*
* @author liuzhe
* @since JDK 1.8
*/
public class DateUtil {
static SimpleDateFormat secDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
static SimpleDateFormat dayDate = new SimpleDateFormat("yyyyMMdd");
public enum DateType {
DAY,
SECOND;
}
/**
* Function: timestampFormat.
* DAY:yyyyMMdd
* SECOND:yyyy-MM-dd HH:mm:ss
* Reason: timestamp时间戳格式化.
* Date: 2019/12/25 下午4:43
*
* @author liuzhe
* @since JDK 1.8
*/
public static String timestampFormat(String timestamp, DateType type) {
Double timestampDouble = Double.parseDouble(timestamp);
Long timestampLong = new Double(timestampDouble * 1000).longValue();
Date date= new Date(timestampLong);
switch (type) {
case DAY:
return dayDate.format(date);
case SECOND:
return secDate.format(date);
default:
return null;
}
}
}
src/main/java/com/gmei/utils/DimTableList.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/cache/DimCardContentTypeDaoTest.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/cache/DimCityDaoTest.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/cache/DimPageTypeDaoTest.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/cache/DimTableDaoTest.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/etl/BlMaiDianSplitEtl.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/etl/BlPreciseExposureEtl.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/etl/MlPreciseExposureEtl.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/kafka/MyConsumer.java 0 → 100755
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/kafka/MyProducter.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/sink/BlPreciseExposureKafkaSink.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/source/BlMaiDianKafkaSourceTest.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/utils/BeanReflectUtilTest.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
src/test/java/com/gmei/utils/DateUtilTest.java 0 → 100644
View file @ e15a2105
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment