Commit bc84c25e authored by 赵建伟's avatar 赵建伟

update codes

parent 76f6d7f2
...@@ -11,4 +11,5 @@ public class Constants { ...@@ -11,4 +11,5 @@ public class Constants {
public static final String FORMAT_ERROR_COUNT = "format_error_count"; public static final String FORMAT_ERROR_COUNT = "format_error_count";
public static final String PORTRAIT_INPUT_COUNT = "portrait_input_count"; public static final String PORTRAIT_INPUT_COUNT = "portrait_input_count";
public static final String PORTRAIT_OUTPUT_COUNT = "portrait_output_count"; public static final String PORTRAIT_OUTPUT_COUNT = "portrait_output_count";
public static final String MYSQL_DRIVER_CLASS = "com.mysql.jdbc.Driver";
} }
...@@ -50,9 +50,10 @@ public class PortraitMonitorErrOperator implements BaseOperator{ ...@@ -50,9 +50,10 @@ public class PortraitMonitorErrOperator implements BaseOperator{
String action = sysObject.getString("action"); String action = sysObject.getString("action");
if (null != action) { if (null != action) {
if ("/api/private_conversation/".equals(action) if ("/api/private_conversation/".equals(action)
|| "/api/initiate/interest_record".equals(action) //|| "/api/initiate/interest_record".equals(action)
|| "/api/one_image/share/v3".equals(action) //|| "/api/one_image/share/v3".equals(action)
|| "/gm_ai/face_app/test_skin".equals(action)) { //|| "/gm_ai/face_app/test_skin".equals(action)
) {
jsonObject.put("statistics_action", action); jsonObject.put("statistics_action", action);
} }
} }
...@@ -85,7 +86,8 @@ public class PortraitMonitorErrOperator implements BaseOperator{ ...@@ -85,7 +86,8 @@ public class PortraitMonitorErrOperator implements BaseOperator{
if ("do_search".equals(type) if ("do_search".equals(type)
|| "goto_welfare_detail".equals(type) || "goto_welfare_detail".equals(type)
|| "on_click_card".equals(type) || "on_click_card".equals(type)
|| "home_click_section".equals(type)) { //|| "home_click_section".equals(type)
) {
jsonObject.put("statistics_action", type); jsonObject.put("statistics_action", type);
} }
} }
......
...@@ -180,7 +180,8 @@ public class PortraitMonitorShdOperator implements BaseOperator{ ...@@ -180,7 +180,8 @@ public class PortraitMonitorShdOperator implements BaseOperator{
} }
} }
}); });
process.addSink(new PortraitShdMysqlSink(outJdbcUrl,maxRetry,retryInteral)) process
.addSink(new PortraitShdMysqlSink(outJdbcUrl,maxRetry,retryInteral))
.setParallelism(parallelism); .setParallelism(parallelism);
} }
} }
...@@ -48,12 +48,14 @@ public class PortraitMonitorSucOperator implements BaseOperator{ ...@@ -48,12 +48,14 @@ public class PortraitMonitorSucOperator implements BaseOperator{
@Override @Override
public void run() { public void run() {
SingleOutputStreamOperator map = dataStream.filter(new FilterFunction<String>() { SingleOutputStreamOperator map = dataStream
.filter(new FilterFunction<String>() {
@Override @Override
public boolean filter(String value) throws Exception { public boolean filter(String value) throws Exception {
return JSON.isValid(value); return JSON.isValid(value);
} }
}).assignTimestampsAndWatermarks(new AscendingTimestampExtractor<String>() { })
.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<String>() {
@Override @Override
public long extractAscendingTimestamp(String element) { public long extractAscendingTimestamp(String element) {
JSONObject jsonObject = JSON.parseObject(element); JSONObject jsonObject = JSON.parseObject(element);
...@@ -61,7 +63,8 @@ public class PortraitMonitorSucOperator implements BaseOperator{ ...@@ -61,7 +63,8 @@ public class PortraitMonitorSucOperator implements BaseOperator{
long dateTime = Math.round(Double.valueOf(dateTimeStr)) * 1000; long dateTime = Math.round(Double.valueOf(dateTimeStr)) * 1000;
return dateTime; return dateTime;
} }
}).map(new MapFunction<String, GmPortraitResult>() { })
.map(new MapFunction<String, GmPortraitResult>() {
@Override @Override
public GmPortraitResult map(String value) { public GmPortraitResult map(String value) {
try { try {
...@@ -77,7 +80,7 @@ public class PortraitMonitorSucOperator implements BaseOperator{ ...@@ -77,7 +80,7 @@ public class PortraitMonitorSucOperator implements BaseOperator{
} }
} }
}); });
map.print(); //map.print();
SingleOutputStreamOperator filter = map.filter(new FilterFunction<GmPortraitResult>() { SingleOutputStreamOperator filter = map.filter(new FilterFunction<GmPortraitResult>() {
@Override @Override
public boolean filter(GmPortraitResult value) throws Exception { public boolean filter(GmPortraitResult value) throws Exception {
...@@ -88,7 +91,7 @@ public class PortraitMonitorSucOperator implements BaseOperator{ ...@@ -88,7 +91,7 @@ public class PortraitMonitorSucOperator implements BaseOperator{
return true; return true;
} }
}); });
filter.print(); //filter.print();
filter filter
.keyBy("event") .keyBy("event")
.timeWindow(Time.seconds(windownSize),Time.seconds(slideSize)) .timeWindow(Time.seconds(windownSize),Time.seconds(slideSize))
......
package com.gmei.data.monitor.sink; package com.gmei.data.monitor.sink;
import com.gmei.data.monitor.common.Constants;
import com.gmei.data.monitor.utils.JDBCUtils; import com.gmei.data.monitor.utils.JDBCUtils;
import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.Configuration;
...@@ -32,7 +33,7 @@ public class PortraitErrMysqlSink extends RichSinkFunction<Tuple2<String,String> ...@@ -32,7 +33,7 @@ public class PortraitErrMysqlSink extends RichSinkFunction<Tuple2<String,String>
@Override @Override
public void open(Configuration parameters) throws Exception { public void open(Configuration parameters) throws Exception {
Class.forName("com.mysql.jdbc.Driver"); Class.forName(Constants.MYSQL_DRIVER_CLASS);
connection = DriverManager.getConnection(jdbcUrl); connection = DriverManager.getConnection(jdbcUrl);
super.open(parameters); super.open(parameters);
} }
......
package com.gmei.data.monitor.sink; package com.gmei.data.monitor.sink;
import com.gmei.data.monitor.bean.TblMonitorPortraitShd; import com.gmei.data.monitor.bean.TblMonitorPortraitShd;
import com.gmei.data.monitor.common.Constants;
import com.gmei.data.monitor.utils.JDBCUtils; import com.gmei.data.monitor.utils.JDBCUtils;
import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
...@@ -33,7 +33,7 @@ public class PortraitShdMysqlSink extends RichSinkFunction<TblMonitorPortraitShd ...@@ -33,7 +33,7 @@ public class PortraitShdMysqlSink extends RichSinkFunction<TblMonitorPortraitShd
@Override @Override
public void open(Configuration parameters) throws Exception { public void open(Configuration parameters) throws Exception {
Class.forName("com.mysql.jdbc.Driver"); Class.forName(Constants.MYSQL_DRIVER_CLASS);
connection = DriverManager.getConnection(jdbcUrl); connection = DriverManager.getConnection(jdbcUrl);
super.open(parameters); super.open(parameters);
} }
......
package com.gmei.data.monitor.sink; package com.gmei.data.monitor.sink;
import com.gmei.data.monitor.bean.TblMonitorPortraitSuc; import com.gmei.data.monitor.bean.TblMonitorPortraitSuc;
import com.gmei.data.monitor.common.Constants;
import com.gmei.data.monitor.utils.JDBCUtils; import com.gmei.data.monitor.utils.JDBCUtils;
import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
...@@ -31,7 +32,7 @@ public class PortraitSucMysqlSink extends RichSinkFunction<TblMonitorPortraitSuc ...@@ -31,7 +32,7 @@ public class PortraitSucMysqlSink extends RichSinkFunction<TblMonitorPortraitSuc
@Override @Override
public void open(Configuration parameters) throws Exception { public void open(Configuration parameters) throws Exception {
Class.forName("com.mysql.jdbc.Driver"); Class.forName(Constants.MYSQL_DRIVER_CLASS);
connection = DriverManager.getConnection(jdbcUrl); connection = DriverManager.getConnection(jdbcUrl);
super.open(parameters); super.open(parameters);
} }
......
...@@ -15,7 +15,6 @@ import java.util.Properties; ...@@ -15,7 +15,6 @@ import java.util.Properties;
* @since JDK 1.8 * @since JDK 1.8
*/ */
public class GmeiKafkaSource { public class GmeiKafkaSource {
private String topic; private String topic;
private Properties prop; private Properties prop;
private FlinkKafkaConsumer flinkKafkaConsumer; private FlinkKafkaConsumer flinkKafkaConsumer;
......
...@@ -10,7 +10,7 @@ import java.sql.Statement; ...@@ -10,7 +10,7 @@ import java.sql.Statement;
* Reason: jdbc工具类 * Reason: jdbc工具类
* Date: 2020-03-16 00:00:00 * Date: 2020-03-16 00:00:00
* *
* @author apple * @author zhaojianwei
* @since JDK 1.8 * @since JDK 1.8
*/ */
public class JDBCUtils { public class JDBCUtils {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment