Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
C
ctr-estimate
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
赵建伟
ctr-estimate
Commits
20b76a54
Commit
20b76a54
authored
Apr 11, 2020
by
赵建伟
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update codes
parent
5e21efc3
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
9 additions
and
42 deletions
+9
-42
CtrEstimatePfrOperator.java
...va/com/gmei/data/ctr/operator/CtrEstimatePfrOperator.java
+8
-7
CtrEstimatePfrMysqlSink.java
.../java/com/gmei/data/ctr/sink/CtrEstimatePfrMysqlSink.java
+1
-35
No files found.
src/main/java/com/gmei/data/ctr/operator/CtrEstimatePfrOperator.java
View file @
20b76a54
...
@@ -49,22 +49,23 @@ public class CtrEstimatePfrOperator implements BaseOperator{
...
@@ -49,22 +49,23 @@ public class CtrEstimatePfrOperator implements BaseOperator{
}
}
@Override
@Override
public
void
run
()
{
public
void
run
()
{
SingleOutputStreamOperator
jsonStream
=
dataStream
SingleOutputStreamOperator
filter01
=
dataStream
.
filter
(
new
FilterFunction
<
String
>()
{
.
filter
(
new
FilterFunction
<
String
>()
{
@Override
@Override
public
boolean
filter
(
String
value
)
throws
Exception
{
public
boolean
filter
(
String
value
)
throws
Exception
{
return
JSON
.
isValid
(
value
);
return
JSON
.
isValid
(
value
);
}
}
})
}).
setParallelism
(
parallelism
);
SingleOutputStreamOperator
map01
=
filter01
.
map
(
new
MapFunction
<
String
,
JSONObject
>()
{
.
map
(
new
MapFunction
<
String
,
JSONObject
>()
{
@Override
@Override
public
JSONObject
map
(
String
value
)
throws
Exception
{
public
JSONObject
map
(
String
value
)
throws
Exception
{
return
JSON
.
parseObject
(
value
);
return
JSON
.
parseObject
(
value
);
}
}
});
})
.
setParallelism
(
parallelism
)
;
// jsonStream.print();
// jsonStream.print();
SingleOutputStreamOperator
filter
=
SingleOutputStreamOperator
filter
2
=
jsonStream
.
filter
(
map01
.
filter
(
new
FilterFunction
<
JSONObject
>()
{
new
FilterFunction
<
JSONObject
>()
{
@Override
@Override
public
boolean
filter
(
JSONObject
jsonObject
)
throws
Exception
{
public
boolean
filter
(
JSONObject
jsonObject
)
throws
Exception
{
...
@@ -113,7 +114,7 @@ public class CtrEstimatePfrOperator implements BaseOperator{
...
@@ -113,7 +114,7 @@ public class CtrEstimatePfrOperator implements BaseOperator{
}
}
}).
setParallelism
(
parallelism
);
}).
setParallelism
(
parallelism
);
//filter.print();
//filter.print();
SingleOutputStreamOperator
map
=
filter
SingleOutputStreamOperator
map
02
=
filter2
.
map
(
new
MapFunction
<
JSONObject
,
CtrEstimatePfrEtl
>()
{
.
map
(
new
MapFunction
<
JSONObject
,
CtrEstimatePfrEtl
>()
{
@Override
@Override
public
CtrEstimatePfrEtl
map
(
JSONObject
jsonObject
)
throws
Exception
{
public
CtrEstimatePfrEtl
map
(
JSONObject
jsonObject
)
throws
Exception
{
...
@@ -158,7 +159,7 @@ public class CtrEstimatePfrOperator implements BaseOperator{
...
@@ -158,7 +159,7 @@ public class CtrEstimatePfrOperator implements BaseOperator{
}).
setParallelism
(
parallelism
);
}).
setParallelism
(
parallelism
);
//map.print();
//map.print();
DataStream
<
DeviceCurrentEstimatePfrTmp
>
tidbAsyncDataStream
=
AsyncDataStream
DataStream
<
DeviceCurrentEstimatePfrTmp
>
tidbAsyncDataStream
=
AsyncDataStream
.
unorderedWait
(
map
,
new
TidbMysqlAsyncPfrSource
(
jerryJdbcUrl
,
jerryUsername
,
jerryPassword
),
1
,
TimeUnit
.
MINUTES
,
1000
)
.
unorderedWait
(
map
02
,
new
TidbMysqlAsyncPfrSource
(
jerryJdbcUrl
,
jerryUsername
,
jerryPassword
),
1
,
TimeUnit
.
MINUTES
,
1000
)
.
uid
(
"tidbAsyncDataStream"
)
.
uid
(
"tidbAsyncDataStream"
)
.
setParallelism
(
parallelism
);
.
setParallelism
(
parallelism
);
...
...
src/main/java/com/gmei/data/ctr/sink/CtrEstimatePfrMysqlSink.java
View file @
20b76a54
...
@@ -64,21 +64,12 @@ public class CtrEstimatePfrMysqlSink extends RichSinkFunction<DeviceCurrentEstim
...
@@ -64,21 +64,12 @@ public class CtrEstimatePfrMysqlSink extends RichSinkFunction<DeviceCurrentEstim
super
.
close
();
super
.
close
();
}
}
/**
* 最近偏好持久化
* @param deviceCurrentEstimatePfrTmp
*/
private
void
insertAndDel
(
DeviceCurrentEstimatePfrTmp
deviceCurrentEstimatePfrTmp
)
{
insert
(
deviceCurrentEstimatePfrTmp
);
del
(
deviceCurrentEstimatePfrTmp
);
}
/**
/**
* 插入最新数据
* 插入最新数据
* @param deviceCurrentEstimatePfrTmp
* @param deviceCurrentEstimatePfrTmp
* @throws SQLException
* @throws SQLException
*/
*/
private
void
insert
(
DeviceCurrentEstimatePfrTmp
deviceCurrentEstimatePfrTmp
)
{
private
void
insert
AndDel
(
DeviceCurrentEstimatePfrTmp
deviceCurrentEstimatePfrTmp
)
{
Statement
statement
=
null
;
Statement
statement
=
null
;
Date
date
=
new
Date
();
Date
date
=
new
Date
();
try
{
try
{
...
@@ -113,31 +104,6 @@ public class CtrEstimatePfrMysqlSink extends RichSinkFunction<DeviceCurrentEstim
...
@@ -113,31 +104,6 @@ public class CtrEstimatePfrMysqlSink extends RichSinkFunction<DeviceCurrentEstim
DateUtils
.
getTimeStr
(
date
)
DateUtils
.
getTimeStr
(
date
)
)
)
);
);
}
catch
(
Exception
e
){
e
.
printStackTrace
();
}
finally
{
try
{
if
(
statement
!=
null
)
{
statement
.
close
();
}
if
(
connection
!=
null
)
{
connection
.
close
();
}
}
catch
(
Exception
e
){
e
.
printStackTrace
();
}
}
}
/**
* 删除过期数据
* @param deviceCurrentEstimatePfrTmp
*/
private
void
del
(
DeviceCurrentEstimatePfrTmp
deviceCurrentEstimatePfrTmp
)
{
Statement
statement
=
null
;
Date
date
=
new
Date
();
try
{
statement
=
connection
.
createStatement
();
statement
.
executeUpdate
(
statement
.
executeUpdate
(
String
.
format
(
String
.
format
(
"delete from device_recently_estimate_view_pfr where "
+
"delete from device_recently_estimate_view_pfr where "
+
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment