Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
F
ffm-baseline
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ML
ffm-baseline
Commits
c0236d50
Commit
c0236d50
authored
6 years ago
by
张彦钊
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
change test file
parent
dbdda60b
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
4 deletions
+4
-4
feature_test.py
tensnsorflow/feature_test.py
+4
-4
No files found.
tensnsorflow/feature_test.py
View file @
c0236d50
...
@@ -226,7 +226,7 @@ def feature_engineer():
...
@@ -226,7 +226,7 @@ def feature_engineer():
train
=
rdd
.
map
(
train
=
rdd
.
map
(
lambda
x
:
(
x
[
1
],
x
[
2
],
x
[
3
],
x
[
4
],
x
[
5
],
x
[
6
],
x
[
7
],
x
[
8
],
x
[
9
],
lambda
x
:
(
x
[
1
],
x
[
2
],
x
[
3
],
x
[
4
],
x
[
5
],
x
[
6
],
x
[
7
],
x
[
8
],
x
[
9
],
x
[
10
],
x
[
11
],
x
[
12
],
x
[
13
]))
x
[
10
],
x
[
11
],
x
[
12
],
x
[
13
]
,
x
[
14
]
))
f
=
time
.
time
()
f
=
time
.
time
()
spark
.
createDataFrame
(
train
)
.
toDF
(
"y"
,
"z"
,
"app_list"
,
"level2_list"
,
"level3_list"
,
spark
.
createDataFrame
(
train
)
.
toDF
(
"y"
,
"z"
,
"app_list"
,
"level2_list"
,
"level3_list"
,
"tag1_list"
,
"tag2_list"
,
"tag3_list"
,
"tag4_list"
,
"tag1_list"
,
"tag2_list"
,
"tag3_list"
,
"tag4_list"
,
...
@@ -243,7 +243,7 @@ def feature_engineer():
...
@@ -243,7 +243,7 @@ def feature_engineer():
# test = rdd.filter(lambda x: x[0] == validate_date).map(
# test = rdd.filter(lambda x: x[0] == validate_date).map(
# lambda x: (x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9],
# lambda x: (x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9],
# x[10], x[11], x[12], x[13]))
# x[10], x[11], x[12], x[13]
,x[14]
))
#
#
# spark.createDataFrame(test).toDF("y", "z", "app_list", "level2_list", "level3_list",
# spark.createDataFrame(test).toDF("y", "z", "app_list", "level2_list", "level3_list",
# "tag1_list", "tag2_list", "tag3_list", "tag4_list",
# "tag1_list", "tag2_list", "tag3_list", "tag4_list",
...
@@ -320,7 +320,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
...
@@ -320,7 +320,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
# print("native csv")
# print("native csv")
# native_pre.toPandas().to_csv(local_path+"native.csv", header=True)
# native_pre.toPandas().to_csv(local_path+"native.csv", header=True)
# spark.createDataFrame(rdd.filter(lambda x: x[0] == 0)
# spark.createDataFrame(rdd.filter(lambda x: x[0] == 0)
# .map(lambda x: (x[1],x[2],x[6],x[7],x[8],x[9],x[10],x[11],x[12],x[13],x[14],x[15],x[16]))) \
# .map(lambda x: (x[1],x[2],x[6],x[7],x[8],x[9],x[10],x[11],x[12],x[13],x[14],x[15],x[16]
,x[17]
))) \
# .toDF("y","z","app_list", "level2_list", "level3_list","tag1_list", "tag2_list", "tag3_list", "tag4_list",
# .toDF("y","z","app_list", "level2_list", "level3_list","tag1_list", "tag2_list", "tag3_list", "tag4_list",
# "tag5_list", "tag6_list", "tag7_list", "ids","number").repartition(100).write.format("tfrecords") \
# "tag5_list", "tag6_list", "tag7_list", "ids","number").repartition(100).write.format("tfrecords") \
# .save(path=path+"native/", mode="overwrite")
# .save(path=path+"native/", mode="overwrite")
...
@@ -335,7 +335,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
...
@@ -335,7 +335,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
spark
.
createDataFrame
(
rdd
.
filter
(
lambda
x
:
x
[
0
]
==
1
)
spark
.
createDataFrame
(
rdd
.
filter
(
lambda
x
:
x
[
0
]
==
1
)
.
map
(
.
map
(
lambda
x
:
(
x
[
1
],
x
[
2
],
x
[
6
],
x
[
7
],
x
[
8
],
x
[
9
],
x
[
10
],
x
[
11
],
x
[
12
],
x
[
13
],
x
[
14
],
x
[
15
],
x
[
16
])))
\
lambda
x
:
(
x
[
1
],
x
[
2
],
x
[
6
],
x
[
7
],
x
[
8
],
x
[
9
],
x
[
10
],
x
[
11
],
x
[
12
],
x
[
13
],
x
[
14
],
x
[
15
],
x
[
16
]
,
x
[
17
]
)))
\
.
toDF
(
"y"
,
"z"
,
"app_list"
,
"level2_list"
,
"level3_list"
,
"tag1_list"
,
"tag2_list"
,
"tag3_list"
,
"tag4_list"
,
.
toDF
(
"y"
,
"z"
,
"app_list"
,
"level2_list"
,
"level3_list"
,
"tag1_list"
,
"tag2_list"
,
"tag3_list"
,
"tag4_list"
,
"tag5_list"
,
"tag6_list"
,
"tag7_list"
,
"ids"
,
"number"
)
.
repartition
(
100
)
.
write
.
format
(
"tfrecords"
)
\
"tag5_list"
,
"tag6_list"
,
"tag7_list"
,
"ids"
,
"number"
)
.
repartition
(
100
)
.
write
.
format
(
"tfrecords"
)
\
.
save
(
path
=
path
+
"test_nearby/"
,
mode
=
"overwrite"
)
.
save
(
path
=
path
+
"test_nearby/"
,
mode
=
"overwrite"
)
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment