Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
S
serviceRec
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
郭羽
serviceRec
Commits
25b34121
Commit
25b34121
authored
3 years ago
by
郭羽
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
service model 优化
parent
5e5fa6d6
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
4 deletions
+4
-4
featureEng2.py
spark/featureEng2.py
+4
-4
No files found.
spark/featureEng2.py
View file @
25b34121
...
...
@@ -129,17 +129,17 @@ def addStaticsFeatures(samples,dataVocab):
bucket_suffix
=
"_Bucket"
for
col
in
[
"userRatingCount"
,
"userRatingAvg"
,
"userClickCount"
,
"userExpCount"
,
"itemRatingCount"
,
"itemRatingAvg"
,
"itemClickCount"
,
"itemExpCount"
]:
new_col
=
col
+
bucket_suffix
samples
=
samples
.
withColumn
(
new_col
,
numberToBucketUdf
(
F
.
col
(
col
))
.
cast
(
"int"
)
.
cast
(
"string"
))
.
drop
(
col
)
samples
=
samples
.
withColumn
(
new_col
,
F
.
when
(
F
.
col
(
col
)
.
isNull
(),
"0"
)
.
otherwise
(
numberToBucketUdf
(
F
.
col
(
col
))
.
cast
(
"int"
)
.
cast
(
"string"
)
))
.
drop
(
col
)
dataVocab
[
new_col
]
=
bucket_vocab
# 方差处理
number_suffix
=
"_number"
for
col
in
[
"userRatingStddev"
,
"itemRatingStddev"
]:
new_col
=
col
+
number_suffix
samples
=
samples
.
withColumn
(
new_col
,
1
/
(
F
.
col
(
col
)
+
1
))
.
drop
(
col
)
samples
=
samples
.
withColumn
(
new_col
,
F
.
when
(
F
.
col
(
col
)
.
isNull
(),
0
)
.
otherwise
(
1
/
(
F
.
col
(
col
)
+
1
)
))
.
drop
(
col
)
for
col
in
[
"userCtr"
,
"itemCtr"
]:
new_col
=
col
+
number_suffix
samples
=
samples
.
withColumnRenamed
(
col
,
new_col
)
samples
=
samples
.
withColumn
(
col
,
F
.
when
(
F
.
col
(
col
)
.
isNull
(),
0
)
.
otherwise
(
F
.
col
(
col
)))
.
withColumn
Renamed
(
col
,
new_col
)
samples
.
printSchema
()
samples
.
show
(
10
,
truncate
=
False
)
...
...
@@ -814,7 +814,7 @@ if __name__ == '__main__':
# model columns
print
(
"model columns to redis..."
)
model_columns
=
user_columns
+
item_columns
featureColumnsToRedis
(
model_columns
)
#
featureColumnsToRedis(model_columns)
print
(
"数据字典save..."
)
print
(
"dataVocab:"
,
str
(
dataVocab
.
keys
()))
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment