Skip to content

Commit 5d6d417

Browse files
Tomasz Chedanv-kkudrynski
Tomasz Cheda
authored and
nv-kkudrynski
committed
[NCF/PyT] Adding BYOD capabilities
1 parent 4fdd014 commit 5d6d417

38 files changed

Lines changed: 2235 additions & 479 deletions

‎PyTorch/Recommendation/NCF/.dockerignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.git
2+
data/

‎PyTorch/Recommendation/NCF/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:20.06-py3
15+
ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:21.04-py3
1616
FROM ${FROM_IMAGE_NAME}
1717

1818
RUN apt-get update && \

‎PyTorch/Recommendation/NCF/README.md‎

Lines changed: 555 additions & 219 deletions
Large diffs are not rendered by default.

‎PyTorch/Recommendation/NCF/convert.py‎

Lines changed: 109 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
#
1515
# -----------------------------------------------------------------------
1616
#
17-
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
17+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1818
#
1919
# Licensed under the Apache License, Version 2.0 (the "License");
2020
# you may not use this file except in compliance with the License.
@@ -31,13 +31,21 @@
3131
from argparse import ArgumentParser
3232
import pandas as pd
3333
from load import implicit_load
34+
from feature_spec import FeatureSpec
35+
from neumf_constants import USER_CHANNEL_NAME, ITEM_CHANNEL_NAME, LABEL_CHANNEL_NAME, TEST_SAMPLES_PER_SERIES
3436
import torch
37+
import os
3538
import tqdm
3639

37-
MIN_RATINGS = 20
40+
TEST_1 = 'test_data_1.pt'
41+
TEST_0 = 'test_data_0.pt'
42+
TRAIN_1 = 'train_data_1.pt'
43+
TRAIN_0 = 'train_data_0.pt'
44+
3845
USER_COLUMN = 'user_id'
3946
ITEM_COLUMN = 'item_id'
4047

48+
4149
def parse_args():
4250
parser = ArgumentParser()
4351
parser.add_argument('--path', type=str, default='/data/ml-20m/ratings.csv',
@@ -61,7 +69,7 @@ def __init__(self, train_ratings, nb_neg):
6169
ids = (train_ratings[:, 0] * self.nb_items) + train_ratings[:, 1]
6270
self.set = set(ids)
6371

64-
def generate(self, batch_size=128*1024):
72+
def generate(self, batch_size=128 * 1024):
6573
users = torch.arange(0, self.nb_users).reshape([1, -1]).repeat([self.nb_neg, 1]).transpose(0, 1).reshape(-1)
6674

6775
items = [-1] * len(users)
@@ -82,6 +90,71 @@ def generate(self, batch_size=128*1024):
8290
return items
8391

8492

93+
def save_feature_spec(user_cardinality, item_cardinality, dtypes, test_negative_samples, output_path,
94+
user_feature_name='user',
95+
item_feature_name='item',
96+
label_feature_name='label'):
97+
feature_spec = {
98+
user_feature_name: {
99+
'dtype': dtypes[user_feature_name],
100+
'cardinality': int(user_cardinality)
101+
},
102+
item_feature_name: {
103+
'dtype': dtypes[item_feature_name],
104+
'cardinality': int(item_cardinality)
105+
},
106+
label_feature_name: {
107+
'dtype': dtypes[label_feature_name],
108+
}
109+
}
110+
metadata = {
111+
TEST_SAMPLES_PER_SERIES: test_negative_samples + 1
112+
}
113+
train_mapping = [
114+
{
115+
'type': 'torch_tensor',
116+
'features': [
117+
user_feature_name,
118+
item_feature_name
119+
],
120+
'files': [TRAIN_0]
121+
},
122+
{
123+
'type': 'torch_tensor',
124+
'features': [
125+
label_feature_name
126+
],
127+
'files': [TRAIN_1]
128+
}
129+
]
130+
test_mapping = [
131+
{
132+
'type': 'torch_tensor',
133+
'features': [
134+
user_feature_name,
135+
item_feature_name
136+
],
137+
'files': [TEST_0],
138+
},
139+
{
140+
'type': 'torch_tensor',
141+
'features': [
142+
label_feature_name
143+
],
144+
'files': [TEST_1],
145+
}
146+
]
147+
channel_spec = {
148+
USER_CHANNEL_NAME: [user_feature_name],
149+
ITEM_CHANNEL_NAME: [item_feature_name],
150+
LABEL_CHANNEL_NAME: [label_feature_name]
151+
}
152+
source_spec = {'train': train_mapping, 'test': test_mapping}
153+
feature_spec = FeatureSpec(feature_spec=feature_spec, metadata=metadata, source_spec=source_spec,
154+
channel_spec=channel_spec, base_directory="")
155+
feature_spec.to_yaml(output_path=output_path)
156+
157+
85158
def main():
86159
args = parse_args()
87160

@@ -91,39 +164,54 @@ def main():
91164
print("Loading raw data from {}".format(args.path))
92165
df = implicit_load(args.path, sort=False)
93166

94-
print("Filtering out users with less than {} ratings".format(MIN_RATINGS))
95-
grouped = df.groupby(USER_COLUMN)
96-
df = grouped.filter(lambda x: len(x) >= MIN_RATINGS)
97-
98167
print("Mapping original user and item IDs to new sequential IDs")
99168
df[USER_COLUMN] = pd.factorize(df[USER_COLUMN])[0]
100169
df[ITEM_COLUMN] = pd.factorize(df[ITEM_COLUMN])[0]
101170

171+
user_cardinality = df[USER_COLUMN].max() + 1
172+
item_cardinality = df[ITEM_COLUMN].max() + 1
173+
102174
# Need to sort before popping to get last item
103175
df.sort_values(by='timestamp', inplace=True)
104176

105177
# clean up data
106178
del df['rating'], df['timestamp']
107-
df = df.drop_duplicates() # assuming it keeps order
179+
df = df.drop_duplicates() # assuming it keeps order
108180

109-
# now we have filtered and sorted by time data, we can split test data out
181+
# Test set is the last interaction for a given user
110182
grouped_sorted = df.groupby(USER_COLUMN, group_keys=False)
111-
test_data = grouped_sorted.tail(1).sort_values(by='user_id')
112-
# need to pop for each group
183+
test_data = grouped_sorted.tail(1).sort_values(by=USER_COLUMN)
184+
# Train set is all interactions but the last one
113185
train_data = grouped_sorted.apply(lambda x: x.iloc[:-1])
114186

115-
# Note: no way to keep reference training data ordering because use of python set and multi-process
116-
# It should not matter since it will be later randomized again
117-
# save train and val data that is fixed.
118-
train_ratings = torch.from_numpy(train_data.values)
119-
torch.save(train_ratings, args.output+'/train_ratings.pt')
120-
test_ratings = torch.from_numpy(test_data.values)
121-
torch.save(test_ratings, args.output+'/test_ratings.pt')
122-
123-
sampler = _TestNegSampler(train_ratings.cpu().numpy(), args.valid_negative)
187+
sampler = _TestNegSampler(train_data.values, args.valid_negative)
124188
test_negs = sampler.generate().cuda()
125189
test_negs = test_negs.reshape(-1, args.valid_negative)
126-
torch.save(test_negs, args.output+'/test_negatives.pt')
190+
191+
# Reshape train set into user,item,label tabular and save
192+
train_ratings = torch.from_numpy(train_data.values).cuda()
193+
train_labels = torch.ones_like(train_ratings[:, 0:1], dtype=torch.float32)
194+
torch.save(train_ratings, os.path.join(args.output, TRAIN_0))
195+
torch.save(train_labels, os.path.join(args.output, TRAIN_1))
196+
197+
# Reshape test set into user,item,label tabular and save
198+
# All users have the same number of items, items for a given user appear consecutively
199+
test_ratings = torch.from_numpy(test_data.values).cuda()
200+
test_users_pos = test_ratings[:, 0:1] # slicing instead of indexing to keep dimensions
201+
test_items_pos = test_ratings[:, 1:2]
202+
test_users = test_users_pos.repeat_interleave(args.valid_negative + 1, dim=0)
203+
test_items = torch.cat((test_items_pos.reshape(-1, 1), test_negs), dim=1).reshape(-1, 1)
204+
positive_labels = torch.ones_like(test_users_pos, dtype=torch.float32)
205+
negative_labels = torch.zeros_like(test_users_pos, dtype=torch.float32).repeat(1, args.valid_negative)
206+
test_labels = torch.cat((positive_labels, negative_labels), dim=1).reshape(-1, 1)
207+
dtypes = {'user': str(test_users.dtype), 'item': str(test_items.dtype), 'label': str(test_labels.dtype)}
208+
test_tensor = torch.cat((test_users, test_items), dim=1)
209+
torch.save(test_tensor, os.path.join(args.output, TEST_0))
210+
torch.save(test_labels, os.path.join(args.output, TEST_1))
211+
212+
save_feature_spec(user_cardinality=user_cardinality, item_cardinality=item_cardinality, dtypes=dtypes,
213+
test_negative_samples=args.valid_negative, output_path=args.output + '/feature_spec.yaml')
214+
127215

128216
if __name__ == '__main__':
129217
main()

‎PyTorch/Recommendation/NCF/convert_test.py‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# Copyright (c) 2018, deepakn94, codyaustun, robieta. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
# -----------------------------------------------------------------------
16+
#
17+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
18+
#
19+
# Licensed under the Apache License, Version 2.0 (the "License");
20+
# you may not use this file except in compliance with the License.
21+
# You may obtain a copy of the License at
22+
#
23+
# http://www.apache.org/licenses/LICENSE-2.0
24+
#
25+
# Unless required by applicable law or agreed to in writing, software
26+
# distributed under the License is distributed on an "AS IS" BASIS,
27+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
28+
# See the License for the specific language governing permissions and
29+
# limitations under the License.
30+
31+
from argparse import ArgumentParser
32+
import pandas as pd
33+
import numpy as np
34+
from load import implicit_load
35+
from convert import save_feature_spec, _TestNegSampler, TEST_0, TEST_1, TRAIN_0, TRAIN_1
36+
import torch
37+
import os
38+
39+
USER_COLUMN = 'user_id'
40+
ITEM_COLUMN = 'item_id'
41+
42+
43+
def parse_args():
44+
parser = ArgumentParser()
45+
parser.add_argument('--path', type=str, default='/data/ml-20m/ratings.csv',
46+
help='Path to reviews CSV file from MovieLens')
47+
parser.add_argument('--output', type=str, default='/data',
48+
help='Output directory for train and test files')
49+
parser.add_argument('--valid_negative', type=int, default=100,
50+
help='Number of negative samples for each positive test example')
51+
parser.add_argument('--seed', '-s', type=int, default=1,
52+
help='Manually set random seed for torch')
53+
parser.add_argument('--test', type=str, help='select modification to be applied to the set')
54+
return parser.parse_args()
55+
56+
57+
def main():
58+
args = parse_args()
59+
60+
if args.seed is not None:
61+
torch.manual_seed(args.seed)
62+
63+
print("Loading raw data from {}".format(args.path))
64+
df = implicit_load(args.path, sort=False)
65+
66+
if args.test == 'less_user':
67+
to_drop = set(list(df[USER_COLUMN].unique())[-100:])
68+
df = df[~df[USER_COLUMN].isin(to_drop)]
69+
if args.test == 'less_item':
70+
to_drop = set(list(df[ITEM_COLUMN].unique())[-100:])
71+
df = df[~df[ITEM_COLUMN].isin(to_drop)]
72+
if args.test == 'more_user':
73+
sample = df.sample(frac=0.2).copy()
74+
sample[USER_COLUMN] = sample[USER_COLUMN] + 10000000
75+
df = df.append(sample)
76+
users = df[USER_COLUMN]
77+
df = df[users.isin(users[users.duplicated(keep=False)])] # make sure something remains in the train set
78+
if args.test == 'more_item':
79+
sample = df.sample(frac=0.2).copy()
80+
sample[ITEM_COLUMN] = sample[ITEM_COLUMN] + 10000000
81+
df = df.append(sample)
82+
83+
print("Mapping original user and item IDs to new sequential IDs")
84+
df[USER_COLUMN] = pd.factorize(df[USER_COLUMN])[0]
85+
df[ITEM_COLUMN] = pd.factorize(df[ITEM_COLUMN])[0]
86+
87+
user_cardinality = df[USER_COLUMN].max() + 1
88+
item_cardinality = df[ITEM_COLUMN].max() + 1
89+
90+
# Need to sort before popping to get last item
91+
df.sort_values(by='timestamp', inplace=True)
92+
93+
# clean up data
94+
del df['rating'], df['timestamp']
95+
df = df.drop_duplicates() # assuming it keeps order
96+
97+
# Test set is the last interaction for a given user
98+
grouped_sorted = df.groupby(USER_COLUMN, group_keys=False)
99+
test_data = grouped_sorted.tail(1).sort_values(by=USER_COLUMN)
100+
# Train set is all interactions but the last one
101+
train_data = grouped_sorted.apply(lambda x: x.iloc[:-1])
102+
103+
sampler = _TestNegSampler(train_data.values, args.valid_negative)
104+
test_negs = sampler.generate().cuda()
105+
if args.valid_negative > 0:
106+
test_negs = test_negs.reshape(-1, args.valid_negative)
107+
else:
108+
test_negs = test_negs.reshape(test_data.shape[0], 0)
109+
110+
if args.test == 'more_pos':
111+
mask = np.random.rand(len(test_data)) < 0.5
112+
sample = test_data[mask].copy()
113+
sample[ITEM_COLUMN] = sample[ITEM_COLUMN] + 5
114+
test_data = test_data.append(sample)
115+
test_negs_copy = test_negs[mask]
116+
test_negs = torch.cat((test_negs, test_negs_copy), dim=0)
117+
if args.test == 'less_pos':
118+
mask = np.random.rand(len(test_data)) < 0.5
119+
test_data = test_data[mask]
120+
test_negs = test_negs[mask]
121+
122+
# Reshape train set into user,item,label tabular and save
123+
train_ratings = torch.from_numpy(train_data.values).cuda()
124+
train_labels = torch.ones_like(train_ratings[:, 0:1], dtype=torch.float32)
125+
torch.save(train_ratings, os.path.join(args.output, TRAIN_0))
126+
torch.save(train_labels, os.path.join(args.output, TRAIN_1))
127+
128+
# Reshape test set into user,item,label tabular and save
129+
# All users have the same number of items, items for a given user appear consecutively
130+
test_ratings = torch.from_numpy(test_data.values).cuda()
131+
test_users_pos = test_ratings[:, 0:1] # slicing instead of indexing to keep dimensions
132+
test_items_pos = test_ratings[:, 1:2]
133+
test_users = test_users_pos.repeat_interleave(args.valid_negative + 1, dim=0)
134+
test_items = torch.cat((test_items_pos.reshape(-1, 1), test_negs), dim=1).reshape(-1, 1)
135+
positive_labels = torch.ones_like(test_users_pos, dtype=torch.float32)
136+
negative_labels = torch.zeros_like(test_users_pos, dtype=torch.float32).repeat(1, args.valid_negative)
137+
test_labels = torch.cat((positive_labels, negative_labels), dim=1).reshape(-1, 1)
138+
dtypes = {'user': str(test_users.dtype), 'item': str(test_items.dtype), 'label': str(test_labels.dtype)}
139+
test_tensor = torch.cat((test_users, test_items), dim=1)
140+
torch.save(test_tensor, os.path.join(args.output, TEST_0))
141+
torch.save(test_labels, os.path.join(args.output, TEST_1))
142+
143+
if args.test == 'other_names':
144+
dtypes = {'user_2': str(test_users.dtype),
145+
'item_2': str(test_items.dtype),
146+
'label_2': str(test_labels.dtype)}
147+
save_feature_spec(user_cardinality=user_cardinality, item_cardinality=item_cardinality, dtypes=dtypes,
148+
test_negative_samples=args.valid_negative, output_path=args.output + '/feature_spec.yaml',
149+
user_feature_name='user_2',
150+
item_feature_name='item_2',
151+
label_feature_name='label_2')
152+
else:
153+
save_feature_spec(user_cardinality=user_cardinality, item_cardinality=item_cardinality, dtypes=dtypes,
154+
test_negative_samples=args.valid_negative, output_path=args.output + '/feature_spec.yaml')
155+
156+
157+
if __name__ == '__main__':
158+
main()

0 commit comments

Comments
 (0)