def __init__(self):

punct = '"“”#$%&\'()*+,-/:;<=>@[\\]^_`{|}~'

self.table = str.maketrans({key: None for key in punct})

@staticmethod

def unicodeToAscii(s):

return ''.join(

c for c in unicodedata.normalize('NFD', s)

if unicodedata.category(c) != 'Mn'

)

# Lowercase, trim, and remove non-letter characters

@staticmethod

def normalizeString(s):

s = preprocessing.unicodeToAscii(s.lower().strip())

s = re.sub(r"([.!?])", r" \1", s)

s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)

return s

def extract_stem(self, fname, l= 'ko'):

spaced = []

with open(fname) as f:

if l == 'ko':

spaced = [['START', ] + mecab.morphs(unicodedata.normalize('NFKC', i.strip()).translate(self.table)) +

['END', ] for i in f.readlines()]

elif l == 'en':

spaced = [['START', ] + [stem(j) for j in mecab.morphs(self.normalizeString(i)) ] +

['END', ] for i in f.readlines()]

else:

assert(False)

return(spaced)

def prepare_embedding(self, ko, en, filenm="embedding_train.txt"):

ko_en = [ list(filter(None.__ne__, list(chain.from_iterable(list(zip_longest(i, j[1:-1]))))))

for i, j in zip(ko, en) ]

with open(filenm, "wt") as f:

f.writelines([" ".join(i) + '\n' for i in ko_en])

return(filenm)

def train_embedding(self, data_files, model_file, embeddings_file, vocab_file, min_count, iter, size, workers, window, splitc=' '):

create_embeddings(data_files, model_file=model_file, embeddings_file=embeddings_file, vocab_file=vocab_file, splitc=' ',

tr_idx_ko = [idx for idx, i in enumerate(ko) if len(i) <= max_len]

tr_idx_en = [idx for idx, i in enumerate(en) if len(i) <= max_len]

tr = np.intersect1d(tr_idx_ko, tr_idx_en)

tr = set(tr)

ko_f = [i for idx, i in enumerate(ko) if idx in tr]

en_f = [i for idx, i in enumerate(en) if idx in tr]

#모형 인스턴스 생성 및 트래이너, loss 정의

#n_hidden, vocab_size, embed_dim, max_seq_length

model = korean_english_translator(n_hidden, vocab_size, embed_dim, max_seq_length, end_idx, attention=True)

model.collect_params().initialize(mx.init.Xavier(), ctx=ctx)

model.embedding.weight.set_data(embed_weights)

trainer = gluon.Trainer(model.collect_params(), 'rmsprop')

loss = SoftmaxCrossEntropyLossMask(end_idx, axis = 2)

test_loss = []

for i, (x_data, y_data, z_data) in enumerate(data_iter):

x_data_l = gluon.utils.split_and_load(x_data, ctx, even_split=False)

y_data_l = gluon.utils.split_and_load(y_data, ctx, even_split=False)

z_data_l = gluon.utils.split_and_load(z_data, ctx, even_split=False)

with autograd.predict_mode():

losses = [loss_obj(model(x, y, F.random.normal(0,1,(x.shape[0], n_hidden), ctx=x.context),F.arange(x.shape[0], ctx=x.context)), z) for x, y, z in zip(x_data_l, y_data_l, z_data_l)]

curr_loss = np.mean([mx.nd.mean(i).asscalar() for i in losses])

test_loss.append(curr_loss)

### 학습 코드

tot_test_loss = []

tot_train_loss = []

epochs = epochs + start_epoch

for e in range(start_epoch, epochs):

tic = time.time()

# Decay learning rate.

if e > 1 and decay:

trainer.set_learning_rate(trainer.learning_rate * 0.7)

train_loss = []

batches = tqdm(tr_data_iterator, 'Batches')

for i, (x_data, y_data, z_data) in enumerate(batches):

x_data_l = gluon.utils.split_and_load(x_data, ctx, even_split=False)

y_data_l = gluon.utils.split_and_load(y_data, ctx, even_split=False)

z_data_l = gluon.utils.split_and_load(z_data, ctx, even_split=False)

with autograd.record():

losses = [loss(model(x, y, F.random.normal(0,1,(x.shape[0], n_hidden), ctx=x.context),F.arange(x.shape[0], ctx=x.context) ),

z) for x, y, z in zip(x_data_l, y_data_l, z_data_l)]

for l in losses:

l.backward()

trainer.step(x_data.shape[0])

curr_loss = np.mean([mx.nd.mean(l).asscalar() for l in losses])

train_loss.append(curr_loss)

batches.set_description("loss {}".format(curr_loss))

mx.nd.waitall()

#caculate test loss

test_loss = calculate_loss(model, te_data_iterator, loss_obj = loss, ctx=ctx)

print('[Epoch %d] time cost: %f'%(e, time.time()-tic))

print("Epoch %s. Train Loss: %s, Test Loss : %s" % (e, np.mean(train_loss), test_loss))

tot_test_loss.append(test_loss)

tot_train_loss.append(np.mean(train_loss))

model.save_params("{}_{}.params".format(mdl_desc, e))