remove former train method , instead with tensorflow's log likehood, add bilstm

supercoderhawk · supercoderhawk · commit 63fdb13b88b8 · 2017-12-02T14:51:32.000+08:00
diff --git a/python/dnlp/core/dnn_crf.py b/python/dnlp/core/dnn_crf.py
@@ -8,7 +8,7 @@
 
 class DnnCrf(DnnCrfBase):
   def __init__(self, *, config: DnnCrfConfig = None, data_path: str = '', dtype: type = tf.float32, mode: str = 'train',
-               train: str = 'll', nn: str, model_path: str = ''):
+               predict: str = 'll', nn: str, model_path: str = ''):
     if mode not in ['train', 'predict']:
       raise Exception('mode error')
     if nn not in ['mlp', 'rnn', 'lstm', 'bilstm', 'gru']:
@@ -27,113 +27,42 @@ def __init__(self, *, config: DnnCrfConfig = None, data_path: str = '', dtype: t
     if mode == 'train':
       self.input = tf.placeholder(tf.int32, [self.batch_size, self.batch_length, self.windows_size])
       self.real_indices = tf.placeholder(tf.int32, [self.batch_size, self.batch_length])
-      self.seq_length = tf.placeholder(tf.int32, [self.batch_size])
     else:
       self.input = tf.placeholder(tf.int32, [None, self.windows_size])
 
+    self.seq_length = tf.placeholder(tf.int32, [None])
+
     # &#26597;&#25214;&#34920;&#23618;
     self.embedding_layer = self.get_embedding_layer()
     # &#38544;&#34255;&#23618;
     if nn == 'mlp':
       self.hidden_layer = self.get_mlp_layer(tf.transpose(self.embedding_layer))
     elif nn == 'lstm':
       self.hidden_layer = self.get_lstm_layer(self.embedding_layer)
+    elif nn == 'bilstm':
+      self.hidden_layer = self.get_bilstm_layer(self.embedding_layer)
     elif nn == 'gru':
-      self.hidden_layer = self.get_gru_layer(tf.transpose(self.embedding_layer))
+      self.hidden_layer = self.get_gru_layer(self.embedding_layer)
     else:
-      self.hidden_layer = self.get_rnn_layer(tf.transpose(self.embedding_layer))
+      self.hidden_layer = self.get_rnn_layer(self.embedding_layer)
     # &#36755;&#20986;&#23618;
     self.output = self.get_output_layer(self.hidden_layer)
 
     if mode == 'predict':
-      self.output = tf.squeeze(tf.transpose(self.output), axis=2)
+      if predict != 'll':
+        self.output = tf.squeeze(tf.transpose(self.output), axis=2)
+      self.seq, self.best_score = tf.contrib.crf.crf_decode(self.output, self.transition, self.seq_length)
       self.sess = tf.Session()
       self.sess.run(tf.global_variables_initializer())
       tf.train.Saver().restore(save_path=self.model_path, sess=self.sess)
-    elif train == 'll':
-      self.ll_loss, _ = tf.contrib.crf.crf_log_likelihood(self.output, self.real_indices, self.seq_length,
-                                                          self.transition)
-      self.optimizer = tf.train.AdagradOptimizer(self.learning_rate)
-      self.train_ll = self.optimizer.minimize(-self.ll_loss)
     else:
-      # &#26500;&#24314;&#35757;&#32451;&#20989;&#25968;
-      # &#35757;&#32451;&#29992;placeholder
-      self.ll_corr = tf.placeholder(tf.int32, shape=[None, 3])
-      self.ll_curr = tf.placeholder(tf.int32, shape=[None, 3])
-      self.trans_corr = tf.placeholder(tf.int32, [None, 2])
-      self.trans_curr = tf.placeholder(tf.int32, [None, 2])
-      self.trans_init_corr = tf.placeholder(tf.int32, [None, 1])
-      self.trans_init_curr = tf.placeholder(tf.int32, [None, 1])
-      # &#25439;&#22833;&#20989;&#25968;
-      self.loss, self.loss_with_init = self.get_loss()
+      self.loss, _ = tf.contrib.crf.crf_log_likelihood(self.output, self.real_indices, self.seq_length,
+                                                       self.transition)
       self.optimizer = tf.train.AdagradOptimizer(self.learning_rate)
-      self.train = self.optimizer.minimize(self.loss)
-      self.train_with_init = self.optimizer.minimize(self.loss_with_init)
+      self.new_optimizer = tf.train.AdamOptimizer()
+      self.train = self.optimizer.minimize(-self.loss)
 
   def fit(self, epochs: int = 100, interval: int = 20):
-    with tf.Session() as sess:
-      tf.global_variables_initializer().run()
-      saver = tf.train.Saver(max_to_keep=100)
-      for epoch in range(1, epochs + 1):
-        print('epoch:', epoch)
-        for _ in range(self.batch_count):
-          characters, labels, lengths = self.get_batch()
-          self.fit_batch(characters, labels, lengths, sess)
-        # if epoch % interval == 0:
-        model_path = '../dnlp/models/cws{0}.ckpt'.format(epoch)
-        saver.save(sess, model_path)
-        self.save_config(model_path)
-
-  def fit_batch(self, characters, labels, lengths, sess):
-    scores = sess.run(self.output, feed_dict={self.input: characters})
-    transition = self.transition.eval(session=sess)
-    transition_init = self.transition_init.eval(session=sess)
-    update_labels_pos = None
-    update_labels_neg = None
-    current_labels = []
-    trans_pos_indices = []
-    trans_neg_indices = []
-    trans_init_pos_indices = []
-    trans_init_neg_indices = []
-    for i in range(self.batch_size):
-      current_label = self.viterbi(scores[:, :lengths[i], i], transition, transition_init)
-      current_labels.append(current_label)
-      diff_tag = np.subtract(labels[i, :lengths[i]], current_label)
-      update_index = np.where(diff_tag != 0)[0]
-      update_length = len(update_index)
-      if update_length == 0:
-        continue
-      update_label_pos = np.stack([labels[i, update_index], update_index, i * np.ones([update_length])], axis=-1)
-      update_label_neg = np.stack([current_label[update_index], update_index, i * np.ones([update_length])], axis=-1)
-      if update_labels_pos is not None:
-        np.concatenate((update_labels_pos, update_label_pos))
-        np.concatenate((update_labels_neg, update_label_neg))
-      else:
-        update_labels_pos = update_label_pos
-        update_labels_neg = update_label_neg
-
-      trans_pos_index, trans_neg_index, trans_init_pos, trans_init_neg, update_init = self.generate_transition_update_index(
-        labels[i, :lengths[i]], current_labels[i])
-
-      trans_pos_indices.extend(trans_pos_index)
-      trans_neg_indices.extend(trans_neg_index)
-
-      if update_init:
-        trans_init_pos_indices.append(trans_init_pos)
-        trans_init_neg_indices.append(trans_init_neg)
-
-    if update_labels_pos is not None and update_labels_neg is not None:
-      feed_dict = {self.input: characters, self.ll_curr: update_labels_neg, self.ll_corr: update_labels_pos,
-                   self.trans_curr: trans_neg_indices, self.trans_corr: trans_pos_indices}
-
-      if not trans_init_pos_indices:
-        sess.run(self.train, feed_dict)
-      else:
-        feed_dict[self.trans_init_corr] = trans_init_pos_indices
-        feed_dict[self.trans_init_curr] = trans_init_neg_indices
-        sess.run(self.train_with_init, feed_dict)
-
-  def fit_ll(self, epochs: int = 100, interval: int = 20):
     with tf.Session() as sess:
       tf.global_variables_initializer().run()
       saver = tf.train.Saver(max_to_keep=epochs)
@@ -143,44 +72,13 @@ def fit_ll(self, epochs: int = 100, interval: int = 20):
           characters, labels, lengths = self.get_batch()
           # scores = sess.run(self.output, feed_dict={self.input: characters})
           feed_dict = {self.input: characters, self.real_indices: labels, self.seq_length: lengths}
-          sess.run(self.train_ll, feed_dict=feed_dict)
+          sess.run(self.train, feed_dict=feed_dict)
           # self.fit_batch(characters, labels, lengths, sess)
         # if epoch % interval == 0:
         model_path = '../dnlp/models/cws{0}.ckpt'.format(epoch)
         saver.save(sess, model_path)
         self.save_config(model_path)
 
-  def fit_batch_ll(self):
-    pass
-
-  def generate_transition_update_index(self, correct_labels, current_labels):
-    if correct_labels.shape != current_labels.shape:
-      print('sequence length is not equal')
-      return None
-
-    before_corr = correct_labels[0]
-    before_curr = current_labels[0]
-    update_init = False
-
-    trans_init_pos = None
-    trans_init_neg = None
-    trans_pos = []
-    trans_neg = []
-
-    if before_corr != before_curr:
-      trans_init_pos = [before_corr]
-      trans_init_neg = [before_curr]
-      update_init = True
-
-    for _, (corr_label, curr_label) in enumerate(zip(correct_labels[1:], current_labels[1:])):
-      if corr_label != curr_label or before_corr != before_curr:
-        trans_pos.append([before_corr, corr_label])
-        trans_neg.append([before_curr, curr_label])
-      before_corr = corr_label
-      before_curr = curr_label
-
-    return trans_pos, trans_neg, trans_init_pos, trans_init_neg, update_init
-
   def predict(self, sentence: str, return_labels=False):
     if self.mode != 'predict':
       raise Exception('mode is not allowed to predict')
@@ -194,6 +92,22 @@ def predict(self, sentence: str, return_labels=False):
     else:
       return self.tags2words(sentence, labels), self.tag2sequences(labels)
 
+  def predict_ll(self, sentence: str, return_labels=False):
+    if self.mode != 'predict':
+      raise Exception('mode is not allowed to predict')
+
+    input = self.indices2input(self.sentence2indices(sentence))
+    runner = [self.seq, self.best_score, self.output, self.transition]
+    labels, best_score, output, trans = self.sess.run(runner,
+                                                      feed_dict={self.input: input, self.seq_length: [len(sentence)]})
+    # print(output)
+    # print(trans)
+    labels = np.squeeze(labels)
+    if return_labels:
+      return self.tags2words(sentence, labels), self.tag2sequences(labels)
+    else:
+      return self.tags2words(sentence, labels)
+
   def get_embedding_layer(self) -> tf.Tensor:
     embeddings = self.__get_variable([self.dict_size, self.embed_size], 'embeddings')
     self.params.append(embeddings)
@@ -215,19 +129,27 @@ def get_rnn_layer(self, layer: tf.Tensor) -> tf.Tensor:
     rnn = tf.nn.rnn_cell.RNNCell(self.hidden_units)
     rnn_output, rnn_out_state = tf.nn.dynamic_rnn(rnn, layer, dtype=self.dtype)
     self.params += [v for v in tf.global_variables() if v.name.startswith('rnn')]
-    return tf.transpose(rnn_output)
+    return rnn_output
 
   def get_lstm_layer(self, layer: tf.Tensor) -> tf.Tensor:
     lstm = tf.nn.rnn_cell.LSTMCell(self.hidden_units)
     lstm_output, lstm_out_state = tf.nn.dynamic_rnn(lstm, layer, dtype=self.dtype)
     self.params += [v for v in tf.global_variables() if v.name.startswith('rnn')]
     return lstm_output
 
+  def get_bilstm_layer(self, layer: tf.Tensor) -> tf.Tensor:
+    lstm_fw = tf.nn.rnn_cell.LSTMCell(self.hidden_units//2)
+    lstm_bw = tf.nn.rnn_cell.LSTMCell(self.hidden_units//2)
+    bilstm_output, bilstm_output_state = tf.nn.bidirectional_dynamic_rnn(lstm_fw, lstm_bw, layer, self.seq_length,
+                                                                         dtype=self.dtype)
+    self.params += [v for v in tf.global_variables() if v.name.startswith('rnn')]
+    return tf.concat([bilstm_output[0],bilstm_output[1]],-1)
+
   def get_gru_layer(self, layer: tf.Tensor) -> tf.Tensor:
     gru = tf.nn.rnn_cell.GRUCell(self.hidden_units)
     gru_output, gru_out_state = tf.nn.dynamic_rnn(gru, layer, dtype=self.dtype)
     self.params += [v for v in tf.global_variables() if v.name.startswith('rnn')]
-    return tf.transpose(gru_output)
+    return gru_output
 
   def get_dropout_layer(self, layer: tf.Tensor) -> tf.Tensor:
     return tf.layers.dropout(layer, self.dropout_rate)
@@ -238,17 +160,5 @@ def get_output_layer(self, layer: tf.Tensor) -> tf.Tensor:
     self.params += [output_weight, output_bias]
     return tf.tensordot(layer, output_weight, [[2], [0]]) + output_bias
 
-  def get_loss(self) -> (tf.Tensor, tf.Tensor):
-    output_loss = tf.reduce_sum(tf.gather_nd(self.output, self.ll_curr) - tf.gather_nd(self.output, self.ll_corr))
-    trans_loss = tf.gather_nd(self.transition, self.trans_curr) - tf.gather_nd(self.transition, self.trans_corr)
-    trans_i_curr = tf.gather_nd(self.transition_init, self.trans_init_curr)
-    trans_i_corr = tf.gather_nd(self.transition_init, self.trans_init_corr)
-    trans_init_loss = tf.reduce_sum(trans_i_curr - trans_i_corr)
-    loss = output_loss + trans_loss
-    regu = tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(self.lam), self.params)
-    l1 = loss + regu
-    l2 = l1 + trans_init_loss
-    return l1, l2
-
   def __get_variable(self, size, name) -> tf.Variable:
     return tf.Variable(tf.truncated_normal(size, stddev=1.0 / math.sqrt(size[-1]), dtype=self.dtype), name=name)
