Add sentence start and sentence end tokens

surmenok · Feb 21, 2017 · 967b935 · 967b935
1 parent f5e4c59
commit 967b935
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/textsum_data_convert.py b/textsum_data_convert.py
@@ -59,6 +59,8 @@ def _text_to_vocabulary(input_directories, vocabulary_filename, max_words=200000
   with open(vocabulary_filename, 'w') as writer:
     for word, count in counter.most_common(max_words - 2):
       writer.write(word + ' ' + str(count) + '\n')
+    writer.write('<s> 0\n')
+    writer.write('</s> 0\n')
     writer.write('<UNK> 0\n')
     writer.write('<PAD> 0\n')