Fixed pender/chatbot-rnn pender#44

egg82 · Mar 23, 2019 · d65bceb · d65bceb
1 parent 35e4042
commit d65bceb
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/utils.py b/utils.py
@@ -102,8 +102,8 @@ def _load_vocab(self, vocab_file):
         self.vocab = dict(zip(self.chars, range(len(self.chars))))
 
     def _preprocess(self, input_file, tensor_file):
-        if input_file.endswith(".bz2"): file_reference = bz2.open(input_file, mode='rt')
-        elif input_file.endswith(".txt"): file_reference = io.open(input_file, mode='rt')
+        if input_file.endswith(".bz2"): file_reference = bz2.open(input_file, mode='rt', encoding='utf8')
+        elif input_file.endswith(".txt"): file_reference = io.open(input_file, mode='rt', encoding='utf8')
         data = file_reference.read()
         file_reference.close()
         # Convert the entirety of the data file from characters to indices via the vocab dictionary.