Add default value for tokenizer path (#699)

Co-authored-by: Ammar Ahmad Awan <[email protected]> Co-authored-by: Michael Wyatt <[email protected]>
microsoft · Oct 3, 2023 · 0d11c63 · 0d11c63
1 parent ca03bd7
commit 0d11c63
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/applications/DeepSpeed-Chat/training/utils/utils.py b/applications/DeepSpeed-Chat/training/utils/utils.py
@@ -69,7 +69,8 @@ def load_hf_tokenizer(model_name_or_path, fast_tokenizer=True):
         model_json = os.path.join(model_name_or_path, "config.json")
         if os.path.exists(model_json):
             model_json_file = json.load(open(model_json))
-            model_name = model_json_file["_name_or_path"]
+            model_name = model_json_file.get("_name_or_path",
+                                             model_name_or_path)
             tokenizer = get_tokenizer(model_name,
                                       fast_tokenizer=fast_tokenizer)
     else: