Not raising error when valid tokenizer not found

docs/docs/sdk/resource/checkpoint.mdx

@@ -636,4 +636,3 @@ Convert Hugging Face model checkpoint to PeriFlow format.
 |------|-------------|
 | `NotFoundError` | Raised when `model_name_or_path` is not found. Also raised when `tokenizer_output_dir` is not found. |
 | `CheckpointConversionError` | Raised when given model architecture from checkpoint is not supported to convert. |
-| `TokenizerNotFoundError` | Raised when `tokenizer_output_dir` is not `None` and the model does not have the PeriFlow-compatible tokenizer implementation, which is equivalent to Hugging Face 'fast' tokenizer. Refer to [this link](https://huggingface.co/docs/transformers/main_classes/tokenizer#tokenizer) to get more info. |

periflow/cli/checkpoint.py

@@ -21,7 +21,6 @@
     InvalidPathError,
     NotFoundError,
     NotSupportedError,
-    TokenizerNotFoundError,
 )
 from periflow.formatter import (
     JSONFormatter,
@@ -692,7 +691,7 @@ def convert(
             cache_dir=cache_dir,
             dry_run=dry_run,
         )
-    except (NotFoundError, CheckpointConversionError, TokenizerNotFoundError) as exc:
+    except (NotFoundError, CheckpointConversionError) as exc:
         secho_error_and_exit(str(exc))
 
     msg = (

periflow/cli/main.py

@@ -32,6 +32,7 @@
     context_settings={"help_option_names": ["-h", "--help"]},
     add_completion=False,
     callback=validate_cli_version,
+    pretty_exceptions_enable=False,
 )
 
 app.add_typer(credential.app, name="credential", help="Manage credentials")

periflow/converter/utils.py

@@ -121,15 +121,22 @@ def save_tokenizer(
     if not os.path.isdir(save_dir):
         raise NotFoundError(f"Directory '{save_dir}' is not found.")
 
-    tokenizer = AutoTokenizer.from_pretrained(
-        model_name_or_path,
-        cache_dir=cache_dir,
-        trust_remote_code=True,
-    )
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_name_or_path,
+            cache_dir=cache_dir,
+            trust_remote_code=True,
+        )
+    except OSError as exc:
+        raise TokenizerNotFoundError(str(exc)) from exc
+
     if not tokenizer.is_fast:
-        raise TokenizerNotFoundError
+        raise TokenizerNotFoundError(
+            "This model does not support PeriFlow-compatible tokenizer"
+        )
 
     saved_file_paths = tokenizer.save_pretrained(save_directory=save_dir)
+
     tokenizer_json_path = None
     for path in saved_file_paths:
         if "tokenizer.json" == os.path.basename(path):

periflow/sdk/resource/checkpoint.py

@@ -24,6 +24,7 @@
     NotFoundError,
     NotSupportedCheckpointError,
     PeriFlowInternalError,
+    TokenizerNotFoundError,
 )
 from periflow.logging import logger
 from periflow.schema.resource.v1.checkpoint import V1Checkpoint
@@ -818,7 +819,6 @@ def convert(
         Raises:
             NotFoundError: Raised when `model_name_or_path` is not found. Also raised when `tokenizer_output_dir` is not found.
             CheckpointConversionError: Raised when given model architecture from checkpoint is not supported to convert.
-            TokenizerNotFoundError: Raised when `tokenizer_output_dir` is not `None` and the model does not have the PeriFlow-compatible tokenizer implementation, which is equivalent to Hugging Face 'fast' tokenizer. Refer to [this link](https://huggingface.co/docs/transformers/main_classes/tokenizer#tokenizer) to get more info.
 
         """
         try:
@@ -923,8 +923,11 @@ def convert(
                 yaml.dump(attr, file, sort_keys=False)
 
         if tokenizer_output_dir is not None:
-            save_tokenizer(
-                model_name_or_path=model_name_or_path,
-                cache_dir=cache_dir,
-                save_dir=tokenizer_output_dir,
-            )
+            try:
+                save_tokenizer(
+                    model_name_or_path=model_name_or_path,
+                    cache_dir=cache_dir,
+                    save_dir=tokenizer_output_dir,
+                )
+            except TokenizerNotFoundError as exc:
+                logger.warn(str(exc))