update tokenizer.h

雾聪

2023-11-09 b0c108279275a5c0011dfcf76b3e725b3263209b

 funasr/bin/lm_inference_launch.py

@@ -183,6 +183,7 @@
            dtype=dtype,
            batch_size=batch_size,
            key_file=key_file,
            preprocess_fn=preprocessor,
            num_workers=num_workers,
        )