update text tokenize

haoneng.lhn

2023-04-25 a88a1d9938eec790aa7333fe6c3d33e5e2067256

1
2
3
4
5
6
export FUNASR_DIR=$PWD/../../..
 
# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
export PYTHONIOENCODING=UTF-8
export PYTHONPATH=../../../:$PYTHONPATH
export PATH=$FUNASR_DIR/funasr/bin:$PATH