From 28ccfbfc51068a663a80764e14074df5edf2b5ba Mon Sep 17 00:00:00 2001
From: kongdeqiang <kongdeqiang960204@163.com>
Date: 星期五, 13 三月 2026 17:41:41 +0800
Subject: [PATCH] 提交
---
fun_text_processing/text_normalization/token_parser.py | 32 +++++++++-----------------------
1 files changed, 9 insertions(+), 23 deletions(-)
diff --git a/fun_text_processing/text_normalization/token_parser.py b/fun_text_processing/text_normalization/token_parser.py
index 3e39cc3..cb5dfe1 100644
--- a/fun_text_processing/text_normalization/token_parser.py
+++ b/fun_text_processing/text_normalization/token_parser.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import string
from collections import OrderedDict
from typing import Dict, List, Union
@@ -34,7 +20,7 @@
Args:
text: text to be parsed
-
+
"""
self.text = text
self.len_text = len(text)
@@ -88,9 +74,9 @@
if self.char == ":":
self.parse_char(":")
self.parse_ws()
- self.parse_char("\"")
+ self.parse_char('"')
value_string = self.parse_string_value()
- self.parse_char("\"")
+ self.parse_char('"')
return value_string
elif self.char == "{":
d = OrderedDict()
@@ -107,11 +93,11 @@
def parse_char(self, exp) -> bool:
"""
- Parses character
+ Parses character
Args:
exp: character to read in
-
+
Returns true if successful
"""
assert self.char == exp
@@ -124,7 +110,7 @@
Args:
exp: characters to read in
-
+
Returns true if successful
"""
ok = False
@@ -159,7 +145,7 @@
# assert self.char not in string.whitespace and self.char != EOS
assert self.char != EOS
l = []
- while self.char != "\"" or self.text[self.index + 1] != " ":
+ while self.char != '"' or self.text[self.index + 1] != " ":
l.append(self.char)
if not self.read():
raise ValueError()
@@ -181,8 +167,8 @@
def read(self):
"""
- Reads in next char.
-
+ Reads in next char.
+
Returns true if not EOS
"""
if self.index < self.len_text - 1: # should be unique
--
Gitblit v1.9.1