Replace Jieba with PyCantonese as Tokenizer

chinesebleu.py

@@ -11,13 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""TODO: Add a description here."""
+"""Chinese BLEU"""
 
 import evaluate
 import datasets
 import math
 from collections import Counter
-import jieba_fast as jieba
+#import jieba_fast as jieba
+import pycantonese
 
 # TODO: Add BibTeX citation
 #_CITATION = """\
@@ -31,7 +32,7 @@ _CITATION = ""
 
 # TODO: Add description of the module here
 _DESCRIPTION = """\
-This evaluation metric is tailor-made to evaluate the translation quality of Chinese translation.
+This evaluation metric is tailor-made to evaluate the translation quality of Chinese translation using customized implementation of BLEU evaluation metric.
 """
 
 
@@ -57,37 +58,26 @@ Examples:
     {'score': 71.89393375176813, 'counts': [9, 7, 5, 4], 'totals': [9, 8, 7, 6], 'bp': 1.0}
 """
 
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
-
-
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class ChineseBLEU(evaluate.Metric):
-    """TODO: Short description of my evaluation module."""
+    """TODO: Chinese BLEU - a BLEU-based metric for Chinese sentences"""
 
     def _info(self):
-        # TODO: Specifies the evaluate.EvaluationModuleInfo object
         return evaluate.MetricInfo(
-            # This is the description that will appear on the modules page.
             module_type="metric",
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,
-            # This defines the format of each prediction and reference
             features=datasets.Features({
                 'predictions': datasets.Value('string'),
                 'references': datasets.Value('string'),
             }),
-            # Homepage of the module for documentation
-            homepage="https://yourappapp.com",
-            # Additional links to the codebase or references
-            codebase_urls=["https://github.com/shivanraptor/chinesebleu"],
-            reference_urls=["http://path.to.reference.url/new_module"]
+            homepage="https://github.com/shivanraptor/chinesebleu",
+            codebase_urls=["https://github.com/shivanraptor/chinesebleu"]
         )
 
     def _download_and_prepare(self, dl_manager):
-        """Optional: download external resources useful to compute the scores"""
-        # TODO: Download external resources if needed
+        """No extra files required to download, pass"""
         pass
 
     def _tokenize_chinese(self, sentence, tokenizer='char'):
@@ -102,7 +92,8 @@ class ChineseBLEU(evaluate.Metric):
             list: List of tokens.
         """
         if tokenizer == 'chinese':
-            return list(jieba.cut(sentence, cut_all=False))
+            #return list(jieba.cut(sentence, cut_all=False))
+            return pycantonese.segment(sentence)
         else:
             return list(sentence)  # Character-level tokenization
 
@@ -146,6 +137,12 @@ class ChineseBLEU(evaluate.Metric):
         
         pred_tokens = [self._tokenize_chinese(p, tokenizer) for p in predictions]
         ref_tokens = [self._tokenize_chinese(r, tokenizer) for r in references]
+
+        # For total number of tokens < 4, fallback to character-level tokenizations
+        if len(pred_tokens) < 4 or len(ref_tokens) < 4:
+            tokenizer = 'character'
+            pred_tokens = [self._tokenize_chinese(p, tokenizer) for p in predictions]
+            ref_tokens = [self._tokenize_chinese(r, tokenizer) for r in references]
     
         #print("Prediction Tokens:", pred_tokens[0])
         #print("Reference Tokens:", ref_tokens[0])