fix related to issue found in editquality repository tests

.gitignore

@@ -16,6 +16,7 @@ datasets/
 models/*.model
 
 # Distribution / packaging
+.vscode/
 .Python
 env/
 bin/

revscoring/features/wikitext/datasources/tokenized.py

@@ -455,7 +455,7 @@ def tokenized(text_datasource, name=None, tok_strategy="Latin"):
     Constructs a :class:`revision.Datasource` that generates a list of tokens
     """
     if name is None:
-        name = "{0}({1})".format("tokenized", text_datasource)
+        name = "{0}({1!r}, {2!r})".format("tokenized", text_datasource, tok_strategy)
 
     if tok_strategy == "Latin":
         return Datasource(

tests/features/wikitext/tests/test_tokenized.py

@@ -240,3 +240,24 @@ def test_cjk_tokens_features():
 def test_tokens_diff_features():
     assert (solve(revision.diff.token_delta_increase, cache={r_text: text, p_text: p_text_text}) == 0)
     assert (solve(revision.diff.token_delta_decrease, cache={r_text: text, p_text: p_text_text}) == -4)
+
+
+# related to https://github.com/wikimedia/editquality/pull/232
+def test_cjk_tokenization_naming_01():
+    r_text = revision_oriented.revision.text
+    r_text_text = 'れた'
+
+    cache = {r_text: r_text_text}
+
+    assert (list(solve([revision.cjk_chars, revision.cjk.tokens], cache=cache)) ==
+                [2, 2.0])
+
+
+def test_cjk_tokenization_naming_02():
+    r_text = revision_oriented.revision.text
+    r_text_text = 'れた'
+
+    cache = {r_text: r_text_text}
+
+    assert (list(solve([revision.cjk.tokens, revision.cjk_chars], cache=cache)) ==
+                [2.0, 2])