reorg tests

supercoderhawk · supercoderhawk · commit 7b4d43e019a2 · 2019-12-05T09:38:12.000+08:00
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py
@@ -0,0 +1 @@
+# -*- coding: UTF-8 -*-
diff --git a/tests/test_utils/test_tokenizer.py b/tests/test_utils/test_tokenizer.py
@@ -0,0 +1,7 @@
+# -*- coding: UTF-8 -*-
+from deep_keyphrase.utils.tokenizer import token_char_tokenize
+
+
+def test_token_char_tokenize():
+    tokens = token_char_tokenize('1.&#22312;10~11&#20010;&#20043;&#38388;&#12290;')
+    assert tokens == ['<digit>', '.', '&#22312;', '<digit>', '~', '<digit>', '&#20010;', '&#20043;', '&#38388;', '&#12290;']

-Original file line number
+Diff line change
@@ @@ -0,0 +1,7 @@ @@
 +# -*- coding: UTF-8 -*-
 +from deep_keyphrase.utils.tokenizer import token_char_tokenize
++
++
 +def test_token_char_tokenize():
 +    tokens = token_char_tokenize('1.在10~11个之间。')
 +    assert tokens == ['<digit>', '.', '在', '<digit>', '~', '<digit>', '个', '之', '间', '。']