refactor: remove special cases

Signed-off-by: jupyterjazz <[email protected]>
docarray · JoanFM · Jun 27, 2023 · Jun 26, 2023 · Jun 26, 2023 · Jun 26, 2023
commit d438dc626d3014887c17615a4fb43a327ec4ad11
diff --git a/docarray/typing/url/any_url.py b/docarray/typing/url/any_url.py
@@ -21,10 +21,7 @@
 
 T = TypeVar('T', bound='AnyUrl')
 
-mime_types_path = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)), '..', 'resources', 'mime.types'
-)
-mimetypes.init([mime_types_path])
+mimetypes.init([])
 
 
 @_register_proto(proto_type_name='any_url')
@@ -68,25 +65,12 @@ def is_extension_allowed(cls, value: Any) -> bool:
             return True
         mimetype, _ = mimetypes.guess_type(value.split("?")[0])
         print('mimetype for value', mimetype, value, value.split("?")[0])
-        if mimetype:
-            return mimetype.startswith(cls.mime_type())
-        else:
-            # check if the extension is among the extra extensions of that class
-            print('extra extensions for value', value, cls.extra_extensions())
-            return any(
-                value.endswith(ext) or value.split("?")[0].endswith(ext)
-                for ext in cls.extra_extensions()
-            )
-
-    @classmethod
-    def is_special_case(cls, value: Any) -> bool:
-        """
-        Check if the url is a special case.
-
-        :param value: url to the file
-        :return: True if the url is a special case, False otherwise
-        """
-        return False
+        if mimetype and mimetype.startswith(cls.mime_type()):
+            return True
+        return any(
+            value.endswith(ext) or value.split("?")[0].endswith(ext)
+            for ext in cls.extra_extensions()
+        )
 
     @classmethod
     def validate(
@@ -112,10 +96,7 @@ def validate(
         url = super().validate(abs_path, field, config)  # basic url validation
 
         if not cls.is_extension_allowed(value):
-            if not cls.is_special_case(value):  # check for special cases
-                raise ValueError(
-                    f'file {value} is not a valid file format for class {cls}'
-                )
+            raise ValueError(f'file {value} is not a valid file format for class {cls}')
 
         return cls(str(value if input_is_relative_path else url), scheme=None)
 

diff --git a/docarray/typing/url/text_url.py b/docarray/typing/url/text_url.py
@@ -24,21 +24,6 @@ def extra_extensions(cls) -> List[str]:
         """
         return ['.md']
 
-    @classmethod
-    def is_special_case(cls, value: 'AnyUrl') -> bool:
-        """
-        Check if the url is a special case that needs to be handled differently.
-
-        :param value: url to the file
-        :return: True if the url is a special case, False otherwise
-        """
-        if value.startswith('http') or value.startswith('https'):
-            if len(value.split('/')[-1].split('.')) == 1:
-                # This handles the case where the value is a URL without a file extension
-                # for e.g. https://de.wikipedia.org/wiki/Brixen
-                return True
-        return False
-
     def load(self, charset: str = 'utf-8', timeout: Optional[float] = None) -> str:
         """
         Load the text file into a string.

diff --git a/tests/integrations/typing/test_typing_proto.py b/tests/integrations/typing/test_typing_proto.py
@@ -34,7 +34,7 @@ class Mymmdoc(BaseDoc):
         embedding=np.zeros((100, 1)),
         any_url='http://jina.ai',
         image_url='http://jina.ai/bla.jpg',
-        text_url='http://jina.ai',
+        text_url='http://jina.ai/jina.txt',
         mesh_url='http://jina.ai/mesh.obj',
         point_cloud_url='http://jina.ai/mesh.obj',
     )