Python: Optional dependencies (microsoft#820)

awharrison-28 · web-flow · commit a9b61b2cf45a · 2023-05-06T03:35:13.000Z
### Motivation and Context
The release of Python SK from 0.2.4 -&gt; 0.2.6 significantly increased the
size of the pip package. The dependencies causing this are not core to
the kernel, so this PR establishes a pattern for SK connector
dependencies (ex: Hugging Face, memory storage)

### Description
- Added a poetry group called hugging_face - this is the same name as
the connectors namespace in the code.
- Added dependency installs to the Hugging Face sample notebook.
- removed Hugging Face dependencies from requirements.txt.
- Added import errors for torch, transformers, and sentence-transformers
in the hugging_face connector classes.
- Added command `--with hugging_face` to integration test setup.
diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml
@@ -36,7 +36,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install poetry pytest
-        cd python && poetry install
+        cd python && poetry install --with hugging_face
     - name: Run Integration Tests
       shell: bash
       env: # Set Azure credentials secret as an input
diff --git a/python/DEV_SETUP.md b/python/DEV_SETUP.md
@@ -68,9 +68,13 @@ Note: SK requires at least Poetry 1.2.0.
 # Install poetry package
 pip3 install poetry
 
-# Use poetry to install project deps
+# Use poetry to install base project dependencies
 poetry install
 
+# If you want to use connectors such as hugging face
+# poetry install --with <connector group name>
+# example: poetry install --with hugging_face
+
 # Use poetry to activate project venv
 poetry shell
 ```
diff --git a/python/poetry.lock b/python/poetry.lock
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -11,9 +11,6 @@ python = "^3.8"
 numpy = "^1.24.2"
 openai = "^0.27.0"
 aiofiles = "^23.1.0"
-transformers = "^4.28.1"
-torch = "^2.0.0"
-sentence-transformers = "^2.2.2"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^2.21.0"
@@ -23,6 +20,12 @@ pytest = "7.2.0"
 ruff = "^0.0.257"
 pytest-asyncio = "^0.21.0"
 
+
+[tool.poetry.group.hugging_face.dependencies]
+torch = "^2.0.0"
+transformers = "^4.28.1"
+sentence-transformers = "^2.2.2"
+
 [tool.isort]
 profile = "black"
 
diff --git a/python/requirements.txt b/python/requirements.txt
@@ -1,6 +1,3 @@
 openai==0.27.0
 numpy==1.24.2
-aiofiles==23.1.0
-transformers==4.28.0
-sentence-transformers==2.2.2
-torch==2.0.0
+aiofiles==23.1.0
diff --git a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py
@@ -3,8 +3,6 @@
 from logging import Logger
 from typing import Optional
 
-import torch
-from transformers import GenerationConfig, pipeline
 
 from semantic_kernel.connectors.ai.ai_exception import AIException
 from semantic_kernel.connectors.ai.complete_request_settings import (
@@ -48,10 +46,17 @@ def __init__(
         self._model_id = model_id
         self._task = "text2text-generation" if task is None else task
         self._log = log if log is not None else NullLogger()
+
+        try:
+            import transformers
+            import torch
+        except (ImportError, ModuleNotFoundError):
+            raise ImportError("Please ensure that torch and transformers are installed to use HuggingFaceTextCompletion")
+        
         self.device = (
             "cuda:" + device if device >= 0 and torch.cuda.is_available() else "cpu"
         )
-        self.generator = pipeline(
+        self.generator = transformers.pipeline(
             task=self._task, model=self._model_id, device=self.device
         )
 
@@ -69,7 +74,8 @@ async def complete_async(
             str -- Completion result.
         """
         try:
-            generation_config = GenerationConfig(
+            import transformers
+            generation_config = transformers.GenerationConfig(
                 temperature=request_settings.temperature,
                 top_p=request_settings.top_p,
                 max_new_tokens=request_settings.max_tokens,
diff --git a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py
@@ -2,10 +2,7 @@
 
 from logging import Logger
 from typing import List, Optional
-
-import torch
 from numpy import array, ndarray
-from sentence_transformers import SentenceTransformer
 
 from semantic_kernel.connectors.ai.ai_exception import AIException
 from semantic_kernel.connectors.ai.embeddings.embedding_generator_base import (
@@ -38,10 +35,17 @@ def __init__(
         """
         self._model_id = model_id
         self._log = log if log is not None else NullLogger()
+
+        try:
+            import torch
+            import sentence_transformers
+        except (ImportError):
+            raise ImportError("Please ensure that torch and sentence-transformers are installed to use HuggingFaceTextEmbedding")
+        
         self.device = (
             "cuda:" + device if device >= 0 and torch.cuda.is_available() else "cpu"
         )
-        self.generator = SentenceTransformer(
+        self.generator = sentence_transformers.SentenceTransformer(
             model_name_or_path=self._model_id, device=self.device
         )
 
diff --git a/samples/notebooks/python/07-hugging-face-for-skills.ipynb b/samples/notebooks/python/07-hugging-face-for-skills.ipynb
@@ -20,7 +20,12 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "!python -m pip install -r requirements.txt"
+        "!python -m pip install -r requirements.txt\n",
+        "\n",
+        "# Note that additional dependencies are required for the Hugging Face connectors:\n",
+        "!python -m pip install torch==2.0.0\n",
+        "!python -m pip install transformers==4.28.1\n",
+        "!python -m pip install sentence-transformers==2.2.2"
       ]
     },
     {

Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,12 @@`
`20`	`20`	`"metadata": {},`
`21`	`21`	`"outputs": [],`
`22`	`22`	`"source": [`
`23`		`- "!python -m pip install -r requirements.txt"`
	`23`	`+ "!python -m pip install -r requirements.txt\n",`
	`24`	`+ "\n",`
	`25`	`+ "# Note that additional dependencies are required for the Hugging Face connectors:\n",`
	`26`	`+ "!python -m pip install torch==2.0.0\n",`
	`27`	`+ "!python -m pip install transformers==4.28.1\n",`
	`28`	`+ "!python -m pip install sentence-transformers==2.2.2"`
`24`	`29`	`]`
`25`	`30`	`},`
`26`	`31`	`{`