GoogleCloudPlatform · holtskinner · Nov 14, 2024 · Nov 14, 2024 · Nov 14, 2024 · Nov 14, 2024
@@ -15,7 +15,7 @@
 
 from typing import Optional
 
-from google.cloud.aiplatform_v1beta1 import RagCorpus
+from vertexai.preview.rag import RagCorpus
 
 PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
 
@@ -37,7 +37,7 @@ def create_corpus(
     # Initialize Vertex AI API once per session
     vertexai.init(project=PROJECT_ID, location="us-central1")
 
-    # Configure embedding model
+    # Configure embedding model (Optional)
     embedding_model_config = rag.EmbeddingModelConfig(
         publisher_model="publishers/google/models/text-embedding-004"
     )

@@ -0,0 +1,71 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from typing import Optional
+
+from vertexai.preview.rag import RagCorpus
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def create_corpus_feature_store(
+    feature_view_name: str,
+    display_name: Optional[str] = None,
+    description: Optional[str] = None,
+) -> RagCorpus:
+    # [START generativeaionvertexai_rag_create_corpus_feature_store]
+
+    from vertexai.preview import rag
+    import vertexai
+
+    # TODO(developer): Update and un-comment below lines
+    # PROJECT_ID = "your-project-id"
+    # feature_view_name = "projects/{PROJECT_ID}/locations/{LOCATION}/featureOnlineStores/{FEATURE_ONLINE_STORE_ID}/featureViews/{FEATURE_VIEW_ID}"
+    # display_name = "test_corpus"
+    # description = "Corpus Description"
+
+    # Initialize Vertex AI API once per session
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    # Configure embedding model (Optional)
+    embedding_model_config = rag.EmbeddingModelConfig(
+        publisher_model="publishers/google/models/text-embedding-004"
+    )
+
+    # Configure Vector DB
+    vector_db = rag.VertexFeatureStore(resource_name=feature_view_name)
+
+    corpus = rag.create_corpus(
+        display_name=display_name,
+        description=description,
+        embedding_model_config=embedding_model_config,
+        vector_db=vector_db,
+    )
+    print(corpus)
+    # Example response:
+    # RagCorpus(name='projects/1234567890/locations/us-central1/ragCorpora/1234567890',
+    # display_name='test_corpus', description='Corpus Description', embedding_model_config=...
+    # ...
+
+    # [END generativeaionvertexai_rag_create_corpus_feature_store]
+    return corpus
+
+
+if __name__ == "__main__":
+    create_corpus_feature_store(
+        feature_view_name="projects/{PROJECT_ID}/locations/{LOCATION}/featureOnlineStores/{FEATURE_ONLINE_STORE_ID}/featureViews/{FEATURE_VIEW_ID}",
+        display_name="test_corpus",
+        description="Corpus Description",
+    )
@@ -0,0 +1,77 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from typing import Optional
+
+from vertexai.preview.rag import RagCorpus
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def create_corpus_pinecone(
+    pinecone_index_name: str,
+    pinecone_api_key_secret_manager_version: str,
+    display_name: Optional[str] = None,
+    description: Optional[str] = None,
+) -> RagCorpus:
+    # [START generativeaionvertexai_rag_create_corpus_pinecone]
+
+    from vertexai.preview import rag
+    import vertexai
+
+    # TODO(developer): Update and un-comment below lines
+    # PROJECT_ID = "your-project-id"
+    # pinecone_index_name = "pinecone-index-name"
+    # pinecone_api_key_secret_manager_version = "projects/{PROJECT_ID}/secrets/{SECRET_NAME}/versions/latest"
+    # display_name = "test_corpus"
+    # description = "Corpus Description"
+
+    # Initialize Vertex AI API once per session
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    # Configure embedding model (Optional)
+    embedding_model_config = rag.EmbeddingModelConfig(
+        publisher_model="publishers/google/models/text-embedding-004"
+    )
+
+    # Configure Vector DB
+    vector_db = rag.Pinecone(
+        index_name=pinecone_index_name,
+        api_key=pinecone_api_key_secret_manager_version,
+    )
+
+    corpus = rag.create_corpus(
+        display_name=display_name,
+        description=description,
+        embedding_model_config=embedding_model_config,
+        vector_db=vector_db,
+    )
+    print(corpus)
+    # Example response:
+    # RagCorpus(name='projects/1234567890/locations/us-central1/ragCorpora/1234567890',
+    # display_name='test_corpus', description='Corpus Description', embedding_model_config=...
+    # ...
+
+    # [END generativeaionvertexai_rag_create_corpus_pinecone]
+    return corpus
+
+
+if __name__ == "__main__":
+    create_corpus_pinecone(
+        pinecone_index_name="pinecone-index-name",
+        pinecone_api_key_secret_manager_version="projects/{PROJECT_ID}/secrets/{SECRET_NAME}/versions/latest",
+        display_name="test_corpus",
+        description="Corpus Description",
+    )
@@ -0,0 +1,76 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from typing import Optional
+
+from vertexai.preview.rag import RagCorpus
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def create_corpus_vector_search(
+    vector_search_index_name: str,
+    vector_search_index_endpoint_name: str,
+    display_name: Optional[str] = None,
+    description: Optional[str] = None,
+) -> RagCorpus:
+    # [START generativeaionvertexai_rag_create_corpus_vector_search]
+
+    from vertexai.preview import rag
+    import vertexai
+
+    # TODO(developer): Update and un-comment below lines
+    # PROJECT_ID = "your-project-id"
+    # vector_search_index_name = "projects/{PROJECT_ID}/locations/{LOCATION}/indexes/{INDEX_ID}"
+    # vector_search_index_endpoint_name = "projects/{PROJECT_ID}/locations/{LOCATION}/indexEndpoints/{INDEX_ENDPOINT_ID}"
+    # display_name = "test_corpus"
+    # description = "Corpus Description"
+
+    # Initialize Vertex AI API once per session
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    # Configure embedding model (Optional)
+    embedding_model_config = rag.EmbeddingModelConfig(
+        publisher_model="publishers/google/models/text-embedding-004"
+    )
+
+    # Configure Vector DB
+    vector_db = rag.VertexVectorSearch(
+        index=vector_search_index_name, index_endpoint=vector_search_index_endpoint_name
+    )
+
+    corpus = rag.create_corpus(
+        display_name=display_name,
+        description=description,
+        embedding_model_config=embedding_model_config,
+        vector_db=vector_db,
+    )
+    print(corpus)
+    # Example response:
+    # RagCorpus(name='projects/1234567890/locations/us-central1/ragCorpora/1234567890',
+    # display_name='test_corpus', description='Corpus Description', embedding_model_config=...
+    # ...
+
+    # [END generativeaionvertexai_rag_create_corpus_vector_search]
+    return corpus
+
+
+if __name__ == "__main__":
+    create_corpus_vector_search(
+        vector_search_index_name="projects/{PROJECT_ID}/locations/{LOCATION}/indexes/{INDEX_ID}",
+        vector_search_index_endpoint_name="projects/{PROJECT_ID}/locations/{LOCATION}/indexEndpoints/{INDEX_ENDPOINT_ID}",
+        display_name="test_corpus",
+        description="Corpus Description",
+    )
@@ -0,0 +1,81 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from typing import Optional
+
+from vertexai.preview.rag import RagCorpus
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def create_corpus_weaviate(
+    weaviate_http_endpoint: str,
+    weaviate_collection_name: str,
+    weaviate_api_key_secret_manager_version: str,
+    display_name: Optional[str] = None,
+    description: Optional[str] = None,
+) -> RagCorpus:
+    # [START generativeaionvertexai_rag_create_corpus_weaviate]
+
+    from vertexai.preview import rag
+    import vertexai
+
+    # TODO(developer): Update and un-comment below lines
+    # PROJECT_ID = "your-project-id"
+    # weaviate_http_endpoint = "weaviate-http-endpoint"
+    # weaviate_collection_name = "weaviate-collection-name"
+    # weaviate_api_key_secret_manager_version = "projects/{PROJECT_ID}/secrets/{SECRET_NAME}/versions/latest"
+    # display_name = "test_corpus"
+    # description = "Corpus Description"
+
+    # Initialize Vertex AI API once per session
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    # Configure embedding model (Optional)
+    embedding_model_config = rag.EmbeddingModelConfig(
+        publisher_model="publishers/google/models/text-embedding-004"
+    )
+
+    # Configure Vector DB
+    vector_db = rag.Weaviate(
+        weaviate_http_endpoint=weaviate_http_endpoint,
+        collection_name=weaviate_collection_name,
+        api_key=weaviate_api_key_secret_manager_version,
+    )
+
+    corpus = rag.create_corpus(
+        display_name=display_name,
+        description=description,
+        embedding_model_config=embedding_model_config,
+        vector_db=vector_db,
+    )
+    print(corpus)
+    # Example response:
+    # RagCorpus(name='projects/1234567890/locations/us-central1/ragCorpora/1234567890',
+    # display_name='test_corpus', description='Corpus Description', embedding_model_config=...
+    # ...
+
+    # [END generativeaionvertexai_rag_create_corpus_weaviate]
+    return corpus
+
+
+if __name__ == "__main__":
+    create_corpus_weaviate(
+        weaviate_http_endpoint="weaviate-http-endpoint",
+        weaviate_collection_name="weaviate-collection-name",
+        weaviate_api_key_secret_manager_version="projects/{PROJECT_ID}/secrets/{SECRET_NAME}/versions/latest",
+        display_name="test_corpus",
+        description="Corpus Description",
+    )
@@ -45,6 +45,7 @@ def retrieval_query(
         text="Hello World!",
         similarity_top_k=10,  # Optional
         vector_distance_threshold=0.5,  # Optional
+        # vector_search_alpha=0.5, # Optional - Only supported for Weaviate
     )
     print(response)
     # Example response:

@@ -20,6 +20,7 @@
 import vertexai
 
 import create_corpus_example
+import create_corpus_feature_store_example
 import delete_corpus_example
 import delete_file_example
 import generate_content_example
@@ -77,6 +78,17 @@ def test_create_corpus() -> None:
     delete_corpus_example.delete_corpus(corpus.name)
 
 
+def test_create_corpus_feature_store() -> None:
+    FEATURE_ONLINE_STORE_ID = "rag_test_feature_store"
+    FEATURE_VIEW_ID = "rag_test_feature_view"
+    feature_view_name = f"projects/{PROJECT_ID}/locations/{LOCATION}/featureOnlineStores/{FEATURE_ONLINE_STORE_ID}/featureViews/{FEATURE_VIEW_ID}"
+    corpus = create_corpus_feature_store_example.create_corpus_feature_store(
+        feature_view_name,
+    )
+    assert corpus
+    delete_corpus_example.delete_corpus(corpus.name)
+
+
 def test_get_corpus(test_corpus: pytest.fixture) -> None:
     retrieved_corpus = get_corpus_example.get_corpus(test_corpus.name)
     assert retrieved_corpus.name == test_corpus.name