From 38f218691dca97fac8b78de8861b39f479b85e96 Mon Sep 17 00:00:00 2001 From: johnpccd Date: Sun, 25 May 2025 13:23:52 +0200 Subject: [PATCH] fix generating text embeddings from searhc query --- src/services/embedding_service.py | 98 +++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 26 deletions(-) diff --git a/src/services/embedding_service.py b/src/services/embedding_service.py index 4cd3a13..b142bd4 100644 --- a/src/services/embedding_service.py +++ b/src/services/embedding_service.py @@ -4,37 +4,52 @@ import os from typing import List, Dict, Any, Union, Optional import numpy as np from PIL import Image +import vertexai +from vertexai.vision_models import MultiModalEmbeddingModel from src.config.config import settings logger = logging.getLogger(__name__) class EmbeddingService: - """Service for generating image and text embeddings""" + """Service for generating image and text embeddings using Vertex AI multimodal model""" def __init__(self): self.model = None - self.processor = None - self.model_name = "openai/clip-vit-base-patch32" - self.device = "cpu" # Simplified without PyTorch - self.embedding_dim = 512 # Dimension of CLIP's embeddings + self.model_name = "multimodalembedding@001" + self.embedding_dim = 1408 # Vertex AI multimodal embedding dimensions + self._initialize_vertex_ai() + + def _initialize_vertex_ai(self): + """Initialize Vertex AI""" + try: + project_id = os.environ.get('GOOGLE_CLOUD_PROJECT') or settings.FIRESTORE_PROJECT_ID + location = os.environ.get('VERTEX_AI_LOCATION', 'us-central1') + + if project_id: + vertexai.init(project=project_id, location=location) + logger.info(f"Initialized Vertex AI with project {project_id} in location {location}") + else: + logger.error("PROJECT_ID not found in environment variables") + raise ValueError("Google Cloud Project ID not configured") + except Exception as e: + logger.error(f"Error initializing Vertex AI: {e}") + raise def _load_model(self): - """ - Load the embedding model if not already loaded - """ + """Load the Vertex AI multimodal embedding model""" if self.model is None: try: - logger.info(f"Loading embedding model on {self.device}") - # Placeholder for model loading logic - logger.info("Embedding model loaded successfully") + logger.info(f"Loading Vertex AI multimodal embedding model: {self.model_name}") + self.model = MultiModalEmbeddingModel.from_pretrained(self.model_name) + logger.info("Vertex AI multimodal embedding model loaded successfully") except Exception as e: - logger.error(f"Error loading embedding model: {e}") + logger.error(f"Error loading Vertex AI model: {e}") raise def generate_image_embedding(self, image_data: bytes) -> List[float]: """ - Generate embedding for an image + Generate embedding for an image using Vertex AI multimodal model Args: image_data: Binary image data @@ -45,22 +60,38 @@ class EmbeddingService: try: self._load_model() - # Load the image - image = Image.open(io.BytesIO(image_data)) + # Create Vertex AI image object + from vertexai.vision_models import Image as VertexImage + vertex_image = VertexImage(image_data) - # Placeholder for image embedding generation - # Returns a random normalized vector as placeholder - embedding = np.random.randn(self.embedding_dim).astype(np.float32) - embedding = embedding / np.linalg.norm(embedding) + # Generate image embedding using Vertex AI multimodal model + embeddings = self.model.get_embeddings(image=vertex_image) + + if embeddings is None or embeddings.image_embedding is None: + logger.error("Failed to generate image embeddings - no image embedding returned") + raise ValueError("Failed to generate image embeddings") + + # Get the image embedding vector + embedding_vector = embeddings.image_embedding + + # Convert to numpy array and normalize + embeddings_array = np.array(embedding_vector, dtype=np.float32) + + # Normalize the feature vector + norm = np.linalg.norm(embeddings_array) + if norm > 0: + embeddings_array = embeddings_array / norm + + logger.info(f"Generated image embeddings with shape: {embeddings_array.shape}") + return embeddings_array.tolist() - return embedding.tolist() except Exception as e: logger.error(f"Error generating image embedding: {e}") raise async def generate_text_embedding(self, text: str) -> Optional[List[float]]: """ - Generate embedding for a text query + Generate embedding for a text query using Vertex AI multimodal model Args: text: Text query @@ -71,12 +102,27 @@ class EmbeddingService: try: self._load_model() - # Placeholder for text embedding generation - # Returns a random normalized vector as placeholder - embedding = np.random.randn(self.embedding_dim).astype(np.float32) - embedding = embedding / np.linalg.norm(embedding) + # Generate text embedding using Vertex AI multimodal model + embeddings = self.model.get_embeddings(contextual_text=text) + + if embeddings is None or embeddings.text_embedding is None: + logger.error("Failed to generate text embeddings - no text embedding returned") + return None + + # Get the text embedding vector + embedding_vector = embeddings.text_embedding + + # Convert to numpy array and normalize + embeddings_array = np.array(embedding_vector, dtype=np.float32) + + # Normalize the feature vector + norm = np.linalg.norm(embeddings_array) + if norm > 0: + embeddings_array = embeddings_array / norm + + logger.info(f"Generated text embeddings with shape: {embeddings_array.shape}") + return embeddings_array.tolist() - return embedding.tolist() except Exception as e: logger.error(f"Error generating text embedding: {e}") return None