From e16a2b138a63d890301d8a3805a6e264f668ccff Mon Sep 17 00:00:00 2001 From: John MELEK Date: Mon, 26 May 2025 20:39:33 +0000 Subject: [PATCH] Fix image service to use pub/sub for asynchronous image processing --- deployment/terraform/terraform.tfstate | 20 +++++++++---------- deployment/terraform/terraform.tfstate.backup | 18 ++++++++--------- src/services/image_service.py | 20 ++++++++----------- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/deployment/terraform/terraform.tfstate b/deployment/terraform/terraform.tfstate index cd39785..0b03d56 100644 --- a/deployment/terraform/terraform.tfstate +++ b/deployment/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.12.1", - "serial": 485, + "serial": 488, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { "cloud_function_name": { @@ -174,7 +174,7 @@ "effective_annotations": { "run.googleapis.com/ingress": "all", "run.googleapis.com/ingress-status": "all", - "run.googleapis.com/operation-id": "273b1403-cab6-4397-8714-83b0d8b96d2e", + "run.googleapis.com/operation-id": "f9d17943-2f76-476d-8b92-d7305b21bc51", "run.googleapis.com/urls": "[\"https://contoso-761163285547.us-central1.run.app\",\"https://contoso-p64zpdtkta-uc.a.run.app\"]", "serving.knative.dev/creator": "johnpccd3@gmail.com", "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" @@ -186,12 +186,12 @@ "generation": 1, "labels": null, "namespace": "gen-lang-client-0424120530", - "resource_version": "AAY2DhNWeIk", + "resource_version": "AAY2D+JdakI", "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/contoso", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "5fbbdac4-c4de-4a73-b144-3a2d5206401b" + "uid": "7120ad66-df9b-43b7-814b-d03c008523b3" } ], "name": "contoso", @@ -218,14 +218,14 @@ "type": "RoutesReady" } ], - "latest_created_revision_name": "contoso-00001-t4p", - "latest_ready_revision_name": "contoso-00001-t4p", + "latest_created_revision_name": "contoso-00001-777", + "latest_ready_revision_name": "contoso-00001-777", "observed_generation": 1, "traffic": [ { "latest_revision": true, "percent": 100, - "revision_name": "contoso-00001-t4p", + "revision_name": "contoso-00001-777", "tag": "", "url": "" } @@ -393,7 +393,7 @@ "schema_version": 0, "attributes": { "condition": [], - "etag": "BwY2DhPXkYo=", + "etag": "BwY2D+KLdMY=", "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/contoso/roles/run.invoker/allUsers", "location": "us-central1", "member": "allUsers", @@ -801,8 +801,8 @@ "database_edition": "STANDARD", "delete_protection_state": "DELETE_PROTECTION_DISABLED", "deletion_policy": "ABANDON", - "earliest_version_time": "2025-05-26T17:30:34.654251Z", - "etag": "IOjogY/hwY0DMLiE6qfXwY0D", + "earliest_version_time": "2025-05-26T19:34:39.545778Z", + "etag": "INXNl4T+wY0DMLiE6qfXwY0D", "id": "projects/gen-lang-client-0424120530/databases/contoso-imagedb", "key_prefix": "", "location_id": "us-central1", diff --git a/deployment/terraform/terraform.tfstate.backup b/deployment/terraform/terraform.tfstate.backup index fa76fd0..cd39785 100644 --- a/deployment/terraform/terraform.tfstate.backup +++ b/deployment/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.12.1", - "serial": 482, + "serial": 485, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { "cloud_function_name": { @@ -174,7 +174,7 @@ "effective_annotations": { "run.googleapis.com/ingress": "all", "run.googleapis.com/ingress-status": "all", - "run.googleapis.com/operation-id": "0425a798-bd0b-4453-83ba-38983e678875", + "run.googleapis.com/operation-id": "273b1403-cab6-4397-8714-83b0d8b96d2e", "run.googleapis.com/urls": "[\"https://contoso-761163285547.us-central1.run.app\",\"https://contoso-p64zpdtkta-uc.a.run.app\"]", "serving.knative.dev/creator": "johnpccd3@gmail.com", "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" @@ -186,12 +186,12 @@ "generation": 1, "labels": null, "namespace": "gen-lang-client-0424120530", - "resource_version": "AAY2Deo4Uy8", + "resource_version": "AAY2DhNWeIk", "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/contoso", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "65c83cd2-1e60-4541-a922-6132aabb9311" + "uid": "5fbbdac4-c4de-4a73-b144-3a2d5206401b" } ], "name": "contoso", @@ -218,14 +218,14 @@ "type": "RoutesReady" } ], - "latest_created_revision_name": "contoso-00001-w6c", - "latest_ready_revision_name": "contoso-00001-w6c", + "latest_created_revision_name": "contoso-00001-t4p", + "latest_ready_revision_name": "contoso-00001-t4p", "observed_generation": 1, "traffic": [ { "latest_revision": true, "percent": 100, - "revision_name": "contoso-00001-w6c", + "revision_name": "contoso-00001-t4p", "tag": "", "url": "" } @@ -393,7 +393,7 @@ "schema_version": 0, "attributes": { "condition": [], - "etag": "BwY2DeqBDyE=", + "etag": "BwY2DhPXkYo=", "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/contoso/roles/run.invoker/allUsers", "location": "us-central1", "member": "allUsers", @@ -802,7 +802,7 @@ "delete_protection_state": "DELETE_PROTECTION_DISABLED", "deletion_policy": "ABANDON", "earliest_version_time": "2025-05-26T17:30:34.654251Z", - "etag": "IJPvj6vewY0DMLiE6qfXwY0D", + "etag": "IOjogY/hwY0DMLiE6qfXwY0D", "id": "projects/gen-lang-client-0424120530/databases/contoso-imagedb", "key_prefix": "", "location_id": "us-central1", diff --git a/src/services/image_service.py b/src/services/image_service.py index ce14690..6d1aa15 100644 --- a/src/services/image_service.py +++ b/src/services/image_service.py @@ -10,7 +10,7 @@ from src.models.user import UserModel from src.schemas.image import ImageResponse, ImageListResponse from src.db.repositories.image_repository import image_repository from src.services.storage import StorageService -from src.services.embedding_service import EmbeddingService +from src.services.pubsub_service import pubsub_service from src.utils.authorization import require_team_access, get_team_filter, AuthorizationError logger = logging.getLogger(__name__) @@ -20,7 +20,6 @@ class ImageService: def __init__(self): self.storage_service = StorageService() - self.embedding_service = EmbeddingService() def _generate_api_download_url(self, request: Request, image_id: str) -> str: """Generate API download URL for an image""" @@ -104,11 +103,15 @@ class ImageService: logger.error(f"Failed to create image record: {e}") raise RuntimeError("Failed to create image record") - # Generate embedding asynchronously (fire and forget) + # Publish to pub/sub for asynchronous processing try: - await self.embedding_service.generate_image_embedding(str(image.id), file_content) + await pubsub_service.publish_image_processing_task( + image_id=str(image.id), + storage_path=storage_path, + team_id=str(user.team_id) + ) except Exception as e: - logger.warning(f"Failed to generate embedding for image {image.id}: {e}") + logger.warning(f"Failed to publish embedding generation task: {e}") return self._convert_to_response(image, request) @@ -318,13 +321,6 @@ class ImageService: except Exception as e: logger.warning(f"Failed to delete file from storage: {e}") - # Delete from vector database if it has embeddings - if image.has_embedding and image.embedding_id: - try: - await self.embedding_service.delete_embedding(image.embedding_id) - except Exception as e: - logger.warning(f"Failed to delete embedding: {e}") - # Delete from database result = await image_repository.delete(obj_id) if not result: