johnpccd/image_management_api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix image service to use pub/sub for asynchronous image processing

Browse Source
This commit is contained in:
John MELEK 2025-05-26 20:39:33 +00:00
parent 90f4d4336d
commit e16a2b138a
3 changed files with 27 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
deployment/terraform/terraform.tfstate
View File

@ -1,7 +1,7 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.12.1", "terraform_version": "1.12.1",
"serial": 485, "serial": 488,
"lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5",
"outputs": { "outputs": {
"cloud_function_name": { "cloud_function_name": {
@ -174,7 +174,7 @@
"effective_annotations": { "effective_annotations": {
"run.googleapis.com/ingress": "all", "run.googleapis.com/ingress": "all",
"run.googleapis.com/ingress-status": "all", "run.googleapis.com/ingress-status": "all",
"run.googleapis.com/operation-id": "273b1403-cab6-4397-8714-83b0d8b96d2e", "run.googleapis.com/operation-id": "f9d17943-2f76-476d-8b92-d7305b21bc51",
"run.googleapis.com/urls": "[\"https://contoso-761163285547.us-central1.run.app\",\"https://contoso-p64zpdtkta-uc.a.run.app\"]", "run.googleapis.com/urls": "[\"https://contoso-761163285547.us-central1.run.app\",\"https://contoso-p64zpdtkta-uc.a.run.app\"]",
"serving.knative.dev/creator": "johnpccd3@gmail.com", "serving.knative.dev/creator": "johnpccd3@gmail.com",
"serving.knative.dev/lastModifier": "johnpccd3@gmail.com" "serving.knative.dev/lastModifier": "johnpccd3@gmail.com"
@ -186,12 +186,12 @@
"generation": 1, "generation": 1,
"labels": null, "labels": null,
"namespace": "gen-lang-client-0424120530", "namespace": "gen-lang-client-0424120530",
"resource_version": "AAY2DhNWeIk", "resource_version": "AAY2D+JdakI",
"self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/contoso", "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/contoso",
"terraform_labels": { "terraform_labels": {
"goog-terraform-provisioned": "true" "goog-terraform-provisioned": "true"
}, },
"uid": "5fbbdac4-c4de-4a73-b144-3a2d5206401b" "uid": "7120ad66-df9b-43b7-814b-d03c008523b3"
} }
], ],
"name": "contoso", "name": "contoso",
@ -218,14 +218,14 @@
"type": "RoutesReady" "type": "RoutesReady"
} }
], ],
"latest_created_revision_name": "contoso-00001-t4p", "latest_created_revision_name": "contoso-00001-777",
"latest_ready_revision_name": "contoso-00001-t4p", "latest_ready_revision_name": "contoso-00001-777",
"observed_generation": 1, "observed_generation": 1,
"traffic": [ "traffic": [
{ {
"latest_revision": true, "latest_revision": true,
"percent": 100, "percent": 100,
"revision_name": "contoso-00001-t4p", "revision_name": "contoso-00001-777",
"tag": "", "tag": "",
"url": "" "url": ""
} }
@ -393,7 +393,7 @@
"schema_version": 0, "schema_version": 0,
"attributes": { "attributes": {
"condition": [], "condition": [],
"etag": "BwY2DhPXkYo=", "etag": "BwY2D+KLdMY=",
"id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/contoso/roles/run.invoker/allUsers", "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/contoso/roles/run.invoker/allUsers",
"location": "us-central1", "location": "us-central1",
"member": "allUsers", "member": "allUsers",
@ -801,8 +801,8 @@
"database_edition": "STANDARD", "database_edition": "STANDARD",
"delete_protection_state": "DELETE_PROTECTION_DISABLED", "delete_protection_state": "DELETE_PROTECTION_DISABLED",
"deletion_policy": "ABANDON", "deletion_policy": "ABANDON",
"earliest_version_time": "2025-05-26T17:30:34.654251Z", "earliest_version_time": "2025-05-26T19:34:39.545778Z",
"etag": "IOjogY/hwY0DMLiE6qfXwY0D", "etag": "INXNl4T+wY0DMLiE6qfXwY0D",
"id": "projects/gen-lang-client-0424120530/databases/contoso-imagedb", "id": "projects/gen-lang-client-0424120530/databases/contoso-imagedb",
"key_prefix": "", "key_prefix": "",
"location_id": "us-central1", "location_id": "us-central1",

18
deployment/terraform/terraform.tfstate.backup
View File

@ -1,7 +1,7 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.12.1", "terraform_version": "1.12.1",
"serial": 482, "serial": 485,
"lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5",
"outputs": { "outputs": {
"cloud_function_name": { "cloud_function_name": {
@ -174,7 +174,7 @@
"effective_annotations": { "effective_annotations": {
"run.googleapis.com/ingress": "all", "run.googleapis.com/ingress": "all",
"run.googleapis.com/ingress-status": "all", "run.googleapis.com/ingress-status": "all",
"run.googleapis.com/operation-id": "0425a798-bd0b-4453-83ba-38983e678875", "run.googleapis.com/operation-id": "273b1403-cab6-4397-8714-83b0d8b96d2e",
"run.googleapis.com/urls": "[\"https://contoso-761163285547.us-central1.run.app\",\"https://contoso-p64zpdtkta-uc.a.run.app\"]", "run.googleapis.com/urls": "[\"https://contoso-761163285547.us-central1.run.app\",\"https://contoso-p64zpdtkta-uc.a.run.app\"]",
"serving.knative.dev/creator": "johnpccd3@gmail.com", "serving.knative.dev/creator": "johnpccd3@gmail.com",
"serving.knative.dev/lastModifier": "johnpccd3@gmail.com" "serving.knative.dev/lastModifier": "johnpccd3@gmail.com"
@ -186,12 +186,12 @@
"generation": 1, "generation": 1,
"labels": null, "labels": null,
"namespace": "gen-lang-client-0424120530", "namespace": "gen-lang-client-0424120530",
"resource_version": "AAY2Deo4Uy8", "resource_version": "AAY2DhNWeIk",
"self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/contoso", "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/contoso",
"terraform_labels": { "terraform_labels": {
"goog-terraform-provisioned": "true" "goog-terraform-provisioned": "true"
}, },
"uid": "65c83cd2-1e60-4541-a922-6132aabb9311" "uid": "5fbbdac4-c4de-4a73-b144-3a2d5206401b"
} }
], ],
"name": "contoso", "name": "contoso",
@ -218,14 +218,14 @@
"type": "RoutesReady" "type": "RoutesReady"
} }
], ],
"latest_created_revision_name": "contoso-00001-w6c", "latest_created_revision_name": "contoso-00001-t4p",
"latest_ready_revision_name": "contoso-00001-w6c", "latest_ready_revision_name": "contoso-00001-t4p",
"observed_generation": 1, "observed_generation": 1,
"traffic": [ "traffic": [
{ {
"latest_revision": true, "latest_revision": true,
"percent": 100, "percent": 100,
"revision_name": "contoso-00001-w6c", "revision_name": "contoso-00001-t4p",
"tag": "", "tag": "",
"url": "" "url": ""
} }
@ -393,7 +393,7 @@
"schema_version": 0, "schema_version": 0,
"attributes": { "attributes": {
"condition": [], "condition": [],
"etag": "BwY2DeqBDyE=", "etag": "BwY2DhPXkYo=",
"id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/contoso/roles/run.invoker/allUsers", "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/contoso/roles/run.invoker/allUsers",
"location": "us-central1", "location": "us-central1",
"member": "allUsers", "member": "allUsers",
@ -802,7 +802,7 @@
"delete_protection_state": "DELETE_PROTECTION_DISABLED", "delete_protection_state": "DELETE_PROTECTION_DISABLED",
"deletion_policy": "ABANDON", "deletion_policy": "ABANDON",
"earliest_version_time": "2025-05-26T17:30:34.654251Z", "earliest_version_time": "2025-05-26T17:30:34.654251Z",
"etag": "IJPvj6vewY0DMLiE6qfXwY0D", "etag": "IOjogY/hwY0DMLiE6qfXwY0D",
"id": "projects/gen-lang-client-0424120530/databases/contoso-imagedb", "id": "projects/gen-lang-client-0424120530/databases/contoso-imagedb",
"key_prefix": "", "key_prefix": "",
"location_id": "us-central1", "location_id": "us-central1",

20
src/services/image_service.py
View File

@ -10,7 +10,7 @@ from src.models.user import UserModel
from src.schemas.image import ImageResponse, ImageListResponse from src.schemas.image import ImageResponse, ImageListResponse
from src.db.repositories.image_repository import image_repository from src.db.repositories.image_repository import image_repository
from src.services.storage import StorageService from src.services.storage import StorageService
from src.services.embedding_service import EmbeddingService from src.services.pubsub_service import pubsub_service
from src.utils.authorization import require_team_access, get_team_filter, AuthorizationError from src.utils.authorization import require_team_access, get_team_filter, AuthorizationError
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -20,7 +20,6 @@ class ImageService:
def __init__(self): def __init__(self):
self.storage_service = StorageService() self.storage_service = StorageService()
self.embedding_service = EmbeddingService()
def _generate_api_download_url(self, request: Request, image_id: str) -> str: def _generate_api_download_url(self, request: Request, image_id: str) -> str:
"""Generate API download URL for an image""" """Generate API download URL for an image"""
@ -104,11 +103,15 @@ class ImageService:
logger.error(f"Failed to create image record: {e}") logger.error(f"Failed to create image record: {e}")
raise RuntimeError("Failed to create image record") raise RuntimeError("Failed to create image record")
# Generate embedding asynchronously (fire and forget) # Publish to pub/sub for asynchronous processing
try: try:
await self.embedding_service.generate_image_embedding(str(image.id), file_content) await pubsub_service.publish_image_processing_task(
image_id=str(image.id),
storage_path=storage_path,
team_id=str(user.team_id)
)
except Exception as e: except Exception as e:
logger.warning(f"Failed to generate embedding for image {image.id}: {e}") logger.warning(f"Failed to publish embedding generation task: {e}")
return self._convert_to_response(image, request) return self._convert_to_response(image, request)
@ -318,13 +321,6 @@ class ImageService:
except Exception as e: except Exception as e:
logger.warning(f"Failed to delete file from storage: {e}") logger.warning(f"Failed to delete file from storage: {e}")
# Delete from vector database if it has embeddings
if image.has_embedding and image.embedding_id:
try:
await self.embedding_service.delete_embedding(image.embedding_id)
except Exception as e:
logger.warning(f"Failed to delete embedding: {e}")
# Delete from database # Delete from database
result = await image_repository.delete(obj_id) result = await image_repository.delete(obj_id)
if not result: if not result: