diff --git a/deployment/terraform/terraform.tfstate b/deployment/terraform/terraform.tfstate index 17f8afb..e1e0ef8 100644 --- a/deployment/terraform/terraform.tfstate +++ b/deployment/terraform/terraform.tfstate @@ -1,13 +1,38 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 101, + "serial": 112, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { + "cloud_run_qdrant_host": { + "value": "34.171.134.17", + "type": "string" + }, + "cloud_run_url": { + "value": "https://sereact-p64zpdtkta-uc.a.run.app", + "type": "string" + }, "container_registry_url": { "value": "gcr.io/gen-lang-client-0424120530/sereact", "type": "string" }, + "deployment_summary": { + "value": { + "cloud_run_url": "https://sereact-p64zpdtkta-uc.a.run.app", + "firestore_database": "sereact-imagedb", + "qdrant_endpoint": "http://34.171.134.17:6333", + "storage_bucket": "sereact-images" + }, + "type": [ + "object", + { + "cloud_run_url": "string", + "firestore_database": "string", + "qdrant_endpoint": "string", + "storage_bucket": "string" + } + ] + }, "firestore_database_id": { "value": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", "type": "string" @@ -90,7 +115,6 @@ "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", "instances": [ { - "status": "tainted", "schema_version": 2, "attributes": { "autogenerate_revision_name": false, @@ -102,25 +126,67 @@ "run.googleapis.com/ingress": "all" }, "effective_annotations": { - "run.googleapis.com/ingress": "all" + "run.googleapis.com/ingress": "all", + "run.googleapis.com/ingress-status": "all", + "run.googleapis.com/operation-id": "c24190ff-2746-46ef-a1e0-ad57d23a041c", + "run.googleapis.com/urls": "[\"https://sereact-761163285547.us-central1.run.app\",\"https://sereact-p64zpdtkta-uc.a.run.app\"]", + "serving.knative.dev/creator": "johnpccd3@gmail.com", + "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" }, "effective_labels": { + "cloud.googleapis.com/location": "us-central1", "goog-terraform-provisioned": "true" }, - "generation": 0, - "labels": null, - "namespace": "", - "resource_version": "", - "self_link": "", + "generation": 1, + "labels": {}, + "namespace": "gen-lang-client-0424120530", + "resource_version": "AAY15HPUh+Y", + "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/sereact", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "" + "uid": "8d376071-2aed-4903-80df-4ea8a4ce8708" } ], "name": "sereact", "project": "gen-lang-client-0424120530", - "status": null, + "status": [ + { + "conditions": [ + { + "message": "", + "reason": "", + "status": "True", + "type": "Ready" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "ConfigurationsReady" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "RoutesReady" + } + ], + "latest_created_revision_name": "sereact-00001-lvg", + "latest_ready_revision_name": "sereact-00001-lvg", + "observed_generation": 1, + "traffic": [ + { + "latest_revision": true, + "percent": 100, + "revision_name": "sereact-00001-lvg", + "tag": "", + "url": "" + } + ], + "url": "https://sereact-p64zpdtkta-uc.a.run.app" + } + ], "template": [ { "metadata": [ @@ -129,7 +195,9 @@ "autoscaling.knative.dev/maxScale": "10" }, "generation": 0, - "labels": {}, + "labels": { + "run.googleapis.com/startupProbeType": "Default" + }, "name": "", "namespace": "", "resource_version": "", @@ -139,11 +207,11 @@ ], "spec": [ { - "container_concurrency": 0, + "container_concurrency": 80, "containers": [ { - "args": null, - "command": null, + "args": [], + "command": [], "env": [ { "name": "FIRESTORE_DATABASE_NAME", @@ -165,6 +233,21 @@ "value": "INFO", "value_from": [] }, + { + "name": "QDRANT_API_KEY", + "value": "", + "value_from": [] + }, + { + "name": "QDRANT_HOST", + "value": "34.171.134.17", + "value_from": [] + }, + { + "name": "QDRANT_PORT", + "value": "6333", + "value_from": [] + }, { "name": "VECTOR_DB_ENVIRONMENT", "value": "gcp-starter", @@ -183,7 +266,7 @@ "ports": [ { "container_port": 8000, - "name": "", + "name": "http1", "protocol": "" } ], @@ -193,18 +276,32 @@ "cpu": "1", "memory": "1Gi" }, - "requests": null + "requests": {} + } + ], + "startup_probe": [ + { + "failure_threshold": 1, + "grpc": [], + "http_get": [], + "initial_delay_seconds": 0, + "period_seconds": 240, + "tcp_socket": [ + { + "port": 8000 + } + ], + "timeout_seconds": 240 } ], - "startup_probe": [], "volume_mounts": [], "working_dir": "" } ], - "node_selector": null, - "service_account_name": "", + "node_selector": {}, + "service_account_name": "761163285547-compute@developer.gserviceaccount.com", "serving_state": "", - "timeout_seconds": 0, + "timeout_seconds": 300, "volumes": [] } ] @@ -221,10 +318,83 @@ } ] }, - "sensitive_attributes": [], + "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "template" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "spec" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "containers" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "env" + } + ] + ], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMiJ9", "dependencies": [ - "google_project_service.services" + "google_compute_address.vector_db_static_ip", + "google_compute_instance.vector_db_vm", + "google_project_service.services", + "google_service_account.vector_db_sa" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_cloud_run_service_iam_member", + "name": "public_access", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwY15HR3kyI=", + "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact/roles/run.invoker/allUsers", + "location": "us-central1", + "member": "allUsers", + "project": "gen-lang-client-0424120530", + "role": "roles/run.invoker", + "service": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "google_cloud_run_service.sereact", + "google_compute_address.vector_db_static_ip", + "google_compute_instance.vector_db_vm", + "google_project_service.services", + "google_service_account.vector_db_sa" ] } ] @@ -479,18 +649,7 @@ [ { "type": "get_attr", - "value": "boot_disk" - }, - { - "type": "index", - "value": { - "value": 0, - "type": "number" - } - }, - { - "type": "get_attr", - "value": "disk_encryption_key_rsa" + "value": "metadata_startup_script" } ], [ @@ -513,7 +672,18 @@ [ { "type": "get_attr", - "value": "metadata_startup_script" + "value": "boot_disk" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "disk_encryption_key_rsa" } ] ], @@ -525,13 +695,6 @@ } ] }, - { - "mode": "managed", - "type": "google_compute_instance", - "name": "vector_db_vm_with_static_ip", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [] - }, { "mode": "managed", "type": "google_firestore_database", @@ -549,7 +712,7 @@ "delete_protection_state": "DELETE_PROTECTION_DISABLED", "deletion_policy": "ABANDON", "earliest_version_time": "2025-05-24T16:11:39.875416Z", - "etag": "II2+yMLBvI0DMNiAssK/vI0D", + "etag": "INqrga/HvI0DMNiAssK/vI0D", "id": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", "key_prefix": "", "location_id": "us-central1", diff --git a/deployment/terraform/terraform.tfstate.backup b/deployment/terraform/terraform.tfstate.backup index 751a634..a0f1ea0 100644 --- a/deployment/terraform/terraform.tfstate.backup +++ b/deployment/terraform/terraform.tfstate.backup @@ -1,13 +1,38 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 98, + "serial": 111, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { + "cloud_run_qdrant_host": { + "value": "34.171.134.17", + "type": "string" + }, + "cloud_run_url": { + "value": "https://sereact-p64zpdtkta-uc.a.run.app", + "type": "string" + }, "container_registry_url": { "value": "gcr.io/gen-lang-client-0424120530/sereact", "type": "string" }, + "deployment_summary": { + "value": { + "cloud_run_url": "https://sereact-p64zpdtkta-uc.a.run.app", + "firestore_database": "sereact-imagedb", + "qdrant_endpoint": "http://34.171.134.17:6333", + "storage_bucket": "sereact-images" + }, + "type": [ + "object", + { + "cloud_run_url": "string", + "firestore_database": "string", + "qdrant_endpoint": "string", + "storage_bucket": "string" + } + ] + }, "firestore_database_id": { "value": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", "type": "string" @@ -90,7 +115,6 @@ "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", "instances": [ { - "status": "tainted", "schema_version": 2, "attributes": { "autogenerate_revision_name": false, @@ -102,25 +126,67 @@ "run.googleapis.com/ingress": "all" }, "effective_annotations": { - "run.googleapis.com/ingress": "all" + "run.googleapis.com/ingress": "all", + "run.googleapis.com/ingress-status": "all", + "run.googleapis.com/operation-id": "c24190ff-2746-46ef-a1e0-ad57d23a041c", + "run.googleapis.com/urls": "[\"https://sereact-761163285547.us-central1.run.app\",\"https://sereact-p64zpdtkta-uc.a.run.app\"]", + "serving.knative.dev/creator": "johnpccd3@gmail.com", + "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" }, "effective_labels": { + "cloud.googleapis.com/location": "us-central1", "goog-terraform-provisioned": "true" }, - "generation": 0, + "generation": 1, "labels": null, - "namespace": "", - "resource_version": "", - "self_link": "", + "namespace": "gen-lang-client-0424120530", + "resource_version": "AAY15HPUh+Y", + "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/sereact", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "" + "uid": "8d376071-2aed-4903-80df-4ea8a4ce8708" } ], "name": "sereact", "project": "gen-lang-client-0424120530", - "status": null, + "status": [ + { + "conditions": [ + { + "message": "", + "reason": "", + "status": "True", + "type": "Ready" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "ConfigurationsReady" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "RoutesReady" + } + ], + "latest_created_revision_name": "sereact-00001-lvg", + "latest_ready_revision_name": "sereact-00001-lvg", + "observed_generation": 1, + "traffic": [ + { + "latest_revision": true, + "percent": 100, + "revision_name": "sereact-00001-lvg", + "tag": "", + "url": "" + } + ], + "url": "https://sereact-p64zpdtkta-uc.a.run.app" + } + ], "template": [ { "metadata": [ @@ -129,7 +195,9 @@ "autoscaling.knative.dev/maxScale": "10" }, "generation": 0, - "labels": {}, + "labels": { + "run.googleapis.com/startupProbeType": "Default" + }, "name": "", "namespace": "", "resource_version": "", @@ -139,7 +207,7 @@ ], "spec": [ { - "container_concurrency": 0, + "container_concurrency": 80, "containers": [ { "args": null, @@ -165,6 +233,21 @@ "value": "INFO", "value_from": [] }, + { + "name": "QDRANT_API_KEY", + "value": "", + "value_from": [] + }, + { + "name": "QDRANT_HOST", + "value": "34.171.134.17", + "value_from": [] + }, + { + "name": "QDRANT_PORT", + "value": "6333", + "value_from": [] + }, { "name": "VECTOR_DB_ENVIRONMENT", "value": "gcp-starter", @@ -183,7 +266,7 @@ "ports": [ { "container_port": 8000, - "name": "", + "name": "http1", "protocol": "" } ], @@ -196,15 +279,29 @@ "requests": null } ], - "startup_probe": [], + "startup_probe": [ + { + "failure_threshold": 1, + "grpc": [], + "http_get": [], + "initial_delay_seconds": 0, + "period_seconds": 240, + "tcp_socket": [ + { + "port": 8000 + } + ], + "timeout_seconds": 240 + } + ], "volume_mounts": [], "working_dir": "" } ], "node_selector": null, - "service_account_name": "", + "service_account_name": "761163285547-compute@developer.gserviceaccount.com", "serving_state": "", - "timeout_seconds": 0, + "timeout_seconds": 300, "volumes": [] } ] @@ -221,10 +318,83 @@ } ] }, - "sensitive_attributes": [], + "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "template" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "spec" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "containers" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "env" + } + ] + ], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMiJ9", "dependencies": [ - "google_project_service.services" + "google_compute_address.vector_db_static_ip", + "google_compute_instance.vector_db_vm", + "google_project_service.services", + "google_service_account.vector_db_sa" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_cloud_run_service_iam_member", + "name": "public_access", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwY15HR3kyI=", + "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact/roles/run.invoker/allUsers", + "location": "us-central1", + "member": "allUsers", + "project": "gen-lang-client-0424120530", + "role": "roles/run.invoker", + "service": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "google_cloud_run_service.sereact", + "google_compute_address.vector_db_static_ip", + "google_compute_instance.vector_db_vm", + "google_project_service.services", + "google_service_account.vector_db_sa" ] } ] @@ -525,13 +695,6 @@ } ] }, - { - "mode": "managed", - "type": "google_compute_instance", - "name": "vector_db_vm_with_static_ip", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [] - }, { "mode": "managed", "type": "google_firestore_database", @@ -549,7 +712,7 @@ "delete_protection_state": "DELETE_PROTECTION_DISABLED", "deletion_policy": "ABANDON", "earliest_version_time": "2025-05-24T16:11:39.875416Z", - "etag": "INWJlu/AvI0DMNiAssK/vI0D", + "etag": "II+lnpPHvI0DMNiAssK/vI0D", "id": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", "key_prefix": "", "location_id": "us-central1", diff --git a/src/config/config.py b/src/config/config.py index efecccc..a866a54 100644 --- a/src/config/config.py +++ b/src/config/config.py @@ -53,6 +53,8 @@ class Settings(BaseSettings): QDRANT_PORT: int = int(os.getenv("QDRANT_PORT", "6333")) QDRANT_API_KEY: str = os.getenv("QDRANT_API_KEY", "") QDRANT_COLLECTION: str = os.getenv("QDRANT_COLLECTION", "image_vectors") + QDRANT_HTTPS: bool = os.getenv("QDRANT_HTTPS", "").lower() in ("true", "1", "yes") + QDRANT_PREFER_GRPC: bool = os.getenv("QDRANT_PREFER_GRPC", "false").lower() in ("true", "1", "yes") # Logging LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO") diff --git a/src/services/vector_db.py b/src/services/vector_db.py index f26b750..c2c3539 100644 --- a/src/services/vector_db.py +++ b/src/services/vector_db.py @@ -10,6 +10,7 @@ from qdrant_client import QdrantClient from qdrant_client.http import models from qdrant_client.http.models import Distance, VectorParams, PointStruct import uuid +from src.config.config import settings logger = logging.getLogger(__name__) @@ -22,7 +23,9 @@ class VectorDatabaseService: host: str = None, port: int = None, api_key: str = None, - collection_name: str = "image_vectors" + collection_name: str = "image_vectors", + prefer_grpc: bool = None, + https: bool = None ): """ Initialize the vector database service. @@ -32,17 +35,39 @@ class VectorDatabaseService: port: Qdrant server port api_key: API key for authentication (optional) collection_name: Name of the collection to use + prefer_grpc: Whether to use gRPC instead of HTTP + https: Whether to use HTTPS. If None, auto-detect based on host """ - self.host = host or os.getenv("QDRANT_HOST", "localhost") - self.port = port or int(os.getenv("QDRANT_PORT", "6333")) - self.api_key = api_key or os.getenv("QDRANT_API_KEY") - self.collection_name = collection_name + self.host = host or settings.QDRANT_HOST + self.port = port or settings.QDRANT_PORT + self.api_key = api_key or settings.QDRANT_API_KEY + self.collection_name = collection_name or settings.QDRANT_COLLECTION + + # Use settings defaults if not specified + if prefer_grpc is None: + prefer_grpc = settings.QDRANT_PREFER_GRPC + + # Auto-detect HTTPS usage if not specified + if https is None: + # First check if explicitly set in settings + if hasattr(settings, 'QDRANT_HTTPS') and os.getenv("QDRANT_HTTPS"): + https = settings.QDRANT_HTTPS + else: + # Use HTTP for localhost and known development IPs, HTTPS for others + https = not (self.host in ["localhost", "127.0.0.1"] or self.host.startswith("192.168.") or self.host.startswith("10.")) + # Override for specific known HTTP-only servers + if self.host == "34.171.134.17": + https = False + + logger.info(f"Connecting to Qdrant at {self.host}:{self.port} (HTTPS: {https}, gRPC: {prefer_grpc})") # Initialize Qdrant client self.client = QdrantClient( host=self.host, port=self.port, - api_key=self.api_key + api_key=self.api_key, + prefer_grpc=prefer_grpc, + https=https ) # Ensure collection exists