From 11923229f607fa9d3820011c6a969ba280c91d39 Mon Sep 17 00:00:00 2001 From: johnpccd Date: Sat, 24 May 2025 20:13:11 +0200 Subject: [PATCH] cp --- README.md | 10 +- debug_vector_db.py | 58 + deployment/README.md | 130 +-- deployment/cloud-run/service.yaml | 4 + deployment/deploy.sh | 14 + deployment/terraform/README.md | 242 +++- deployment/terraform/main.tf | 290 ++--- deployment/terraform/outputs.tf | 138 +-- deployment/terraform/scripts/get_qdrant_ip.sh | 73 ++ deployment/terraform/terraform.tfstate | 112 +- deployment/terraform/terraform.tfstate.backup | 1001 ++++++++++++++++- deployment/terraform/vm.tf | 40 +- src/api/v1/search.py | 20 +- src/services/vector_db.py | 10 +- test_qdrant_connection.py | 26 +- 15 files changed, 1755 insertions(+), 413 deletions(-) create mode 100644 debug_vector_db.py create mode 100644 deployment/terraform/scripts/get_qdrant_ip.sh diff --git a/README.md b/README.md index 10b0955..4724f85 100644 --- a/README.md +++ b/README.md @@ -545,23 +545,19 @@ This modular architecture provides several benefits: ### High Priority - [ ] Remove Pinecone integration and complete Qdrant migration - [ ] Test and validate vector search functionality with Qdrant -- [ ] Implement proper pagination for search results +- [ ] Implement proper pagination for search results and all endpoints - [ ] Test Cloud Function image processing pipeline - [ ] Validate VM setup for self-hosted Qdrant instance ### Medium Priority - [ ] Add comprehensive logging for vector search operations - [ ] Implement caching layer for frequently accessed embeddings +- [ ] Implement caching for frequently accessed data - [ ] Add monitoring and alerting for vector database performance -- [ ] Create backup strategy for vector database - [ ] Document vector search API endpoints +- [ ] Set up Qdrant cluster with multiple nodes ### Low Priority -- [ ] Add support for batch embedding generation -- [ ] Implement embedding versioning -- [ ] Add support for custom embedding models -- [ ] Create visualization tools for embedding space -- [ ] Add support for embedding fine-tuning ## Recent Changes - Migrated from Pinecone to self-hosted Qdrant diff --git a/debug_vector_db.py b/debug_vector_db.py new file mode 100644 index 0000000..f4aaae9 --- /dev/null +++ b/debug_vector_db.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Debug script to test VectorDatabaseService initialization +""" + +import os +import sys +from dotenv import load_dotenv + +# Load environment variables +load_dotenv() + +print("Environment variables:") +print(f"QDRANT_HOST: {os.getenv('QDRANT_HOST')}") +print(f"QDRANT_PORT: {os.getenv('QDRANT_PORT')}") +print(f"QDRANT_HTTPS: {os.getenv('QDRANT_HTTPS')}") +print(f"QDRANT_PREFER_GRPC: {os.getenv('QDRANT_PREFER_GRPC')}") + +# Import and check settings +from src.config.config import settings + +print("\nSettings values:") +print(f"settings.QDRANT_HOST: {settings.QDRANT_HOST}") +print(f"settings.QDRANT_PORT: {settings.QDRANT_PORT}") +print(f"settings.QDRANT_HTTPS: {settings.QDRANT_HTTPS}") +print(f"settings.QDRANT_PREFER_GRPC: {settings.QDRANT_PREFER_GRPC}") + +# Test VectorDatabaseService initialization step by step +print("\nTesting VectorDatabaseService initialization step by step...") +try: + from qdrant_client import QdrantClient + + # Test direct QdrantClient creation + print("Creating QdrantClient directly...") + client = QdrantClient( + host=settings.QDRANT_HOST, + port=settings.QDRANT_PORT, + api_key=settings.QDRANT_API_KEY, + prefer_grpc=settings.QDRANT_PREFER_GRPC, + https=settings.QDRANT_HTTPS + ) + print("QdrantClient created successfully") + + # Test get_collections (this is what fails in _ensure_collection_exists) + print("Testing get_collections...") + collections = client.get_collections() + print(f"Collections retrieved: {[col.name for col in collections.collections]}") + + # Now test full VectorDatabaseService + print("\nTesting full VectorDatabaseService...") + from src.services.vector_db import VectorDatabaseService + vector_db = VectorDatabaseService() + print("VectorDatabaseService created successfully") + +except Exception as e: + print(f"Error: {e}") + import traceback + traceback.print_exc() \ No newline at end of file diff --git a/deployment/README.md b/deployment/README.md index 2109beb..88134c3 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -1,66 +1,66 @@ -# Deployment Options for Sereact - -This directory contains multiple options for deploying the Sereact application: - -## Terraform Infrastructure (`/terraform`) - -The Terraform configuration automates the provisioning of all required Google Cloud resources: - -- Google Cloud Run service -- Google Container Registry (GCR) -- Cloud Firestore -- Cloud Storage buckets - -See [terraform/README.md](terraform/README.md) for detailed instructions. - -## Cloud Run Deployment (`/cloud-run`) - -The `service.yaml` file defines the Cloud Run service configuration which can be deployed using: - -```bash -gcloud run services replace deployment/cloud-run/service.yaml --region=us-central1 -``` - -## Deployment Script (`deploy.sh`) - -For convenience, a deployment script is provided to handle the entire deployment workflow: - -```bash -# Provision infrastructure with Terraform -./deployment/deploy.sh --provision - -# Build and push Docker image -./deployment/deploy.sh --build - -# Deploy to Cloud Run -./deployment/deploy.sh --deploy - -# Do everything (provision, build, deploy) -./deployment/deploy.sh --all -``` - -## CI/CD Pipelines - -For CI/CD integration, consider using: - -1. **GitHub Actions**: Sample workflow included in terraform/README.md -2. **Cloud Build**: Configure a `cloudbuild.yaml` in your repository -3. **Jenkins**: Use the `deploy.sh` script in your pipeline - -## Managing Secrets - -Sensitive data should be managed using Google Secret Manager: - -```bash -# Create a secret -gcloud secrets create sereact-api-key-secret --replication-policy="automatic" -gcloud secrets create sereact-vector-db-key --replication-policy="automatic" - -# Add a secret version -echo -n "your-api-key-secret" | gcloud secrets versions add sereact-api-key-secret --data-file=- -echo -n "your-vector-db-key" | gcloud secrets versions add sereact-vector-db-key --data-file=- - -# Update Cloud Run service to use the secrets -gcloud run services update sereact \ - --update-secrets=API_KEY_SECRET=sereact-api-key-secret:latest,VECTOR_DB_API_KEY=sereact-vector-db-key:latest +# Deployment Options for Sereact + +This directory contains multiple options for deploying the Sereact application: + +## Terraform Infrastructure (`/terraform`) + +The Terraform configuration automates the provisioning of all required Google Cloud resources: + +- Google Cloud Run service +- Google Container Registry (GCR) +- Cloud Firestore +- Cloud Storage buckets + +See [terraform/README.md](terraform/README.md) for detailed instructions. + +## Cloud Run Deployment (`/cloud-run`) + +The `service.yaml` file defines the Cloud Run service configuration which can be deployed using: + +```bash +gcloud run services replace deployment/cloud-run/service.yaml --region=us-central1 +``` + +## Deployment Script (`deploy.sh`) + +For convenience, a deployment script is provided to handle the entire deployment workflow: + +```bash +# Provision infrastructure with Terraform +./deployment/deploy.sh --provision + +# Build and push Docker image +./deployment/deploy.sh --build + +# Deploy to Cloud Run +./deployment/deploy.sh --deploy + +# Do everything (provision, build, deploy) +./deployment/deploy.sh --all +``` + +## CI/CD Pipelines + +For CI/CD integration, consider using: + +1. **GitHub Actions**: Sample workflow included in terraform/README.md +2. **Cloud Build**: Configure a `cloudbuild.yaml` in your repository +3. **Jenkins**: Use the `deploy.sh` script in your pipeline + +## Managing Secrets + +Sensitive data should be managed using Google Secret Manager: + +```bash +# Create a secret +gcloud secrets create sereact-api-key-secret --replication-policy="automatic" +gcloud secrets create sereact-vector-db-key --replication-policy="automatic" + +# Add a secret version +echo -n "your-api-key-secret" | gcloud secrets versions add sereact-api-key-secret --data-file=- +echo -n "your-vector-db-key" | gcloud secrets versions add sereact-vector-db-key --data-file=- + +# Update Cloud Run service to use the secrets +gcloud run services update sereact \ + --update-secrets=API_KEY_SECRET=sereact-api-key-secret:latest,VECTOR_DB_API_KEY=sereact-vector-db-key:latest ``` \ No newline at end of file diff --git a/deployment/cloud-run/service.yaml b/deployment/cloud-run/service.yaml index 261d360..c715cba 100644 --- a/deployment/cloud-run/service.yaml +++ b/deployment/cloud-run/service.yaml @@ -53,6 +53,10 @@ spec: value: "Content-Length,Content-Range" - name: CORS_MAX_AGE value: "3600" + - name: QDRANT_HTTPS + value: "false" + - name: QDRANT_PREFER_GRPC + value: "false" volumeMounts: - name: gcp-sa-key mountPath: /var/secrets/google diff --git a/deployment/deploy.sh b/deployment/deploy.sh index 16b7274..5b69133 100644 --- a/deployment/deploy.sh +++ b/deployment/deploy.sh @@ -310,10 +310,24 @@ if [ "$DEPLOY" = true ]; then # Get service URL from Terraform output SERVICE_URL=$(terraform output -raw cloud_run_url 2>/dev/null || echo "URL not available") + # Get Qdrant VM information + QDRANT_IP=$(terraform output -raw vector_db_vm_external_ip 2>/dev/null || echo "IP not available") + QDRANT_ENDPOINT=$(terraform output -raw qdrant_http_endpoint 2>/dev/null || echo "Endpoint not available") + cd - > /dev/null echo "Deployment completed successfully." + echo "==================================" echo "Service URL: $SERVICE_URL" + echo "Qdrant VM IP: $QDRANT_IP" + echo "Qdrant HTTP Endpoint: $QDRANT_ENDPOINT" + echo "" + echo "Environment variables set in Cloud Run:" + echo " QDRANT_HOST=$QDRANT_IP" + echo " QDRANT_PORT=6333" + echo "" + echo "To get detailed Qdrant information, run:" + echo " ./deployment/terraform/scripts/get_qdrant_ip.sh" fi echo "All operations completed." \ No newline at end of file diff --git a/deployment/terraform/README.md b/deployment/terraform/README.md index c0e0e65..b4ec200 100644 --- a/deployment/terraform/README.md +++ b/deployment/terraform/README.md @@ -90,4 +90,244 @@ To destroy all provisioned resources: ```bash terraform destroy -``` \ No newline at end of file +``` + +# Terraform Infrastructure for Sereact + +This directory contains Terraform configurations to deploy the complete Sereact infrastructure on Google Cloud Platform, including automatic configuration of Qdrant VM IP addresses for Cloud Run. + +## Architecture Overview + +The infrastructure includes: +- **Cloud Run Service**: Main application service +- **Qdrant VM**: Vector database running on Compute Engine +- **Firestore**: Document database +- **Cloud Storage**: File storage +- **Automatic IP Configuration**: Qdrant VM IP is automatically passed to Cloud Run + +## Qdrant VM IP Address Integration + +The Terraform configuration automatically handles passing the Qdrant VM IP address to your Cloud Run service: + +### How It Works + +1. **VM Creation**: Terraform creates a Compute Engine VM running Qdrant +2. **IP Address Extraction**: Terraform extracts the VM's external IP address +3. **Environment Variable Injection**: The IP is automatically set as `QDRANT_HOST` in Cloud Run +4. **Dependency Management**: Cloud Run waits for the VM to be created before deploying + +### Configuration Options + +#### Static vs Dynamic IP + +You can choose between static and dynamic IP addresses: + +```hcl +# In terraform.tfvars +use_static_ip = true # Use static IP (recommended for production) +use_static_ip = false # Use ephemeral IP (cheaper for development) +``` + +#### Environment Variables Set Automatically + +The following environment variables are automatically configured in Cloud Run: + +```bash +QDRANT_HOST= # Automatically set from VM +QDRANT_PORT=6333 # HTTP port +QDRANT_API_KEY= # From terraform.tfvars +``` + +## Quick Start + +### 1. Prerequisites + +```bash +# Install required tools +gcloud auth login +gcloud config set project YOUR_PROJECT_ID +terraform --version +``` + +### 2. Configuration + +```bash +# Copy and edit configuration +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your values +``` + +### 3. Deploy Everything + +```bash +# From the root directory +./deployment/deploy.sh --deploy --build +``` + +### 4. Get Qdrant Information + +```bash +# Get Qdrant VM IP and endpoints +./deployment/terraform/scripts/get_qdrant_ip.sh + +# Get internal IP (for VPC connections) +./deployment/terraform/scripts/get_qdrant_ip.sh --internal +``` + +## Terraform Outputs + +The configuration provides several outputs for integration: + +```bash +# Get specific outputs +terraform output cloud_run_url # Cloud Run service URL +terraform output vector_db_vm_external_ip # Qdrant VM external IP +terraform output vector_db_vm_internal_ip # Qdrant VM internal IP +terraform output qdrant_http_endpoint # Full HTTP endpoint +terraform output cloud_run_qdrant_host # IP configured for Cloud Run + +# Get deployment summary +terraform output deployment_summary +``` + +## File Structure + +``` +terraform/ +├── main.tf # Cloud Run and main resources +├── vm.tf # Qdrant VM configuration +├── outputs.tf # Output definitions +├── variables.tf # Variable definitions +├── terraform.tfvars # Your configuration values +├── scripts/ +│ ├── get_qdrant_ip.sh # Helper script to get VM IP +│ └── install_qdrant.sh # VM startup script +└── README.md # This file +``` + +## Environment Variables Reference + +### Automatically Configured + +These are set automatically by Terraform: + +| Variable | Source | Description | +|----------|--------|-------------| +| `QDRANT_HOST` | VM external IP | Qdrant server IP address | +| `QDRANT_PORT` | Static value | Qdrant HTTP port (6333) | +| `QDRANT_API_KEY` | terraform.tfvars | Qdrant authentication key | +| `FIRESTORE_PROJECT_ID` | terraform.tfvars | GCP project ID | +| `GCS_BUCKET_NAME` | terraform.tfvars | Storage bucket name | + +### Manual Configuration Required + +Set these in `terraform.tfvars`: + +```hcl +project_id = "your-gcp-project-id" +storage_bucket_name = "your-bucket-name" +qdrant_api_key = "your-qdrant-api-key" +use_static_ip = true # or false +``` + +## Networking Configuration + +### Firewall Rules + +The configuration automatically creates firewall rules to allow: +- Qdrant HTTP traffic (port 6333) +- Qdrant gRPC traffic (port 6334) +- Access from Cloud Run to Qdrant VM + +### IP Address Types + +#### External IP (Default) +- Used for Cloud Run → Qdrant communication +- Accessible from internet (with firewall rules) +- Can be static or ephemeral + +#### Internal IP (Alternative) +- Available for VPC-native connections +- More secure but requires VPC configuration +- Use `cloud_run_qdrant_host_internal` output + +## Troubleshooting + +### Check Qdrant Connectivity + +```bash +# Get IP and test connection +./deployment/terraform/scripts/get_qdrant_ip.sh + +# Manual test +QDRANT_IP=$(terraform output -raw vector_db_vm_external_ip) +curl http://$QDRANT_IP:6333/health +``` + +### Verify Cloud Run Environment + +```bash +# Check Cloud Run service +gcloud run services describe sereact --region=us-central1 + +# Check environment variables +gcloud run services describe sereact --region=us-central1 \ + --format="value(spec.template.spec.containers[0].env[].name,spec.template.spec.containers[0].env[].value)" +``` + +### Common Issues + +1. **VM not ready**: Qdrant installation takes 2-3 minutes after VM creation +2. **Firewall blocking**: Check that your IP is in `allowed_cidr_blocks` +3. **Wrong IP type**: Ensure you're using external IP for Cloud Run connections + +## Security Considerations + +### Production Recommendations + +1. **Use Static IP**: Set `use_static_ip = true` +2. **Restrict Access**: Set specific CIDR blocks in `allowed_cidr_blocks` +3. **Use Internal IP**: Consider VPC-native networking for internal communication +4. **Secure API Key**: Store `qdrant_api_key` in Secret Manager + +### Development Setup + +```hcl +# terraform.tfvars for development +use_static_ip = false +allowed_cidr_blocks = "0.0.0.0/0" # Open access (not for production!) +``` + +## Cost Optimization + +### Development +- Use `use_static_ip = false` to avoid static IP charges +- Use smaller VM sizes: `machine_type = "e2-micro"` + +### Production +- Use `use_static_ip = true` for consistent connectivity +- Use appropriate VM sizes based on load + +## Advanced Configuration + +### Custom VM Configuration + +Edit `vm.tf` to customize: +- Machine type +- Disk size +- Startup script +- Network configuration + +### VPC-Native Networking + +For internal-only communication: +1. Create VPC connector +2. Configure Cloud Run to use VPC +3. Use internal IP output: `cloud_run_qdrant_host_internal` + +## Support + +For issues with: +- **Terraform**: Check `terraform plan` output +- **VM connectivity**: Use the helper script `get_qdrant_ip.sh` +- **Cloud Run**: Check logs with `gcloud run logs tail sereact` \ No newline at end of file diff --git a/deployment/terraform/main.tf b/deployment/terraform/main.tf index 0fa33d5..3442f23 100644 --- a/deployment/terraform/main.tf +++ b/deployment/terraform/main.tf @@ -1,146 +1,146 @@ -provider "google" { - project = var.project_id - region = var.region - zone = var.zone -} - -# Get current project information -data "google_project" "current" { - project_id = var.project_id -} - -# Enable required APIs -resource "google_project_service" "services" { - for_each = toset([ - "cloudresourcemanager.googleapis.com", - "containerregistry.googleapis.com", - "run.googleapis.com", - "firestore.googleapis.com", - "storage.googleapis.com", - "compute.googleapis.com" - ]) - - project = var.project_id - service = each.key - - disable_on_destroy = false -} - -# Cloud Storage bucket -resource "google_storage_bucket" "app_bucket" { - name = var.storage_bucket_name - location = var.region - uniform_bucket_level_access = true - - depends_on = [google_project_service.services] -} - -# Firestore Database -resource "google_firestore_database" "database" { - name = var.firestore_db_name - location_id = var.region - type = "FIRESTORE_NATIVE" - - depends_on = [google_project_service.services] -} - -# Container Registry - no explicit resource needed, just enable the API -# You'll push images to gcr.io/${var.project_id}/sereact-api - -# Cloud Run service -resource "google_cloud_run_service" "sereact" { - name = "sereact" - location = var.region - - metadata { - annotations = { - "run.googleapis.com/ingress" = "all" - } - } - - template { - spec { - containers { - # Use our optimized image - image = "gcr.io/${var.project_id}/sereact-api:latest" - - ports { - container_port = 8000 - } - - resources { - limits = { - cpu = "1" - memory = "1Gi" - } - } - - env { - name = "FIRESTORE_PROJECT_ID" - value = var.project_id - } - - env { - name = "FIRESTORE_DATABASE_NAME" - value = var.firestore_db_name - } - - env { - name = "GCS_BUCKET_NAME" - value = var.storage_bucket_name - } - - env { - name = "VECTOR_DB_ENVIRONMENT" - value = var.vector_db_environment - } - - env { - name = "VECTOR_DB_INDEX_NAME" - value = var.vector_db_index_name - } - - env { - name = "QDRANT_HOST" - value = var.use_static_ip ? google_compute_address.vector_db_static_ip.address : google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip - } - - env { - name = "QDRANT_PORT" - value = "6333" - } - - env { - name = "QDRANT_API_KEY" - value = var.qdrant_api_key - } - - env { - name = "LOG_LEVEL" - value = "INFO" - } - } - } - - metadata { - annotations = { - "autoscaling.knative.dev/maxScale" = "10" - } - } - } - - traffic { - percent = 100 - latest_revision = true - } - - depends_on = [google_project_service.services, google_compute_instance.vector_db_vm] -} - -# Make the Cloud Run service publicly accessible -resource "google_cloud_run_service_iam_member" "public_access" { - service = google_cloud_run_service.sereact.name - location = google_cloud_run_service.sereact.location - role = "roles/run.invoker" - member = "allUsers" +provider "google" { + project = var.project_id + region = var.region + zone = var.zone +} + +# Get current project information +data "google_project" "current" { + project_id = var.project_id +} + +# Enable required APIs +resource "google_project_service" "services" { + for_each = toset([ + "cloudresourcemanager.googleapis.com", + "containerregistry.googleapis.com", + "run.googleapis.com", + "firestore.googleapis.com", + "storage.googleapis.com", + "compute.googleapis.com" + ]) + + project = var.project_id + service = each.key + + disable_on_destroy = false +} + +# Cloud Storage bucket +resource "google_storage_bucket" "app_bucket" { + name = var.storage_bucket_name + location = var.region + uniform_bucket_level_access = true + + depends_on = [google_project_service.services] +} + +# Firestore Database +resource "google_firestore_database" "database" { + name = var.firestore_db_name + location_id = var.region + type = "FIRESTORE_NATIVE" + + depends_on = [google_project_service.services] +} + +# Container Registry - no explicit resource needed, just enable the API +# You'll push images to gcr.io/${var.project_id}/sereact-api + +# Cloud Run service +resource "google_cloud_run_service" "sereact" { + name = "sereact" + location = var.region + + metadata { + annotations = { + "run.googleapis.com/ingress" = "all" + } + } + + template { + spec { + containers { + # Use our optimized image + image = "gcr.io/${var.project_id}/sereact-api:latest" + + ports { + container_port = 8000 + } + + resources { + limits = { + cpu = "1" + memory = "1Gi" + } + } + + env { + name = "FIRESTORE_PROJECT_ID" + value = var.project_id + } + + env { + name = "FIRESTORE_DATABASE_NAME" + value = var.firestore_db_name + } + + env { + name = "GCS_BUCKET_NAME" + value = var.storage_bucket_name + } + + env { + name = "VECTOR_DB_ENVIRONMENT" + value = var.vector_db_environment + } + + env { + name = "VECTOR_DB_INDEX_NAME" + value = var.vector_db_index_name + } + + env { + name = "QDRANT_HOST" + value = google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip + } + + env { + name = "QDRANT_PORT" + value = "6333" + } + + env { + name = "QDRANT_API_KEY" + value = var.qdrant_api_key + } + + env { + name = "LOG_LEVEL" + value = "INFO" + } + } + } + + metadata { + annotations = { + "autoscaling.knative.dev/maxScale" = "10" + } + } + } + + traffic { + percent = 100 + latest_revision = true + } + + depends_on = [google_project_service.services, google_compute_instance.vector_db_vm] +} + +# Make the Cloud Run service publicly accessible +resource "google_cloud_run_service_iam_member" "public_access" { + service = google_cloud_run_service.sereact.name + location = google_cloud_run_service.sereact.location + role = "roles/run.invoker" + member = "allUsers" } \ No newline at end of file diff --git a/deployment/terraform/outputs.tf b/deployment/terraform/outputs.tf index 88a860e..9b1e6e1 100644 --- a/deployment/terraform/outputs.tf +++ b/deployment/terraform/outputs.tf @@ -1,66 +1,74 @@ -output "cloud_run_url" { - value = google_cloud_run_service.sereact.status[0].url - description = "The URL of the deployed Cloud Run service" -} - -output "storage_bucket_name" { - value = google_storage_bucket.app_bucket.name - description = "The name of the provisioned Cloud Storage bucket" -} - -output "firestore_database_id" { - value = google_firestore_database.database.id - description = "The ID of the provisioned Firestore database" -} - -output "container_registry_url" { - value = "gcr.io/${var.project_id}/sereact" - description = "The URL of the Container Registry repository" -} - -# Vector Database VM outputs -output "vector_db_vm_name" { - value = google_compute_instance.vector_db_vm.name - description = "The name of the vector database VM" -} - -output "vector_db_vm_external_ip" { - value = google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip - description = "The external IP address of the vector database VM" -} - -output "vector_db_vm_internal_ip" { - value = google_compute_instance.vector_db_vm.network_interface[0].network_ip - description = "The internal IP address of the vector database VM" -} - -output "vector_db_static_ip" { - value = var.use_static_ip ? google_compute_address.vector_db_static_ip.address : null - description = "The static IP address of the vector database VM (if enabled)" -} - -output "qdrant_http_endpoint" { - value = "http://${google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip}:6333" - description = "The HTTP endpoint for Qdrant vector database" -} - -output "qdrant_grpc_endpoint" { - value = "http://${google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip}:6334" - description = "The gRPC endpoint for Qdrant vector database" -} - -# Cloud Run environment configuration -output "cloud_run_qdrant_host" { - value = var.use_static_ip ? google_compute_address.vector_db_static_ip.address : google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip - description = "The Qdrant host configured for Cloud Run" -} - -output "deployment_summary" { - value = { - cloud_run_url = google_cloud_run_service.sereact.status[0].url - qdrant_endpoint = "http://${google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip}:6333" - firestore_database = var.firestore_db_name - storage_bucket = var.storage_bucket_name - } - description = "Summary of deployed resources" +output "cloud_run_url" { + value = google_cloud_run_service.sereact.status[0].url + description = "The URL of the deployed Cloud Run service" +} + +output "storage_bucket_name" { + value = google_storage_bucket.app_bucket.name + description = "The name of the provisioned Cloud Storage bucket" +} + +output "firestore_database_id" { + value = google_firestore_database.database.id + description = "The ID of the provisioned Firestore database" +} + +output "container_registry_url" { + value = "gcr.io/${var.project_id}/sereact" + description = "The URL of the Container Registry repository" +} + +# Vector Database VM outputs +output "vector_db_vm_name" { + value = google_compute_instance.vector_db_vm.name + description = "The name of the vector database VM" +} + +output "vector_db_vm_external_ip" { + value = google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip + description = "The external IP address of the vector database VM" +} + +output "vector_db_vm_internal_ip" { + value = google_compute_instance.vector_db_vm.network_interface[0].network_ip + description = "The internal IP address of the vector database VM" +} + +output "vector_db_static_ip" { + value = var.use_static_ip ? google_compute_address.vector_db_static_ip[0].address : null + description = "The static IP address of the vector database VM (if enabled)" +} + +output "qdrant_http_endpoint" { + value = "http://${google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip}:6333" + description = "The HTTP endpoint for Qdrant vector database" +} + +output "qdrant_grpc_endpoint" { + value = "http://${google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip}:6334" + description = "The gRPC endpoint for Qdrant vector database" +} + +# Cloud Run environment configuration +output "cloud_run_qdrant_host" { + value = google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip + description = "The Qdrant host IP address configured for Cloud Run" +} + +# Separate output for internal IP (useful for VPC-native connections) +output "cloud_run_qdrant_host_internal" { + value = google_compute_instance.vector_db_vm.network_interface[0].network_ip + description = "The internal Qdrant host IP address for VPC-native connections" +} + +output "deployment_summary" { + value = { + cloud_run_url = google_cloud_run_service.sereact.status[0].url + qdrant_endpoint = "http://${google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip}:6333" + qdrant_host_ip = google_compute_instance.vector_db_vm.network_interface[0].access_config[0].nat_ip + firestore_database = var.firestore_db_name + storage_bucket = var.storage_bucket_name + static_ip_enabled = var.use_static_ip + } + description = "Summary of deployed resources" } \ No newline at end of file diff --git a/deployment/terraform/scripts/get_qdrant_ip.sh b/deployment/terraform/scripts/get_qdrant_ip.sh new file mode 100644 index 0000000..18527d4 --- /dev/null +++ b/deployment/terraform/scripts/get_qdrant_ip.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Script to get Qdrant VM IP address from Terraform outputs +# Usage: ./get_qdrant_ip.sh [--internal] + +set -e + +# Change to terraform directory +cd "$(dirname "$0")/.." + +# Check if terraform state exists +if [ ! -f "terraform.tfstate" ]; then + echo "ERROR: No terraform.tfstate found. Please run 'terraform apply' first." + exit 1 +fi + +# Parse command line arguments +INTERNAL=false +if [ "$1" = "--internal" ]; then + INTERNAL=true +fi + +echo "Retrieving Qdrant VM IP address..." +echo "==================================" + +# Get VM information +VM_NAME=$(terraform output -raw vector_db_vm_name 2>/dev/null || echo "N/A") +echo "VM Name: $VM_NAME" + +if [ "$INTERNAL" = true ]; then + # Get internal IP + QDRANT_IP=$(terraform output -raw vector_db_vm_internal_ip 2>/dev/null || echo "N/A") + echo "Internal IP: $QDRANT_IP" + echo "Qdrant HTTP Endpoint (Internal): http://$QDRANT_IP:6333" + echo "Qdrant gRPC Endpoint (Internal): http://$QDRANT_IP:6334" +else + # Get external IP + QDRANT_IP=$(terraform output -raw vector_db_vm_external_ip 2>/dev/null || echo "N/A") + echo "External IP: $QDRANT_IP" + echo "Qdrant HTTP Endpoint: http://$QDRANT_IP:6333" + echo "Qdrant gRPC Endpoint: http://$QDRANT_IP:6334" +fi + +# Check if static IP is enabled +STATIC_IP_ENABLED=$(terraform output -raw vector_db_static_ip 2>/dev/null || echo "null") +if [ "$STATIC_IP_ENABLED" != "null" ] && [ "$STATIC_IP_ENABLED" != "" ]; then + echo "Static IP: $STATIC_IP_ENABLED (enabled)" +else + echo "Static IP: Not enabled (using ephemeral IP)" +fi + +# Get Cloud Run service URL +CLOUD_RUN_URL=$(terraform output -raw cloud_run_url 2>/dev/null || echo "N/A") +echo "Cloud Run URL: $CLOUD_RUN_URL" + +echo "" +echo "Environment Variables for Cloud Run:" +echo "QDRANT_HOST=$QDRANT_IP" +echo "QDRANT_PORT=6333" + +# Test connectivity (optional) +echo "" +echo "Testing Qdrant connectivity..." +if command -v curl &> /dev/null; then + if curl -s --connect-timeout 5 "http://$QDRANT_IP:6333/health" > /dev/null; then + echo "✓ Qdrant is accessible at http://$QDRANT_IP:6333" + else + echo "✗ Qdrant is not accessible at http://$QDRANT_IP:6333" + echo " This might be normal if the VM is still starting up or firewall rules need adjustment." + fi +else + echo "curl not available - skipping connectivity test" +fi \ No newline at end of file diff --git a/deployment/terraform/terraform.tfstate b/deployment/terraform/terraform.tfstate index 9d7be7a..12f5e7b 100644 --- a/deployment/terraform/terraform.tfstate +++ b/deployment/terraform/terraform.tfstate @@ -1,11 +1,15 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 202, + "serial": 254, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { "cloud_run_qdrant_host": { - "value": "104.197.36.94", + "value": "34.71.6.1", + "type": "string" + }, + "cloud_run_qdrant_host_internal": { + "value": "10.128.0.4", "type": "string" }, "container_registry_url": { @@ -25,11 +29,11 @@ "type": "string" }, "qdrant_grpc_endpoint": { - "value": "http://104.197.36.94:6334", + "value": "http://34.71.6.1:6334", "type": "string" }, "qdrant_http_endpoint": { - "value": "http://104.197.36.94:6333", + "value": "http://34.71.6.1:6333", "type": "string" }, "storage_bucket_name": { @@ -37,11 +41,11 @@ "type": "string" }, "vector_db_vm_external_ip": { - "value": "104.197.36.94", + "value": "34.71.6.1", "type": "string" }, "vector_db_vm_internal_ip": { - "value": "10.128.0.3", + "value": "10.128.0.4", "type": "string" }, "vector_db_vm_name": { @@ -172,7 +176,7 @@ }, { "name": "QDRANT_HOST", - "value": "104.197.36.94", + "value": "34.71.6.1", "value_from": [] }, { @@ -292,41 +296,7 @@ "type": "google_compute_address", "name": "vector_db_static_ip", "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "address": "34.171.134.17", - "address_type": "EXTERNAL", - "creation_timestamp": "2025-05-24T10:10:23.021-07:00", - "description": "", - "effective_labels": { - "goog-terraform-provisioned": "true" - }, - "id": "projects/gen-lang-client-0424120530/regions/us-central1/addresses/vector-db-static-ip", - "ip_version": "", - "ipv6_endpoint_type": "", - "label_fingerprint": "vezUS-42LLM=", - "labels": null, - "name": "vector-db-static-ip", - "network": "", - "network_tier": "PREMIUM", - "prefix_length": 0, - "project": "gen-lang-client-0424120530", - "purpose": "", - "region": "us-central1", - "self_link": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/regions/us-central1/addresses/vector-db-static-ip", - "subnetwork": "", - "terraform_labels": { - "goog-terraform-provisioned": "true" - }, - "timeouts": null, - "users": [] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] + "instances": [] }, { "mode": "managed", @@ -346,7 +316,7 @@ "protocol": "tcp" } ], - "creation_timestamp": "2025-05-24T10:10:23.267-07:00", + "creation_timestamp": "2025-05-24T10:57:27.640-07:00", "deny": [], "description": "", "destination_ranges": [], @@ -364,9 +334,9 @@ "0.0.0.0/0", "10.0.0.0/8" ], - "source_service_accounts": null, - "source_tags": null, - "target_service_accounts": null, + "source_service_accounts": [], + "source_tags": [], + "target_service_accounts": [], "target_tags": [ "qdrant" ], @@ -417,7 +387,7 @@ "labels": {}, "provisioned_iops": 0, "provisioned_throughput": 0, - "resource_manager_tags": null, + "resource_manager_tags": {}, "resource_policies": [], "size": 50, "snapshot": "", @@ -436,7 +406,7 @@ "can_ip_forward": false, "confidential_instance_config": [], "cpu_platform": "Intel Broadwell", - "creation_timestamp": "2025-05-24T10:10:36.440-07:00", + "creation_timestamp": "2025-05-24T10:57:41.174-07:00", "current_status": "RUNNING", "deletion_protection": false, "description": "", @@ -449,12 +419,12 @@ "hostname": "", "id": "projects/gen-lang-client-0424120530/zones/us-central1-a/instances/sereact-vector-db", "instance_encryption_key": [], - "instance_id": "670961443312255843", + "instance_id": "8076617322369889915", "key_revocation_action_type": "", "label_fingerprint": "vezUS-42LLM=", - "labels": null, + "labels": {}, "machine_type": "e2-standard-2", - "metadata": null, + "metadata": {}, "metadata_fingerprint": "cE8FbgySELs=", "metadata_startup_script": "#!/bin/bash\n\n# Qdrant Vector Database Installation Script\n# This script installs and configures Qdrant on Ubuntu 22.04\n\nset -e\n\n# Update system packages\napt-get update\napt-get upgrade -y\n\n# Install required packages\napt-get install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates\n\n# Install Docker\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg\necho \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list \u003e /dev/null\napt-get update\napt-get install -y docker-ce docker-ce-cli containerd.io\n\n# Start and enable Docker\nsystemctl start docker\nsystemctl enable docker\n\n# Create qdrant user and directories\nuseradd -r -s /bin/false qdrant || true\nmkdir -p /opt/qdrant/storage\nmkdir -p /opt/qdrant/config\nchown -R qdrant:qdrant /opt/qdrant\n\n# Create Qdrant configuration file\ncat \u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_cors: true\n\nstorage:\n storage_path: /qdrant/storage\n snapshots_path: /qdrant/snapshots\n on_disk_payload: true\n\ncluster:\n enabled: false\n\ntelemetry:\n disabled: true\n\nlog_level: INFO\nEOF\n\n# Create API key configuration if provided\nif [ -n \"\" ] \u0026\u0026 [ \"\" != \"\" ]; then\ncat \u003e\u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\n\nservice:\n api_key: \"\"\nEOF\nfi\n\n# Create systemd service for Qdrant\ncat \u003e /etc/systemd/system/qdrant.service \u003c\u003c EOF\n[Unit]\nDescription=Qdrant Vector Database\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nType=simple\nUser=root\nExecStartPre=-/usr/bin/docker stop qdrant\nExecStartPre=-/usr/bin/docker rm qdrant\nExecStart=/usr/bin/docker run --name qdrant \\\n -p 6333:6333 \\\n -p 6334:6334 \\\n -v /opt/qdrant/storage:/qdrant/storage:z \\\n -v /opt/qdrant/config/config.yaml:/qdrant/config/production.yaml:z \\\n qdrant/qdrant:latest\nExecStop=/usr/bin/docker stop qdrant\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n# Pull Qdrant Docker image\ndocker pull qdrant/qdrant:latest\n\n# Enable and start Qdrant service\nsystemctl daemon-reload\nsystemctl enable qdrant\nsystemctl start qdrant\n\n# Install monitoring tools\napt-get install -y htop iotop nethogs\n\n# Create a simple health check script\ncat \u003e /opt/qdrant/health_check.sh \u003c\u003c 'EOF'\n#!/bin/bash\nresponse=$(curl -s -o /dev/null -w \"%{http_code}\" http://localhost:6333/health)\nif [ \"$response\" = \"200\" ]; then\n echo \"Qdrant is healthy\"\n exit 0\nelse\n echo \"Qdrant is not responding properly (HTTP $response)\"\n exit 1\nfi\nEOF\n\nchmod +x /opt/qdrant/health_check.sh\n\n# Set up log rotation for Docker logs\ncat \u003e /etc/logrotate.d/docker \u003c\u003c EOF\n/var/lib/docker/containers/*/*.log {\n rotate 7\n daily\n compress\n size=1M\n missingok\n delaycompress\n copytruncate\n}\nEOF\n\n# Configure firewall (ufw)\nufw --force enable\nufw allow ssh\nufw allow 6333/tcp # Qdrant HTTP API\nufw allow 6334/tcp # Qdrant gRPC API\n\n# Create a simple backup script\ncat \u003e /opt/qdrant/backup.sh \u003c\u003c 'EOF'\n#!/bin/bash\nBACKUP_DIR=\"/opt/qdrant/backups\"\nDATE=$(date +%Y%m%d_%H%M%S)\nmkdir -p $BACKUP_DIR\n\n# Create snapshot via API\ncurl -X POST \"http://localhost:6333/snapshots\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"snapshot_name\": \"backup_'$DATE'\"}'\n\n# Copy storage directory\ntar -czf $BACKUP_DIR/qdrant_storage_$DATE.tar.gz -C /opt/qdrant storage/\n\n# Keep only last 7 backups\nfind $BACKUP_DIR -name \"*.tar.gz\" -mtime +7 -delete\n\necho \"Backup completed: $DATE\"\nEOF\n\nchmod +x /opt/qdrant/backup.sh\n\n# Set up daily backup cron job\necho \"0 2 * * * root /opt/qdrant/backup.sh \u003e\u003e /var/log/qdrant_backup.log 2\u003e\u00261\" \u003e\u003e /etc/crontab\n\n# Wait for Qdrant to be ready\necho \"Waiting for Qdrant to start...\"\nfor i in {1..30}; do\n if curl -s http://localhost:6333/health \u003e /dev/null; then\n echo \"Qdrant is ready!\"\n break\n fi\n echo \"Waiting... ($i/30)\"\n sleep 10\ndone\n\n# Create a default collection for image vectors\ncurl -X PUT \"http://localhost:6333/collections/image_vectors\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"vectors\": {\n \"size\": 512,\n \"distance\": \"Cosine\"\n },\n \"optimizers_config\": {\n \"default_segment_number\": 2\n },\n \"replication_factor\": 1\n }'\n\necho \"Qdrant installation and configuration completed!\"\necho \"Qdrant is accessible at:\"\necho \" HTTP API: http://$(curl -s ifconfig.me):6333\"\necho \" gRPC API: http://$(curl -s ifconfig.me):6334\"\necho \"Health check: /opt/qdrant/health_check.sh\"\necho \"Backup script: /opt/qdrant/backup.sh\" ", "min_cpu_platform": "", @@ -463,7 +433,7 @@ { "access_config": [ { - "nat_ip": "104.197.36.94", + "nat_ip": "34.71.6.1", "network_tier": "PREMIUM", "public_ptr_domain_name": "" } @@ -476,7 +446,7 @@ "name": "nic0", "network": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/global/networks/default", "network_attachment": "", - "network_ip": "10.128.0.3", + "network_ip": "10.128.0.4", "nic_type": "", "queue_count": 0, "stack_type": "IPV4_ONLY", @@ -488,7 +458,7 @@ "params": [], "project": "gen-lang-client-0424120530", "reservation_affinity": [], - "resource_policies": null, + "resource_policies": [], "scheduling": [ { "automatic_restart": true, @@ -534,6 +504,12 @@ "zone": "us-central1-a" }, "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "metadata_startup_script" + } + ], [ { "type": "get_attr", @@ -567,29 +543,17 @@ "type": "get_attr", "value": "disk_encryption_key_raw" } - ], - [ - { - "type": "get_attr", - "value": "metadata_startup_script" - } ] ], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiNiJ9", "dependencies": [ + "google_compute_address.vector_db_static_ip", "google_project_service.services", "google_service_account.vector_db_sa" ] } ] }, - { - "mode": "managed", - "type": "google_compute_instance", - "name": "vector_db_vm_with_static_ip", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [] - }, { "mode": "managed", "type": "google_project_service", @@ -815,7 +779,7 @@ "schema_version": 0, "attributes": { "condition": [], - "etag": "BwY15MsDMxU=", + "etag": "BwY15XOdN50=", "id": "projects/gen-lang-client-0424120530/subscriptions/image-processing-topic-subscription/roles/pubsub.subscriber", "members": [ "serviceAccount:761163285547-compute@developer.gserviceaccount.com" @@ -927,7 +891,7 @@ "schema_version": 0, "attributes": { "condition": [], - "etag": "BwY15MqTytg=", + "etag": "BwY15XMktMU=", "id": "projects/gen-lang-client-0424120530/topics/image-processing-topic/roles/pubsub.publisher", "members": [ "serviceAccount:761163285547-compute@developer.gserviceaccount.com" @@ -965,7 +929,7 @@ "name": "projects/gen-lang-client-0424120530/serviceAccounts/vector-db-sa@gen-lang-client-0424120530.iam.gserviceaccount.com", "project": "gen-lang-client-0424120530", "timeouts": null, - "unique_id": "104051055732001951756" + "unique_id": "108486040628486621877" }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9fQ==" @@ -997,7 +961,7 @@ } ], "id": "sereact-images", - "labels": null, + "labels": {}, "lifecycle_rule": [], "location": "US-CENTRAL1", "logging": [], @@ -1011,7 +975,7 @@ "self_link": "https://www.googleapis.com/storage/v1/b/sereact-images", "soft_delete_policy": [ { - "effective_time": "2025-05-24T17:10:27.332Z", + "effective_time": "2025-05-24T17:57:31.611Z", "retention_duration_seconds": 604800 } ], @@ -1019,10 +983,10 @@ "terraform_labels": { "goog-terraform-provisioned": "true" }, - "time_created": "2025-05-24T17:10:27.332Z", + "time_created": "2025-05-24T17:57:31.611Z", "timeouts": null, "uniform_bucket_level_access": true, - "updated": "2025-05-24T17:10:27.332Z", + "updated": "2025-05-24T17:57:31.611Z", "url": "gs://sereact-images", "versioning": [], "website": [] diff --git a/deployment/terraform/terraform.tfstate.backup b/deployment/terraform/terraform.tfstate.backup index f7290dc..45ae0dc 100644 --- a/deployment/terraform/terraform.tfstate.backup +++ b/deployment/terraform/terraform.tfstate.backup @@ -1,9 +1,1004 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 182, + "serial": 250, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", - "outputs": {}, - "resources": [], + "outputs": { + "cloud_run_qdrant_host": { + "value": "34.71.6.1", + "type": "string" + }, + "cloud_run_qdrant_host_internal": { + "value": "10.128.0.4", + "type": "string" + }, + "container_registry_url": { + "value": "gcr.io/gen-lang-client-0424120530/sereact", + "type": "string" + }, + "pubsub_dlq_topic_name": { + "value": "image-processing-topic-dlq", + "type": "string" + }, + "pubsub_subscription_name": { + "value": "image-processing-topic-subscription", + "type": "string" + }, + "pubsub_topic_name": { + "value": "image-processing-topic", + "type": "string" + }, + "qdrant_grpc_endpoint": { + "value": "http://34.71.6.1:6334", + "type": "string" + }, + "qdrant_http_endpoint": { + "value": "http://34.71.6.1:6333", + "type": "string" + }, + "storage_bucket_name": { + "value": "sereact-images", + "type": "string" + }, + "vector_db_vm_external_ip": { + "value": "34.71.6.1", + "type": "string" + }, + "vector_db_vm_internal_ip": { + "value": "10.128.0.4", + "type": "string" + }, + "vector_db_vm_name": { + "value": "sereact-vector-db", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "google_project", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "auto_create_network": null, + "billing_account": "00CA97-62E5BD-4A62B5", + "deletion_policy": "PREVENT", + "effective_labels": { + "generative-language": "enabled" + }, + "folder_id": null, + "id": "projects/gen-lang-client-0424120530", + "labels": { + "generative-language": "enabled" + }, + "name": "Gemini API", + "number": "761163285547", + "org_id": null, + "project_id": "gen-lang-client-0424120530", + "tags": null, + "terraform_labels": { + "generative-language": "enabled" + } + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "google_cloud_run_service", + "name": "sereact", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "status": "tainted", + "schema_version": 2, + "attributes": { + "autogenerate_revision_name": false, + "id": "locations/us-central1/namespaces/gen-lang-client-0424120530/services/sereact", + "location": "us-central1", + "metadata": [ + { + "annotations": { + "run.googleapis.com/ingress": "all" + }, + "effective_annotations": { + "run.googleapis.com/ingress": "all" + }, + "effective_labels": { + "goog-terraform-provisioned": "true" + }, + "generation": 0, + "labels": null, + "namespace": "", + "resource_version": "", + "self_link": "", + "terraform_labels": { + "goog-terraform-provisioned": "true" + }, + "uid": "" + } + ], + "name": "sereact", + "project": "gen-lang-client-0424120530", + "status": null, + "template": [ + { + "metadata": [ + { + "annotations": { + "autoscaling.knative.dev/maxScale": "10" + }, + "generation": 0, + "labels": {}, + "name": "", + "namespace": "", + "resource_version": "", + "self_link": "", + "uid": "" + } + ], + "spec": [ + { + "container_concurrency": 0, + "containers": [ + { + "args": null, + "command": null, + "env": [ + { + "name": "FIRESTORE_DATABASE_NAME", + "value": "sereact-imagedb", + "value_from": [] + }, + { + "name": "FIRESTORE_PROJECT_ID", + "value": "gen-lang-client-0424120530", + "value_from": [] + }, + { + "name": "GCS_BUCKET_NAME", + "value": "sereact-images", + "value_from": [] + }, + { + "name": "LOG_LEVEL", + "value": "INFO", + "value_from": [] + }, + { + "name": "QDRANT_API_KEY", + "value": "", + "value_from": [] + }, + { + "name": "QDRANT_HOST", + "value": "34.71.6.1", + "value_from": [] + }, + { + "name": "QDRANT_PORT", + "value": "6333", + "value_from": [] + }, + { + "name": "VECTOR_DB_ENVIRONMENT", + "value": "gcp-starter", + "value_from": [] + }, + { + "name": "VECTOR_DB_INDEX_NAME", + "value": "image-embeddings", + "value_from": [] + } + ], + "env_from": [], + "image": "gcr.io/gen-lang-client-0424120530/sereact-api:latest", + "liveness_probe": [], + "name": "", + "ports": [ + { + "container_port": 8000, + "name": "", + "protocol": "" + } + ], + "resources": [ + { + "limits": { + "cpu": "1", + "memory": "1Gi" + }, + "requests": null + } + ], + "startup_probe": [], + "volume_mounts": [], + "working_dir": "" + } + ], + "node_selector": null, + "service_account_name": "", + "serving_state": "", + "timeout_seconds": 0, + "volumes": [] + } + ] + } + ], + "timeouts": null, + "traffic": [ + { + "latest_revision": true, + "percent": 100, + "revision_name": "", + "tag": "", + "url": "" + } + ] + }, + "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "template" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "spec" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "containers" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "env" + } + ] + ], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMiJ9", + "dependencies": [ + "google_compute_address.vector_db_static_ip", + "google_compute_instance.vector_db_vm", + "google_project_service.services", + "google_service_account.vector_db_sa" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_compute_address", + "name": "vector_db_static_ip", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [] + }, + { + "mode": "managed", + "type": "google_compute_firewall", + "name": "qdrant_firewall", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "allow": [ + { + "ports": [ + "6333", + "6334" + ], + "protocol": "tcp" + } + ], + "creation_timestamp": "2025-05-24T10:57:27.640-07:00", + "deny": [], + "description": "", + "destination_ranges": [], + "direction": "INGRESS", + "disabled": false, + "enable_logging": null, + "id": "projects/gen-lang-client-0424120530/global/firewalls/allow-qdrant", + "log_config": [], + "name": "allow-qdrant", + "network": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/global/networks/default", + "priority": 1000, + "project": "gen-lang-client-0424120530", + "self_link": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/global/firewalls/allow-qdrant", + "source_ranges": [ + "0.0.0.0/0", + "10.0.0.0/8" + ], + "source_service_accounts": null, + "source_tags": null, + "target_service_accounts": null, + "target_tags": [ + "qdrant" + ], + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9" + } + ] + }, + { + "mode": "managed", + "type": "google_compute_instance", + "name": "vector_db_vm", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 6, + "attributes": { + "advanced_machine_features": [], + "allow_stopping_for_update": null, + "attached_disk": [], + "boot_disk": [ + { + "auto_delete": true, + "device_name": "persistent-disk-0", + "disk_encryption_key_raw": "", + "disk_encryption_key_rsa": "", + "disk_encryption_key_sha256": "", + "disk_encryption_service_account": "", + "force_attach": false, + "guest_os_features": [ + "VIRTIO_SCSI_MULTIQUEUE", + "SEV_CAPABLE", + "SEV_SNP_CAPABLE", + "SEV_LIVE_MIGRATABLE", + "SEV_LIVE_MIGRATABLE_V2", + "IDPF", + "TDX_CAPABLE", + "UEFI_COMPATIBLE", + "GVNIC" + ], + "initialize_params": [ + { + "architecture": "X86_64", + "enable_confidential_compute": false, + "image": "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20250523", + "labels": {}, + "provisioned_iops": 0, + "provisioned_throughput": 0, + "resource_manager_tags": null, + "resource_policies": [], + "size": 50, + "snapshot": "", + "source_image_encryption_key": [], + "source_snapshot_encryption_key": [], + "storage_pool": "", + "type": "pd-standard" + } + ], + "interface": "", + "kms_key_self_link": "", + "mode": "READ_WRITE", + "source": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/zones/us-central1-a/disks/sereact-vector-db" + } + ], + "can_ip_forward": false, + "confidential_instance_config": [], + "cpu_platform": "Intel Broadwell", + "creation_timestamp": "2025-05-24T10:57:41.174-07:00", + "current_status": "RUNNING", + "deletion_protection": false, + "description": "", + "desired_status": null, + "effective_labels": { + "goog-terraform-provisioned": "true" + }, + "enable_display": false, + "guest_accelerator": [], + "hostname": "", + "id": "projects/gen-lang-client-0424120530/zones/us-central1-a/instances/sereact-vector-db", + "instance_encryption_key": [], + "instance_id": "8076617322369889915", + "key_revocation_action_type": "", + "label_fingerprint": "vezUS-42LLM=", + "labels": null, + "machine_type": "e2-standard-2", + "metadata": null, + "metadata_fingerprint": "cE8FbgySELs=", + "metadata_startup_script": "#!/bin/bash\n\n# Qdrant Vector Database Installation Script\n# This script installs and configures Qdrant on Ubuntu 22.04\n\nset -e\n\n# Update system packages\napt-get update\napt-get upgrade -y\n\n# Install required packages\napt-get install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates\n\n# Install Docker\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg\necho \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list \u003e /dev/null\napt-get update\napt-get install -y docker-ce docker-ce-cli containerd.io\n\n# Start and enable Docker\nsystemctl start docker\nsystemctl enable docker\n\n# Create qdrant user and directories\nuseradd -r -s /bin/false qdrant || true\nmkdir -p /opt/qdrant/storage\nmkdir -p /opt/qdrant/config\nchown -R qdrant:qdrant /opt/qdrant\n\n# Create Qdrant configuration file\ncat \u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_cors: true\n\nstorage:\n storage_path: /qdrant/storage\n snapshots_path: /qdrant/snapshots\n on_disk_payload: true\n\ncluster:\n enabled: false\n\ntelemetry:\n disabled: true\n\nlog_level: INFO\nEOF\n\n# Create API key configuration if provided\nif [ -n \"\" ] \u0026\u0026 [ \"\" != \"\" ]; then\ncat \u003e\u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\n\nservice:\n api_key: \"\"\nEOF\nfi\n\n# Create systemd service for Qdrant\ncat \u003e /etc/systemd/system/qdrant.service \u003c\u003c EOF\n[Unit]\nDescription=Qdrant Vector Database\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nType=simple\nUser=root\nExecStartPre=-/usr/bin/docker stop qdrant\nExecStartPre=-/usr/bin/docker rm qdrant\nExecStart=/usr/bin/docker run --name qdrant \\\n -p 6333:6333 \\\n -p 6334:6334 \\\n -v /opt/qdrant/storage:/qdrant/storage:z \\\n -v /opt/qdrant/config/config.yaml:/qdrant/config/production.yaml:z \\\n qdrant/qdrant:latest\nExecStop=/usr/bin/docker stop qdrant\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n# Pull Qdrant Docker image\ndocker pull qdrant/qdrant:latest\n\n# Enable and start Qdrant service\nsystemctl daemon-reload\nsystemctl enable qdrant\nsystemctl start qdrant\n\n# Install monitoring tools\napt-get install -y htop iotop nethogs\n\n# Create a simple health check script\ncat \u003e /opt/qdrant/health_check.sh \u003c\u003c 'EOF'\n#!/bin/bash\nresponse=$(curl -s -o /dev/null -w \"%{http_code}\" http://localhost:6333/health)\nif [ \"$response\" = \"200\" ]; then\n echo \"Qdrant is healthy\"\n exit 0\nelse\n echo \"Qdrant is not responding properly (HTTP $response)\"\n exit 1\nfi\nEOF\n\nchmod +x /opt/qdrant/health_check.sh\n\n# Set up log rotation for Docker logs\ncat \u003e /etc/logrotate.d/docker \u003c\u003c EOF\n/var/lib/docker/containers/*/*.log {\n rotate 7\n daily\n compress\n size=1M\n missingok\n delaycompress\n copytruncate\n}\nEOF\n\n# Configure firewall (ufw)\nufw --force enable\nufw allow ssh\nufw allow 6333/tcp # Qdrant HTTP API\nufw allow 6334/tcp # Qdrant gRPC API\n\n# Create a simple backup script\ncat \u003e /opt/qdrant/backup.sh \u003c\u003c 'EOF'\n#!/bin/bash\nBACKUP_DIR=\"/opt/qdrant/backups\"\nDATE=$(date +%Y%m%d_%H%M%S)\nmkdir -p $BACKUP_DIR\n\n# Create snapshot via API\ncurl -X POST \"http://localhost:6333/snapshots\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"snapshot_name\": \"backup_'$DATE'\"}'\n\n# Copy storage directory\ntar -czf $BACKUP_DIR/qdrant_storage_$DATE.tar.gz -C /opt/qdrant storage/\n\n# Keep only last 7 backups\nfind $BACKUP_DIR -name \"*.tar.gz\" -mtime +7 -delete\n\necho \"Backup completed: $DATE\"\nEOF\n\nchmod +x /opt/qdrant/backup.sh\n\n# Set up daily backup cron job\necho \"0 2 * * * root /opt/qdrant/backup.sh \u003e\u003e /var/log/qdrant_backup.log 2\u003e\u00261\" \u003e\u003e /etc/crontab\n\n# Wait for Qdrant to be ready\necho \"Waiting for Qdrant to start...\"\nfor i in {1..30}; do\n if curl -s http://localhost:6333/health \u003e /dev/null; then\n echo \"Qdrant is ready!\"\n break\n fi\n echo \"Waiting... ($i/30)\"\n sleep 10\ndone\n\n# Create a default collection for image vectors\ncurl -X PUT \"http://localhost:6333/collections/image_vectors\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"vectors\": {\n \"size\": 512,\n \"distance\": \"Cosine\"\n },\n \"optimizers_config\": {\n \"default_segment_number\": 2\n },\n \"replication_factor\": 1\n }'\n\necho \"Qdrant installation and configuration completed!\"\necho \"Qdrant is accessible at:\"\necho \" HTTP API: http://$(curl -s ifconfig.me):6333\"\necho \" gRPC API: http://$(curl -s ifconfig.me):6334\"\necho \"Health check: /opt/qdrant/health_check.sh\"\necho \"Backup script: /opt/qdrant/backup.sh\" ", + "min_cpu_platform": "", + "name": "sereact-vector-db", + "network_interface": [ + { + "access_config": [ + { + "nat_ip": "34.71.6.1", + "network_tier": "PREMIUM", + "public_ptr_domain_name": "" + } + ], + "alias_ip_range": [], + "internal_ipv6_prefix_length": 0, + "ipv6_access_config": [], + "ipv6_access_type": "", + "ipv6_address": "", + "name": "nic0", + "network": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/global/networks/default", + "network_attachment": "", + "network_ip": "10.128.0.4", + "nic_type": "", + "queue_count": 0, + "stack_type": "IPV4_ONLY", + "subnetwork": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/regions/us-central1/subnetworks/default", + "subnetwork_project": "gen-lang-client-0424120530" + } + ], + "network_performance_config": [], + "params": [], + "project": "gen-lang-client-0424120530", + "reservation_affinity": [], + "resource_policies": null, + "scheduling": [ + { + "automatic_restart": true, + "availability_domain": 0, + "instance_termination_action": "", + "local_ssd_recovery_timeout": [], + "max_run_duration": [], + "min_node_cpus": 0, + "node_affinities": [], + "on_host_maintenance": "MIGRATE", + "on_instance_stop_action": [], + "preemptible": false, + "provisioning_model": "STANDARD", + "termination_time": "" + } + ], + "scratch_disk": [], + "self_link": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/zones/us-central1-a/instances/sereact-vector-db", + "service_account": [ + { + "email": "vector-db-sa@gen-lang-client-0424120530.iam.gserviceaccount.com", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + } + ], + "shielded_instance_config": [ + { + "enable_integrity_monitoring": true, + "enable_secure_boot": false, + "enable_vtpm": true + } + ], + "tags": [ + "qdrant", + "vector-db" + ], + "tags_fingerprint": "SoSixaaHMG4=", + "terraform_labels": { + "goog-terraform-provisioned": "true" + }, + "timeouts": null, + "zone": "us-central1-a" + }, + "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "metadata_startup_script" + } + ], + [ + { + "type": "get_attr", + "value": "boot_disk" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "disk_encryption_key_raw" + } + ], + [ + { + "type": "get_attr", + "value": "boot_disk" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "disk_encryption_key_rsa" + } + ] + ], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiNiJ9", + "dependencies": [ + "google_compute_address.vector_db_static_ip", + "google_project_service.services", + "google_service_account.vector_db_sa" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_project_service", + "name": "services", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "index_key": "cloudresourcemanager.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "gen-lang-client-0424120530/cloudresourcemanager.googleapis.com", + "project": "gen-lang-client-0424120530", + "service": "cloudresourcemanager.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "compute.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "gen-lang-client-0424120530/compute.googleapis.com", + "project": "gen-lang-client-0424120530", + "service": "compute.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "containerregistry.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "gen-lang-client-0424120530/containerregistry.googleapis.com", + "project": "gen-lang-client-0424120530", + "service": "containerregistry.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "firestore.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "gen-lang-client-0424120530/firestore.googleapis.com", + "project": "gen-lang-client-0424120530", + "service": "firestore.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "run.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "gen-lang-client-0424120530/run.googleapis.com", + "project": "gen-lang-client-0424120530", + "service": "run.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "storage.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "gen-lang-client-0424120530/storage.googleapis.com", + "project": "gen-lang-client-0424120530", + "service": "storage.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + } + ] + }, + { + "mode": "managed", + "type": "google_pubsub_subscription", + "name": "image_processing", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "ack_deadline_seconds": 600, + "bigquery_config": [], + "cloud_storage_config": [], + "dead_letter_policy": [ + { + "dead_letter_topic": "projects/gen-lang-client-0424120530/topics/image-processing-topic-dlq", + "max_delivery_attempts": 5 + } + ], + "effective_labels": { + "component": "image-processing", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "enable_exactly_once_delivery": false, + "enable_message_ordering": false, + "expiration_policy": [ + { + "ttl": "2678400s" + } + ], + "filter": "", + "id": "projects/gen-lang-client-0424120530/subscriptions/image-processing-topic-subscription", + "labels": { + "component": "image-processing", + "environment": "dev", + "service": "sereact" + }, + "message_retention_duration": "604800s", + "name": "image-processing-topic-subscription", + "project": "gen-lang-client-0424120530", + "push_config": [], + "retain_acked_messages": false, + "retry_policy": [ + { + "maximum_backoff": "600s", + "minimum_backoff": "10s" + } + ], + "terraform_labels": { + "component": "image-processing", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "timeouts": null, + "topic": "projects/gen-lang-client-0424120530/topics/image-processing-topic" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "google_pubsub_topic.image_processing", + "google_pubsub_topic.image_processing_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_pubsub_subscription", + "name": "image_processing_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "ack_deadline_seconds": 10, + "bigquery_config": [], + "cloud_storage_config": [], + "dead_letter_policy": [], + "effective_labels": { + "component": "image-processing-dlq", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "enable_exactly_once_delivery": false, + "enable_message_ordering": false, + "expiration_policy": [ + { + "ttl": "2678400s" + } + ], + "filter": "", + "id": "projects/gen-lang-client-0424120530/subscriptions/image-processing-topic-dlq-subscription", + "labels": { + "component": "image-processing-dlq", + "environment": "dev", + "service": "sereact" + }, + "message_retention_duration": "2592000s", + "name": "image-processing-topic-dlq-subscription", + "project": "gen-lang-client-0424120530", + "push_config": [], + "retain_acked_messages": true, + "retry_policy": [], + "terraform_labels": { + "component": "image-processing-dlq", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "timeouts": null, + "topic": "projects/gen-lang-client-0424120530/topics/image-processing-topic-dlq" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "google_pubsub_topic.image_processing_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_pubsub_subscription_iam_binding", + "name": "image_processing_subscriber", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwY15XOdN50=", + "id": "projects/gen-lang-client-0424120530/subscriptions/image-processing-topic-subscription/roles/pubsub.subscriber", + "members": [ + "serviceAccount:761163285547-compute@developer.gserviceaccount.com" + ], + "project": null, + "role": "roles/pubsub.subscriber", + "subscription": "image-processing-topic-subscription" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_project.current", + "google_pubsub_subscription.image_processing", + "google_pubsub_topic.image_processing", + "google_pubsub_topic.image_processing_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_pubsub_topic", + "name": "image_processing", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "effective_labels": { + "component": "image-processing", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "id": "projects/gen-lang-client-0424120530/topics/image-processing-topic", + "ingestion_data_source_settings": [], + "kms_key_name": "", + "labels": { + "component": "image-processing", + "environment": "dev", + "service": "sereact" + }, + "message_retention_duration": "", + "message_storage_policy": [], + "name": "image-processing-topic", + "project": "gen-lang-client-0424120530", + "schema_settings": [], + "terraform_labels": { + "component": "image-processing", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "google_pubsub_topic", + "name": "image_processing_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "effective_labels": { + "component": "image-processing-dlq", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "id": "projects/gen-lang-client-0424120530/topics/image-processing-topic-dlq", + "ingestion_data_source_settings": [], + "kms_key_name": "", + "labels": { + "component": "image-processing-dlq", + "environment": "dev", + "service": "sereact" + }, + "message_retention_duration": "", + "message_storage_policy": [], + "name": "image-processing-topic-dlq", + "project": "gen-lang-client-0424120530", + "schema_settings": [], + "terraform_labels": { + "component": "image-processing-dlq", + "environment": "dev", + "goog-terraform-provisioned": "true", + "service": "sereact" + }, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "google_pubsub_topic_iam_binding", + "name": "image_processing_publisher", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwY15XMktMU=", + "id": "projects/gen-lang-client-0424120530/topics/image-processing-topic/roles/pubsub.publisher", + "members": [ + "serviceAccount:761163285547-compute@developer.gserviceaccount.com" + ], + "project": "gen-lang-client-0424120530", + "role": "roles/pubsub.publisher", + "topic": "projects/gen-lang-client-0424120530/topics/image-processing-topic" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_project.current", + "google_pubsub_topic.image_processing" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_service_account", + "name": "vector_db_sa", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "vector-db-sa", + "create_ignore_already_exists": null, + "description": "Service account for the vector database VM", + "disabled": false, + "display_name": "Vector Database Service Account", + "email": "vector-db-sa@gen-lang-client-0424120530.iam.gserviceaccount.com", + "id": "projects/gen-lang-client-0424120530/serviceAccounts/vector-db-sa@gen-lang-client-0424120530.iam.gserviceaccount.com", + "member": "serviceAccount:vector-db-sa@gen-lang-client-0424120530.iam.gserviceaccount.com", + "name": "projects/gen-lang-client-0424120530/serviceAccounts/vector-db-sa@gen-lang-client-0424120530.iam.gserviceaccount.com", + "project": "gen-lang-client-0424120530", + "timeouts": null, + "unique_id": "108486040628486621877" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9fQ==" + } + ] + }, + { + "mode": "managed", + "type": "google_storage_bucket", + "name": "app_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 3, + "attributes": { + "autoclass": [], + "cors": [], + "custom_placement_config": [], + "default_event_based_hold": false, + "effective_labels": { + "goog-terraform-provisioned": "true" + }, + "enable_object_retention": false, + "encryption": [], + "force_destroy": false, + "hierarchical_namespace": [ + { + "enabled": false + } + ], + "id": "sereact-images", + "labels": null, + "lifecycle_rule": [], + "location": "US-CENTRAL1", + "logging": [], + "name": "sereact-images", + "project": "gen-lang-client-0424120530", + "project_number": 761163285547, + "public_access_prevention": "inherited", + "requester_pays": false, + "retention_policy": [], + "rpo": null, + "self_link": "https://www.googleapis.com/storage/v1/b/sereact-images", + "soft_delete_policy": [ + { + "effective_time": "2025-05-24T17:57:31.611Z", + "retention_duration_seconds": 604800 + } + ], + "storage_class": "STANDARD", + "terraform_labels": { + "goog-terraform-provisioned": "true" + }, + "time_created": "2025-05-24T17:57:31.611Z", + "timeouts": null, + "uniform_bucket_level_access": true, + "updated": "2025-05-24T17:57:31.611Z", + "url": "gs://sereact-images", + "versioning": [], + "website": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsInJlYWQiOjI0MDAwMDAwMDAwMCwidXBkYXRlIjoyNDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjMifQ==", + "dependencies": [ + "google_project_service.services" + ] + } + ] + } + ], "check_results": null } diff --git a/deployment/terraform/vm.tf b/deployment/terraform/vm.tf index 1d9a31a..3ed6fa2 100644 --- a/deployment/terraform/vm.tf +++ b/deployment/terraform/vm.tf @@ -15,7 +15,8 @@ resource "google_compute_instance" "vector_db_vm" { network_interface { network = "default" access_config { - # Ephemeral public IP + # Use static IP if enabled, otherwise ephemeral + nat_ip = var.use_static_ip ? google_compute_address.vector_db_static_ip[0].address : null } } @@ -63,42 +64,7 @@ resource "google_compute_firewall" "qdrant_firewall" { # Static IP for the vector DB VM (optional but recommended) resource "google_compute_address" "vector_db_static_ip" { + count = var.use_static_ip ? 1 : 0 name = "vector-db-static-ip" region = var.region -} - -# Attach the static IP to the VM -resource "google_compute_instance" "vector_db_vm_with_static_ip" { - count = var.use_static_ip ? 1 : 0 - name = "sereact-vector-db-static" - machine_type = "e2-standard-2" - zone = var.zone - - boot_disk { - initialize_params { - image = "ubuntu-os-cloud/ubuntu-2204-lts" - size = 50 - type = "pd-standard" - } - } - - network_interface { - network = "default" - access_config { - nat_ip = google_compute_address.vector_db_static_ip.address - } - } - - metadata_startup_script = templatefile("${path.module}/scripts/install_qdrant.sh", { - qdrant_api_key = var.qdrant_api_key - }) - - service_account { - email = google_service_account.vector_db_sa.email - scopes = ["cloud-platform"] - } - - tags = ["vector-db", "qdrant"] - - depends_on = [google_project_service.services] } \ No newline at end of file diff --git a/src/api/v1/search.py b/src/api/v1/search.py index b03e57f..0919888 100644 --- a/src/api/v1/search.py +++ b/src/api/v1/search.py @@ -16,10 +16,18 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Search"], prefix="/search") -# Initialize services -vector_db_service = VectorDatabaseService() +# Initialize services - delay VectorDatabaseService instantiation +vector_db_service = None embedding_service = EmbeddingService() +def get_vector_db_service(): + """Get or create the vector database service instance""" + global vector_db_service + if vector_db_service is None: + logger.info("Initializing VectorDatabaseService...") + vector_db_service = VectorDatabaseService() + return vector_db_service + @router.get("", response_model=SearchResponse) async def search_images( request: Request, @@ -52,7 +60,7 @@ async def search_images( raise HTTPException(status_code=400, detail="Failed to generate search embedding") # Search in vector database - search_results = vector_db_service.search_similar_images( + search_results = get_vector_db_service().search_similar_images( query_vector=query_embedding, limit=limit, score_threshold=threshold, @@ -156,7 +164,7 @@ async def search_images_advanced( raise HTTPException(status_code=400, detail="Failed to generate search embedding") # Search in vector database - search_results = vector_db_service.search_similar_images( + search_results = get_vector_db_service().search_similar_images( query_vector=query_embedding, limit=search_request.limit, score_threshold=search_request.threshold, @@ -289,14 +297,14 @@ async def find_similar_images( raise HTTPException(status_code=400, detail="Reference image does not have embeddings") # Get the embedding for the reference image - reference_data = vector_db_service.get_image_vector(image_id) + reference_data = get_vector_db_service().get_image_vector(image_id) if not reference_data or not reference_data.get('vector'): raise HTTPException(status_code=400, detail="Failed to get reference image embedding") reference_embedding = reference_data['vector'] # Search for similar images - search_results = vector_db_service.search_similar_images( + search_results = get_vector_db_service().search_similar_images( query_vector=reference_embedding, limit=limit + 1, # +1 to account for the reference image itself score_threshold=threshold, diff --git a/src/services/vector_db.py b/src/services/vector_db.py index c2c3539..b0eab9c 100644 --- a/src/services/vector_db.py +++ b/src/services/vector_db.py @@ -47,13 +47,15 @@ class VectorDatabaseService: if prefer_grpc is None: prefer_grpc = settings.QDRANT_PREFER_GRPC - # Auto-detect HTTPS usage if not specified + # Handle HTTPS setting properly if https is None: - # First check if explicitly set in settings - if hasattr(settings, 'QDRANT_HTTPS') and os.getenv("QDRANT_HTTPS"): + # First check if explicitly set in environment + qdrant_https_env = os.getenv("QDRANT_HTTPS") + if qdrant_https_env is not None: + # Use the parsed boolean value from settings https = settings.QDRANT_HTTPS else: - # Use HTTP for localhost and known development IPs, HTTPS for others + # Auto-detect: Use HTTP for localhost and known development IPs, HTTPS for others https = not (self.host in ["localhost", "127.0.0.1"] or self.host.startswith("192.168.") or self.host.startswith("10.")) # Override for specific known HTTP-only servers if self.host == "34.171.134.17": diff --git a/test_qdrant_connection.py b/test_qdrant_connection.py index 20a2f7b..f98b993 100644 --- a/test_qdrant_connection.py +++ b/test_qdrant_connection.py @@ -8,20 +8,34 @@ import os import sys from src.services.vector_db import VectorDatabaseService +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + def test_qdrant_connection(): """Test the connection to Qdrant""" # Set environment variables from deployed infrastructure - os.environ['QDRANT_HOST'] = '34.171.134.17' - os.environ['QDRANT_PORT'] = '6333' + # os.environ['QDRANT_HOST'] = '34.171.134.17' + # os.environ['QDRANT_PORT'] = '6333' + # os.environ['QDRANT_HTTPS'] = 'false' # Explicitly disable HTTPS + # os.environ['QDRANT_PREFER_GRPC'] = 'false' # Explicitly disable gRPC try: print("Testing Qdrant connection...") - print(f"Host: {os.environ['QDRANT_HOST']}") - print(f"Port: {os.environ['QDRANT_PORT']}") + # print(f"Host: {os.environ['QDRANT_HOST']}") + # print(f"Port: {os.environ['QDRANT_PORT']}") + # print(f"HTTPS: {os.environ['QDRANT_HTTPS']}") + # print(f"gRPC: {os.environ['QDRANT_PREFER_GRPC']}") - # Initialize the service - vector_db = VectorDatabaseService() + # Initialize the service with explicit parameters to ensure HTTP is used + vector_db = VectorDatabaseService( + host=os.environ['QDRANT_HOST'], + port=int(os.environ['QDRANT_PORT']), + prefer_grpc=False, + https=False + ) # Test health check is_healthy = vector_db.health_check()