From 8983114868a4b897ba2f4203dc903a5bdddf49b3 Mon Sep 17 00:00:00 2001 From: johnpccd Date: Sun, 25 May 2025 17:54:40 +0200 Subject: [PATCH] cleanup --- .env.example | 35 +- README.md | 70 ++- deployment/README.md | 66 --- deployment/terraform/README.md | 398 ------------------ deployment/terraform/main.tf | 9 - deployment/terraform/terraform.tfvars.example | 1 - src/config/config.py | 8 - 7 files changed, 39 insertions(+), 548 deletions(-) delete mode 100644 deployment/README.md delete mode 100644 deployment/terraform/README.md diff --git a/.env.example b/.env.example index a2a2d2e..dd2f3a6 100644 --- a/.env.example +++ b/.env.example @@ -1,32 +1,25 @@ -# Application Settings -PROJECT_NAME="Image Management API" -API_V1_STR="/api/v1" -LOG_LEVEL=INFO +# Project Environment Variables +ENVIRONMENT=development +LOG_LEVEL=DEBUG -# CORS Settings -CORS_ORIGINS=* +# CORS settings - Must be a valid JSON list of strings or comma-separated values +CORS_ORIGINS=["*"] -# Database Settings # Firestore settings FIRESTORE_PROJECT_ID=gen-lang-client-0424120530 FIRESTORE_DATABASE_NAME=sereact-imagedb FIRESTORE_CREDENTIALS_FILE=firestore-credentials.json -# Google Cloud Firestore Settings (used when DATABASE_TYPE=firestore) -# Path to service account credentials file (optional, uses application default credentials if not set) -GCS_CREDENTIALS_FILE=path/to/credentials.json +# Google Cloud Storage settings +GCS_BUCKET_NAME=sereact-images +GCS_CREDENTIALS_FILE=firestore-credentials.json -# Google Cloud Storage Settings -GCS_BUCKET_NAME=your-bucket-name - -# Security Settings +# Security settings API_KEY_SECRET=super-secret-key-for-development-only API_KEY_EXPIRY_DAYS=365 -# Vector Database Settings (for image embeddings) -VECTOR_DB_API_KEY=your-pinecone-api-key -VECTOR_DB_ENVIRONMENT=your-pinecone-environment -VECTOR_DB_INDEX_NAME=image-embeddings - -# Rate Limiting -RATE_LIMIT_PER_MINUTE=100 \ No newline at end of file +# Vector Database settings +QDRANT_HOST=35.193.174.125 +QDRANT_PORT=6333 +QDRANT_HTTPS=false +QDRANT_PREFER_GRPC=false \ No newline at end of file diff --git a/README.md b/README.md index 3e0e7db..da2b2d5 100644 --- a/README.md +++ b/README.md @@ -137,23 +137,7 @@ The system includes a dedicated Google Compute Engine VM running Qdrant vector d - **VM Specifications**: 2 vCPUs, 8GB RAM, 50GB disk (e2-standard-2) - **Operating System**: Ubuntu 22.04 LTS - **Vector Database**: Qdrant (latest version via Docker) -- **Ports**: 6333 (HTTP API), 6334 (gRPC API) -- **Features**: - - Automatic installation and configuration via startup script - - Daily automated backups - - Health monitoring - - Firewall configuration - - Optional static IP assignment - - API key authentication support -### **Vector Database Features** - -- **High Performance**: Optimized for image vector similarity search -- **Scalable**: Can handle millions of image vectors -- **Persistent Storage**: Data persisted on VM disk with automated backups -- **RESTful API**: Easy integration with Python client -- **Cosine Similarity**: Optimized for image embedding comparisons -- **Metadata Filtering**: Support for complex search filters ### **AI Embedding Model** @@ -163,11 +147,6 @@ Uses Google's Vertex AI multimodal embedding model for generating high-quality i - **Provider**: Google Vertex AI - **Type**: Multimodal embedding model (supports both images and text) - **Output Dimensions**: **1408-dimensional vectors** -- **Data Type**: Float32, normalized vectors (norm ≈ 1.0) -- **Similarity Metric**: Cosine similarity -- **Use Case**: Semantic image search and similarity matching - -**⚠️ Important Configuration**: Ensure your Qdrant collection is configured with **1408 dimensions** to match the Vertex AI model output. Dimension mismatches will cause embedding storage failures. ## Setup and Installation @@ -199,30 +178,31 @@ Uses Google's Vertex AI multimodal embedding model for generating high-quality i 4. Create a `.env` file with the following environment variables: ``` - # Firestore - FIRESTORE_PROJECT_ID=your-gcp-project-id - FIRESTORE_CREDENTIALS_FILE=path/to/firestore-credentials.json - - # Google Cloud Storage - GCS_BUCKET_NAME=your-bucket-name - GCS_CREDENTIALS_FILE=path/to/credentials.json - - # Google Pub/Sub - PUBSUB_TOPIC=image-processing-topic - PUBSUB_SUBSCRIPTION=image-processing-subscription - - # Google Vertex AI (for image embeddings) - VERTEX_AI_LOCATION=us-central1 - GOOGLE_CLOUD_PROJECT=your-gcp-project-id - - # Security - API_KEY_SECRET=your-secret-key - - # Vector database (Qdrant) - QDRANT_HOST=your-vm-external-ip - QDRANT_API_KEY=your-qdrant-api-key # Optional - QDRANT_COLLECTION=image_vectors - QDRANT_HTTPS=false + # Project Environment Variables + ENVIRONMENT=development + LOG_LEVEL=DEBUG + + # CORS settings - Must be a valid JSON list of strings or comma-separated values + CORS_ORIGINS=["*"] + + # Firestore settings + FIRESTORE_PROJECT_ID=gen-lang-client-0424120530 + FIRESTORE_DATABASE_NAME=sereact-imagedb + FIRESTORE_CREDENTIALS_FILE=firestore-credentials.json + + # Google Cloud Storage settings + GCS_BUCKET_NAME=sereact-images + GCS_CREDENTIALS_FILE=firestore-credentials.json + + # Security settings + API_KEY_SECRET=super-secret-key + API_KEY_EXPIRY_DAYS=365 + + # Vector Database settings + QDRANT_HOST=35.193.174.125 + QDRANT_PORT=6333 + QDRANT_HTTPS=false + QDRANT_PREFER_GRPC=false ``` 5. **Deploy Infrastructure** (Required for vector database): diff --git a/deployment/README.md b/deployment/README.md deleted file mode 100644 index 88134c3..0000000 --- a/deployment/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Deployment Options for Sereact - -This directory contains multiple options for deploying the Sereact application: - -## Terraform Infrastructure (`/terraform`) - -The Terraform configuration automates the provisioning of all required Google Cloud resources: - -- Google Cloud Run service -- Google Container Registry (GCR) -- Cloud Firestore -- Cloud Storage buckets - -See [terraform/README.md](terraform/README.md) for detailed instructions. - -## Cloud Run Deployment (`/cloud-run`) - -The `service.yaml` file defines the Cloud Run service configuration which can be deployed using: - -```bash -gcloud run services replace deployment/cloud-run/service.yaml --region=us-central1 -``` - -## Deployment Script (`deploy.sh`) - -For convenience, a deployment script is provided to handle the entire deployment workflow: - -```bash -# Provision infrastructure with Terraform -./deployment/deploy.sh --provision - -# Build and push Docker image -./deployment/deploy.sh --build - -# Deploy to Cloud Run -./deployment/deploy.sh --deploy - -# Do everything (provision, build, deploy) -./deployment/deploy.sh --all -``` - -## CI/CD Pipelines - -For CI/CD integration, consider using: - -1. **GitHub Actions**: Sample workflow included in terraform/README.md -2. **Cloud Build**: Configure a `cloudbuild.yaml` in your repository -3. **Jenkins**: Use the `deploy.sh` script in your pipeline - -## Managing Secrets - -Sensitive data should be managed using Google Secret Manager: - -```bash -# Create a secret -gcloud secrets create sereact-api-key-secret --replication-policy="automatic" -gcloud secrets create sereact-vector-db-key --replication-policy="automatic" - -# Add a secret version -echo -n "your-api-key-secret" | gcloud secrets versions add sereact-api-key-secret --data-file=- -echo -n "your-vector-db-key" | gcloud secrets versions add sereact-vector-db-key --data-file=- - -# Update Cloud Run service to use the secrets -gcloud run services update sereact \ - --update-secrets=API_KEY_SECRET=sereact-api-key-secret:latest,VECTOR_DB_API_KEY=sereact-vector-db-key:latest -``` \ No newline at end of file diff --git a/deployment/terraform/README.md b/deployment/terraform/README.md deleted file mode 100644 index 523908f..0000000 --- a/deployment/terraform/README.md +++ /dev/null @@ -1,398 +0,0 @@ -# Terraform Configuration for Sereact - -This directory contains Terraform configurations to provision the required Google Cloud resources for Sereact: - -- Google Cloud Run service -- Google Cloud Function for image processing -- Google Container Registry (GCR) -- Firestore database -- Cloud Storage bucket -- Pub/Sub topics and subscriptions -- Qdrant vector database VM - -## Prerequisites - -1. Install [Terraform](https://www.terraform.io/downloads) (v1.0.0+) -2. Install [Google Cloud SDK](https://cloud.google.com/sdk/docs/install) -3. Authenticate with Google Cloud: - ```bash - gcloud auth login - gcloud auth application-default login - ``` -4. Create or select a Google Cloud project: - ```bash - gcloud projects create PROJECT_ID --name="Sereact Project" # optional - gcloud config set project PROJECT_ID - ``` - -## Architecture Overview - -The infrastructure includes: -- **Cloud Run Service**: Main application service -- **Cloud Function**: Image processing and embedding generation -- **Qdrant VM**: Vector database running on Compute Engine -- **Firestore**: Document database -- **Cloud Storage**: File storage -- **Pub/Sub**: Message queue for asynchronous image processing -- **Automatic IP Configuration**: Qdrant VM IP is automatically passed to Cloud Run and Cloud Function - -## Cloud Function Integration - -The Cloud Function is now fully integrated into Terraform and handles: - -1. **Image Processing**: Triggered by Pub/Sub messages -2. **Embedding Generation**: Uses Google Cloud Vision API -3. **Vector Storage**: Stores embeddings in Qdrant -4. **Status Updates**: Updates Firestore with processing results - -### Automatic Configuration - -The Cloud Function is automatically configured with: -- **Qdrant Connection**: VM IP address passed as environment variable -- **IAM Permissions**: Firestore, Storage, and Vision API access -- **Retry Policy**: Automatic retries on failure -- **Scaling**: 0-10 instances based on demand - -## Setup and Usage - -### Option 1: Automated Deployment (Recommended) - -Use the provided script that handles API enablement and proper deployment order: - -```bash -cd deployment/terraform -./deploy-with-apis.sh -``` - -This script will: -1. Enable all required APIs -2. Wait for API propagation -3. Deploy infrastructure in the correct order -4. Handle Cloud Functions v2 dependencies - -### Option 2: Manual Deployment - -1. Copy the example variables file and edit it with your values: - ```bash - cp terraform.tfvars.example terraform.tfvars - # Edit terraform.tfvars with your project-specific values - ``` - -2. Initialize Terraform: - ```bash - terraform init - ``` - -3. Enable APIs first (required for Cloud Functions v2): - ```bash - terraform apply -target=google_project_service.services - ``` - -4. Wait for APIs to propagate (2-3 minutes), then deploy everything: - ```bash - terraform apply - ``` - -## Required APIs - -The following APIs are automatically enabled by Terraform: - -- `cloudresourcemanager.googleapis.com` - Resource management -- `containerregistry.googleapis.com` - Container Registry -- `run.googleapis.com` - Cloud Run -- `firestore.googleapis.com` - Firestore database -- `storage.googleapis.com` - Cloud Storage -- `compute.googleapis.com` - Compute Engine (for Qdrant VM) -- `cloudfunctions.googleapis.com` - Cloud Functions -- `cloudbuild.googleapis.com` - Cloud Build (for function deployment) -- `eventarc.googleapis.com` - Eventarc (required for Cloud Functions v2) -- `pubsub.googleapis.com` - Pub/Sub messaging - -**Note**: If you get an Eventarc API error, use the automated deployment script or wait 2-3 minutes after enabling APIs before deploying the Cloud Function. - -5. After provisioning, you'll see outputs including: - - Cloud Run service URL - - Storage bucket name - - Firestore database ID - - Container Registry URL - -## Managing Secrets - -Secrets for environment variables (API_KEY_SECRET, VECTOR_DB_API_KEY, etc.) should be managed separately using Google Secret Manager: - -```bash -# Create secrets -gcloud secrets create sereact-api-key-secret --replication-policy="automatic" -gcloud secrets create sereact-vector-db-key --replication-policy="automatic" - -# Add secret versions -echo -n "your-secret-value" | gcloud secrets versions add sereact-api-key-secret --data-file=- -echo -n "your-secret-value" | gcloud secrets versions add sereact-vector-db-key --data-file=- - -# Update Cloud Run service to use secrets -gcloud run services update sereact \ - --update-secrets=API_KEY_SECRET=sereact-api-key-secret:latest,VECTOR_DB_API_KEY=sereact-vector-db-key:latest -``` - -## CI/CD Integration - -To integrate this with CI/CD, store the `terraform.tfvars` securely in your CI/CD system and run Terraform as part of your deployment pipeline: - -```yaml -# Example GitHub Actions step -- name: Terraform Apply - run: | - cd deployment/terraform - terraform init - terraform apply -auto-approve - env: - GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SA_KEY }} -``` - -## Destroying Resources - -To destroy all provisioned resources: - -```bash -terraform destroy -``` - -# Terraform Infrastructure for Sereact - -This directory contains Terraform configurations to deploy the complete Sereact infrastructure on Google Cloud Platform, including automatic configuration of Qdrant VM IP addresses for Cloud Run. - -## Architecture Overview - -The infrastructure includes: -- **Cloud Run Service**: Main application service -- **Qdrant VM**: Vector database running on Compute Engine -- **Firestore**: Document database -- **Cloud Storage**: File storage -- **Automatic IP Configuration**: Qdrant VM IP is automatically passed to Cloud Run - -## Qdrant VM IP Address Integration - -The Terraform configuration automatically handles passing the Qdrant VM IP address to your Cloud Run service: - -### How It Works - -1. **VM Creation**: Terraform creates a Compute Engine VM running Qdrant -2. **IP Address Extraction**: Terraform extracts the VM's external IP address -3. **Environment Variable Injection**: The IP is automatically set as `QDRANT_HOST` in Cloud Run -4. **Dependency Management**: Cloud Run waits for the VM to be created before deploying - -### Configuration Options - -#### Static vs Dynamic IP - -You can choose between static and dynamic IP addresses: - -```hcl -# In terraform.tfvars -use_static_ip = true # Use static IP (recommended for production) -use_static_ip = false # Use ephemeral IP (cheaper for development) -``` - -#### Environment Variables Set Automatically - -The following environment variables are automatically configured in Cloud Run: - -```bash -QDRANT_HOST= # Automatically set from VM -QDRANT_PORT=6333 # HTTP port -QDRANT_API_KEY= # From terraform.tfvars -``` - -## Quick Start - -### 1. Prerequisites - -```bash -# Install required tools -gcloud auth login -gcloud config set project YOUR_PROJECT_ID -terraform --version -``` - -### 2. Configuration - -```bash -# Copy and edit configuration -cp terraform.tfvars.example terraform.tfvars -# Edit terraform.tfvars with your values -``` - -### 3. Deploy Everything - -```bash -# From the root directory -./deployment/deploy.sh --deploy --build -``` - -### 4. Get Qdrant Information - -```bash -# Get Qdrant VM IP and endpoints -./deployment/terraform/scripts/get_qdrant_ip.sh - -# Get internal IP (for VPC connections) -./deployment/terraform/scripts/get_qdrant_ip.sh --internal -``` - -## Terraform Outputs - -The configuration provides several outputs for integration: - -```bash -# Get specific outputs -terraform output cloud_run_url # Cloud Run service URL -terraform output vector_db_vm_external_ip # Qdrant VM external IP -terraform output vector_db_vm_internal_ip # Qdrant VM internal IP -terraform output qdrant_http_endpoint # Full HTTP endpoint -terraform output cloud_run_qdrant_host # IP configured for Cloud Run - -# Get deployment summary -terraform output deployment_summary -``` - -## File Structure - -``` -terraform/ -├── main.tf # Cloud Run and main resources -├── vm.tf # Qdrant VM configuration -├── outputs.tf # Output definitions -├── variables.tf # Variable definitions -├── terraform.tfvars # Your configuration values -├── scripts/ -│ ├── get_qdrant_ip.sh # Helper script to get VM IP -│ └── install_qdrant.sh # VM startup script -└── README.md # This file -``` - -## Environment Variables Reference - -### Automatically Configured - -These are set automatically by Terraform: - -| Variable | Source | Description | -|----------|--------|-------------| -| `QDRANT_HOST` | VM external IP | Qdrant server IP address | -| `QDRANT_PORT` | Static value | Qdrant HTTP port (6333) | -| `QDRANT_API_KEY` | terraform.tfvars | Qdrant authentication key | -| `FIRESTORE_PROJECT_ID` | terraform.tfvars | GCP project ID | -| `GCS_BUCKET_NAME` | terraform.tfvars | Storage bucket name | - -### Manual Configuration Required - -Set these in `terraform.tfvars`: - -```hcl -project_id = "your-gcp-project-id" -storage_bucket_name = "your-bucket-name" -qdrant_api_key = "your-qdrant-api-key" -use_static_ip = true # or false -``` - -## Networking Configuration - -### Firewall Rules - -The configuration automatically creates firewall rules to allow: -- Qdrant HTTP traffic (port 6333) -- Qdrant gRPC traffic (port 6334) -- Access from Cloud Run to Qdrant VM - -### IP Address Types - -#### External IP (Default) -- Used for Cloud Run → Qdrant communication -- Accessible from internet (with firewall rules) -- Can be static or ephemeral - -#### Internal IP (Alternative) -- Available for VPC-native connections -- More secure but requires VPC configuration -- Use `cloud_run_qdrant_host_internal` output - -## Troubleshooting - -### Check Qdrant Connectivity - -```bash -# Get IP and test connection -./deployment/terraform/scripts/get_qdrant_ip.sh - -# Manual test -QDRANT_IP=$(terraform output -raw vector_db_vm_external_ip) -curl http://$QDRANT_IP:6333/health -``` - -### Verify Cloud Run Environment - -```bash -# Check Cloud Run service -gcloud run services describe sereact --region=us-central1 - -# Check environment variables -gcloud run services describe sereact --region=us-central1 \ - --format="value(spec.template.spec.containers[0].env[].name,spec.template.spec.containers[0].env[].value)" -``` - -### Common Issues - -1. **VM not ready**: Qdrant installation takes 2-3 minutes after VM creation -2. **Firewall blocking**: Check that your IP is in `allowed_cidr_blocks` -3. **Wrong IP type**: Ensure you're using external IP for Cloud Run connections - -## Security Considerations - -### Production Recommendations - -1. **Use Static IP**: Set `use_static_ip = true` -2. **Restrict Access**: Set specific CIDR blocks in `allowed_cidr_blocks` -3. **Use Internal IP**: Consider VPC-native networking for internal communication -4. **Secure API Key**: Store `qdrant_api_key` in Secret Manager - -### Development Setup - -```hcl -# terraform.tfvars for development -use_static_ip = false -allowed_cidr_blocks = "0.0.0.0/0" # Open access (not for production!) -``` - -## Cost Optimization - -### Development -- Use `use_static_ip = false` to avoid static IP charges -- Use smaller VM sizes: `machine_type = "e2-micro"` - -### Production -- Use `use_static_ip = true` for consistent connectivity -- Use appropriate VM sizes based on load - -## Advanced Configuration - -### Custom VM Configuration - -Edit `vm.tf` to customize: -- Machine type -- Disk size -- Startup script -- Network configuration - -### VPC-Native Networking - -For internal-only communication: -1. Create VPC connector -2. Configure Cloud Run to use VPC -3. Use internal IP output: `cloud_run_qdrant_host_internal` - -## Support - -For issues with: -- **Terraform**: Check `terraform plan` output -- **VM connectivity**: Use the helper script `get_qdrant_ip.sh` -- **Cloud Run**: Check logs with `gcloud run logs tail sereact` \ No newline at end of file diff --git a/deployment/terraform/main.tf b/deployment/terraform/main.tf index 210af6e..4361923 100644 --- a/deployment/terraform/main.tf +++ b/deployment/terraform/main.tf @@ -105,15 +105,6 @@ resource "google_cloud_run_service" "sereact" { value = var.storage_bucket_name } - env { - name = "VECTOR_DB_ENVIRONMENT" - value = var.vector_db_environment - } - - env { - name = "VECTOR_DB_INDEX_NAME" - value = var.vector_db_index_name - } env { name = "QDRANT_HOST" diff --git a/deployment/terraform/terraform.tfvars.example b/deployment/terraform/terraform.tfvars.example index fc73105..97db9dc 100644 --- a/deployment/terraform/terraform.tfvars.example +++ b/deployment/terraform/terraform.tfvars.example @@ -11,6 +11,5 @@ use_static_ip = false # Set to true if you want a static # Secret Configuration (all optional - random values will be generated if not provided) api_key_secret = "" # Optional: will generate random key if empty -vector_db_api_key = "" # Optional: for external vector DB like Pinecone gcs_credentials_json = "" # Optional: JSON content of GCS service account key firestore_credentials_json = "" # Optional: JSON content of Firestore service account key \ No newline at end of file diff --git a/src/config/config.py b/src/config/config.py index 5f53e70..e3d3d62 100644 --- a/src/config/config.py +++ b/src/config/config.py @@ -84,14 +84,6 @@ class Settings(BaseSettings): API_KEY_SECRET: str = os.getenv("API_KEY_SECRET", "super-secret-key-for-development-only") API_KEY_EXPIRY_DAYS: int = int(os.getenv("API_KEY_EXPIRY_DAYS", "365")) - # Vector Database settings (for image embeddings) - VECTOR_DB_API_KEY: str = os.getenv("VECTOR_DB_API_KEY", "") - VECTOR_DB_ENVIRONMENT: str = os.getenv("VECTOR_DB_ENVIRONMENT", "") - VECTOR_DB_INDEX_NAME: str = os.getenv("VECTOR_DB_INDEX_NAME", "image-embeddings") - - # Rate limiting - RATE_LIMIT_PER_MINUTE: int = int(os.getenv("RATE_LIMIT_PER_MINUTE", "100")) - # Qdrant settings QDRANT_HOST: str = os.getenv("QDRANT_HOST", "localhost") QDRANT_PORT: int = int(os.getenv("QDRANT_PORT", "6333"))