From 74fc51e34e9739b226e5b8185a21b9add519cbfb Mon Sep 17 00:00:00 2001 From: johnpccd Date: Sun, 25 May 2025 18:16:13 +0200 Subject: [PATCH] cleanup --- README.md | 14 +--- deployment/deploy.sh | 2 - deployment/terraform/deploy-with-apis.sh | 64 ---------------- deployment/terraform/scripts/get_qdrant_ip.sh | 73 ------------------- 4 files changed, 4 insertions(+), 149 deletions(-) delete mode 100644 deployment/terraform/deploy-with-apis.sh delete mode 100644 deployment/terraform/scripts/get_qdrant_ip.sh diff --git a/README.md b/README.md index 6938783..257de22 100644 --- a/README.md +++ b/README.md @@ -419,22 +419,16 @@ This modular architecture provides several benefits: ## TODO ### High Priority -- [ ] Remove Pinecone integration and complete Qdrant migration -- [ ] Test and validate vector search functionality with Qdrant -- [ ] Test Cloud Function image processing pipeline -- [ ] Validate VM setup for self-hosted Qdrant instance +- [ ] Thumbnail generation +- [ ] Secret management +- [ ] Scale Vector DB to multiple nodes ### Medium Priority -- [ ] Add comprehensive logging for vector search operations - [ ] Implement caching layer for frequently accessed embeddings - [ ] Implement caching for frequently accessed data -- [ ] Add monitoring and alerting for vector database performance -- [ ] Document vector search API endpoints -- [ ] Set up Qdrant cluster with multiple nodes - [ ] Consider adding pagination to admin endpoints (users, teams, API keys) if datasets grow large ### Low Priority -- [ ] Terraform dependencies - [ ] Move all auth logic to auth module - [ ] Move cloud function code to src folder and reuse code with embedding service -- [ ] Thumbnail generation +- [ ] Remove Pinecone integration diff --git a/deployment/deploy.sh b/deployment/deploy.sh index 543e175..a7c36fd 100644 --- a/deployment/deploy.sh +++ b/deployment/deploy.sh @@ -318,8 +318,6 @@ if [ "$DEPLOY" = true ]; then echo " QDRANT_HOST=$QDRANT_IP" echo " QDRANT_PORT=6333" echo "" - echo "To get detailed Qdrant information, run:" - echo " ./deployment/terraform/scripts/get_qdrant_ip.sh" fi echo "All operations completed." \ No newline at end of file diff --git a/deployment/terraform/deploy-with-apis.sh b/deployment/terraform/deploy-with-apis.sh deleted file mode 100644 index 47d9edf..0000000 --- a/deployment/terraform/deploy-with-apis.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# Deployment script that ensures APIs are enabled before deploying Cloud Functions -set -e - -echo "🚀 Starting Terraform deployment with API enablement..." - -# Get project ID from terraform.tfvars or environment -PROJECT_ID=${GOOGLE_CLOUD_PROJECT:-$(grep 'project_id' terraform.tfvars | cut -d'"' -f2)} - -if [ -z "$PROJECT_ID" ]; then - echo "❌ Error: PROJECT_ID not found. Set GOOGLE_CLOUD_PROJECT or ensure terraform.tfvars has project_id" - exit 1 -fi - -echo "📋 Project ID: $PROJECT_ID" - -# Step 1: Enable APIs first -echo "🔧 Step 1: Enabling required APIs..." -terraform apply -target=google_project_service.services -auto-approve - -# Step 2: Wait for APIs to propagate -echo "⏳ Step 2: Waiting for APIs to propagate (30 seconds)..." -sleep 30 - -# Step 3: Verify critical APIs are enabled -echo "✅ Step 3: Verifying API enablement..." -REQUIRED_APIS=( - "cloudfunctions.googleapis.com" - "eventarc.googleapis.com" - "pubsub.googleapis.com" - "cloudbuild.googleapis.com" -) - -for api in "${REQUIRED_APIS[@]}"; do - echo " Checking $api..." - if gcloud services list --enabled --project="$PROJECT_ID" --filter="name:$api" --format="value(name)" | grep -q "$api"; then - echo " ✅ $api is enabled" - else - echo " ❌ $api is not enabled yet, waiting longer..." - sleep 15 - fi -done - -# Step 4: Deploy infrastructure -echo "🏗️ Step 4: Deploying infrastructure..." -terraform apply -target=google_storage_bucket.app_bucket -target=google_firestore_database.database -target=google_compute_instance.vector_db_vm -auto-approve - -# Step 5: Deploy Pub/Sub resources -echo "📨 Step 5: Deploying Pub/Sub resources..." -terraform apply -target=google_pubsub_topic.image_processing -target=google_pubsub_topic.image_processing_dlq -auto-approve - -# Step 6: Deploy Cloud Function -echo "⚡ Step 6: Deploying Cloud Function..." -terraform apply -target=google_cloudfunctions2_function.image_processor -auto-approve - -# Step 7: Deploy remaining resources -echo "🎯 Step 7: Deploying remaining resources..." -terraform apply -auto-approve - -echo "🎉 Deployment completed successfully!" -echo "" -echo "📊 Deployment Summary:" -terraform output deployment_summary 2>/dev/null || echo " Run 'terraform output' to see all outputs" \ No newline at end of file diff --git a/deployment/terraform/scripts/get_qdrant_ip.sh b/deployment/terraform/scripts/get_qdrant_ip.sh deleted file mode 100644 index 18527d4..0000000 --- a/deployment/terraform/scripts/get_qdrant_ip.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# Script to get Qdrant VM IP address from Terraform outputs -# Usage: ./get_qdrant_ip.sh [--internal] - -set -e - -# Change to terraform directory -cd "$(dirname "$0")/.." - -# Check if terraform state exists -if [ ! -f "terraform.tfstate" ]; then - echo "ERROR: No terraform.tfstate found. Please run 'terraform apply' first." - exit 1 -fi - -# Parse command line arguments -INTERNAL=false -if [ "$1" = "--internal" ]; then - INTERNAL=true -fi - -echo "Retrieving Qdrant VM IP address..." -echo "==================================" - -# Get VM information -VM_NAME=$(terraform output -raw vector_db_vm_name 2>/dev/null || echo "N/A") -echo "VM Name: $VM_NAME" - -if [ "$INTERNAL" = true ]; then - # Get internal IP - QDRANT_IP=$(terraform output -raw vector_db_vm_internal_ip 2>/dev/null || echo "N/A") - echo "Internal IP: $QDRANT_IP" - echo "Qdrant HTTP Endpoint (Internal): http://$QDRANT_IP:6333" - echo "Qdrant gRPC Endpoint (Internal): http://$QDRANT_IP:6334" -else - # Get external IP - QDRANT_IP=$(terraform output -raw vector_db_vm_external_ip 2>/dev/null || echo "N/A") - echo "External IP: $QDRANT_IP" - echo "Qdrant HTTP Endpoint: http://$QDRANT_IP:6333" - echo "Qdrant gRPC Endpoint: http://$QDRANT_IP:6334" -fi - -# Check if static IP is enabled -STATIC_IP_ENABLED=$(terraform output -raw vector_db_static_ip 2>/dev/null || echo "null") -if [ "$STATIC_IP_ENABLED" != "null" ] && [ "$STATIC_IP_ENABLED" != "" ]; then - echo "Static IP: $STATIC_IP_ENABLED (enabled)" -else - echo "Static IP: Not enabled (using ephemeral IP)" -fi - -# Get Cloud Run service URL -CLOUD_RUN_URL=$(terraform output -raw cloud_run_url 2>/dev/null || echo "N/A") -echo "Cloud Run URL: $CLOUD_RUN_URL" - -echo "" -echo "Environment Variables for Cloud Run:" -echo "QDRANT_HOST=$QDRANT_IP" -echo "QDRANT_PORT=6333" - -# Test connectivity (optional) -echo "" -echo "Testing Qdrant connectivity..." -if command -v curl &> /dev/null; then - if curl -s --connect-timeout 5 "http://$QDRANT_IP:6333/health" > /dev/null; then - echo "✓ Qdrant is accessible at http://$QDRANT_IP:6333" - else - echo "✗ Qdrant is not accessible at http://$QDRANT_IP:6333" - echo " This might be normal if the VM is still starting up or firewall rules need adjustment." - fi -else - echo "curl not available - skipping connectivity test" -fi \ No newline at end of file