cleanup
This commit is contained in:
parent
cd26c397e3
commit
74fc51e34e
14
README.md
14
README.md
@ -419,22 +419,16 @@ This modular architecture provides several benefits:
|
||||
## TODO
|
||||
|
||||
### High Priority
|
||||
- [ ] Remove Pinecone integration and complete Qdrant migration
|
||||
- [ ] Test and validate vector search functionality with Qdrant
|
||||
- [ ] Test Cloud Function image processing pipeline
|
||||
- [ ] Validate VM setup for self-hosted Qdrant instance
|
||||
- [ ] Thumbnail generation
|
||||
- [ ] Secret management
|
||||
- [ ] Scale Vector DB to multiple nodes
|
||||
|
||||
### Medium Priority
|
||||
- [ ] Add comprehensive logging for vector search operations
|
||||
- [ ] Implement caching layer for frequently accessed embeddings
|
||||
- [ ] Implement caching for frequently accessed data
|
||||
- [ ] Add monitoring and alerting for vector database performance
|
||||
- [ ] Document vector search API endpoints
|
||||
- [ ] Set up Qdrant cluster with multiple nodes
|
||||
- [ ] Consider adding pagination to admin endpoints (users, teams, API keys) if datasets grow large
|
||||
|
||||
### Low Priority
|
||||
- [ ] Terraform dependencies
|
||||
- [ ] Move all auth logic to auth module
|
||||
- [ ] Move cloud function code to src folder and reuse code with embedding service
|
||||
- [ ] Thumbnail generation
|
||||
- [ ] Remove Pinecone integration
|
||||
|
||||
@ -318,8 +318,6 @@ if [ "$DEPLOY" = true ]; then
|
||||
echo " QDRANT_HOST=$QDRANT_IP"
|
||||
echo " QDRANT_PORT=6333"
|
||||
echo ""
|
||||
echo "To get detailed Qdrant information, run:"
|
||||
echo " ./deployment/terraform/scripts/get_qdrant_ip.sh"
|
||||
fi
|
||||
|
||||
echo "All operations completed."
|
||||
@ -1,64 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Deployment script that ensures APIs are enabled before deploying Cloud Functions
|
||||
set -e
|
||||
|
||||
echo "🚀 Starting Terraform deployment with API enablement..."
|
||||
|
||||
# Get project ID from terraform.tfvars or environment
|
||||
PROJECT_ID=${GOOGLE_CLOUD_PROJECT:-$(grep 'project_id' terraform.tfvars | cut -d'"' -f2)}
|
||||
|
||||
if [ -z "$PROJECT_ID" ]; then
|
||||
echo "❌ Error: PROJECT_ID not found. Set GOOGLE_CLOUD_PROJECT or ensure terraform.tfvars has project_id"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "📋 Project ID: $PROJECT_ID"
|
||||
|
||||
# Step 1: Enable APIs first
|
||||
echo "🔧 Step 1: Enabling required APIs..."
|
||||
terraform apply -target=google_project_service.services -auto-approve
|
||||
|
||||
# Step 2: Wait for APIs to propagate
|
||||
echo "⏳ Step 2: Waiting for APIs to propagate (30 seconds)..."
|
||||
sleep 30
|
||||
|
||||
# Step 3: Verify critical APIs are enabled
|
||||
echo "✅ Step 3: Verifying API enablement..."
|
||||
REQUIRED_APIS=(
|
||||
"cloudfunctions.googleapis.com"
|
||||
"eventarc.googleapis.com"
|
||||
"pubsub.googleapis.com"
|
||||
"cloudbuild.googleapis.com"
|
||||
)
|
||||
|
||||
for api in "${REQUIRED_APIS[@]}"; do
|
||||
echo " Checking $api..."
|
||||
if gcloud services list --enabled --project="$PROJECT_ID" --filter="name:$api" --format="value(name)" | grep -q "$api"; then
|
||||
echo " ✅ $api is enabled"
|
||||
else
|
||||
echo " ❌ $api is not enabled yet, waiting longer..."
|
||||
sleep 15
|
||||
fi
|
||||
done
|
||||
|
||||
# Step 4: Deploy infrastructure
|
||||
echo "🏗️ Step 4: Deploying infrastructure..."
|
||||
terraform apply -target=google_storage_bucket.app_bucket -target=google_firestore_database.database -target=google_compute_instance.vector_db_vm -auto-approve
|
||||
|
||||
# Step 5: Deploy Pub/Sub resources
|
||||
echo "📨 Step 5: Deploying Pub/Sub resources..."
|
||||
terraform apply -target=google_pubsub_topic.image_processing -target=google_pubsub_topic.image_processing_dlq -auto-approve
|
||||
|
||||
# Step 6: Deploy Cloud Function
|
||||
echo "⚡ Step 6: Deploying Cloud Function..."
|
||||
terraform apply -target=google_cloudfunctions2_function.image_processor -auto-approve
|
||||
|
||||
# Step 7: Deploy remaining resources
|
||||
echo "🎯 Step 7: Deploying remaining resources..."
|
||||
terraform apply -auto-approve
|
||||
|
||||
echo "🎉 Deployment completed successfully!"
|
||||
echo ""
|
||||
echo "📊 Deployment Summary:"
|
||||
terraform output deployment_summary 2>/dev/null || echo " Run 'terraform output' to see all outputs"
|
||||
@ -1,73 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to get Qdrant VM IP address from Terraform outputs
|
||||
# Usage: ./get_qdrant_ip.sh [--internal]
|
||||
|
||||
set -e
|
||||
|
||||
# Change to terraform directory
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Check if terraform state exists
|
||||
if [ ! -f "terraform.tfstate" ]; then
|
||||
echo "ERROR: No terraform.tfstate found. Please run 'terraform apply' first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
INTERNAL=false
|
||||
if [ "$1" = "--internal" ]; then
|
||||
INTERNAL=true
|
||||
fi
|
||||
|
||||
echo "Retrieving Qdrant VM IP address..."
|
||||
echo "=================================="
|
||||
|
||||
# Get VM information
|
||||
VM_NAME=$(terraform output -raw vector_db_vm_name 2>/dev/null || echo "N/A")
|
||||
echo "VM Name: $VM_NAME"
|
||||
|
||||
if [ "$INTERNAL" = true ]; then
|
||||
# Get internal IP
|
||||
QDRANT_IP=$(terraform output -raw vector_db_vm_internal_ip 2>/dev/null || echo "N/A")
|
||||
echo "Internal IP: $QDRANT_IP"
|
||||
echo "Qdrant HTTP Endpoint (Internal): http://$QDRANT_IP:6333"
|
||||
echo "Qdrant gRPC Endpoint (Internal): http://$QDRANT_IP:6334"
|
||||
else
|
||||
# Get external IP
|
||||
QDRANT_IP=$(terraform output -raw vector_db_vm_external_ip 2>/dev/null || echo "N/A")
|
||||
echo "External IP: $QDRANT_IP"
|
||||
echo "Qdrant HTTP Endpoint: http://$QDRANT_IP:6333"
|
||||
echo "Qdrant gRPC Endpoint: http://$QDRANT_IP:6334"
|
||||
fi
|
||||
|
||||
# Check if static IP is enabled
|
||||
STATIC_IP_ENABLED=$(terraform output -raw vector_db_static_ip 2>/dev/null || echo "null")
|
||||
if [ "$STATIC_IP_ENABLED" != "null" ] && [ "$STATIC_IP_ENABLED" != "" ]; then
|
||||
echo "Static IP: $STATIC_IP_ENABLED (enabled)"
|
||||
else
|
||||
echo "Static IP: Not enabled (using ephemeral IP)"
|
||||
fi
|
||||
|
||||
# Get Cloud Run service URL
|
||||
CLOUD_RUN_URL=$(terraform output -raw cloud_run_url 2>/dev/null || echo "N/A")
|
||||
echo "Cloud Run URL: $CLOUD_RUN_URL"
|
||||
|
||||
echo ""
|
||||
echo "Environment Variables for Cloud Run:"
|
||||
echo "QDRANT_HOST=$QDRANT_IP"
|
||||
echo "QDRANT_PORT=6333"
|
||||
|
||||
# Test connectivity (optional)
|
||||
echo ""
|
||||
echo "Testing Qdrant connectivity..."
|
||||
if command -v curl &> /dev/null; then
|
||||
if curl -s --connect-timeout 5 "http://$QDRANT_IP:6333/health" > /dev/null; then
|
||||
echo "✓ Qdrant is accessible at http://$QDRANT_IP:6333"
|
||||
else
|
||||
echo "✗ Qdrant is not accessible at http://$QDRANT_IP:6333"
|
||||
echo " This might be normal if the VM is still starting up or firewall rules need adjustment."
|
||||
fi
|
||||
else
|
||||
echo "curl not available - skipping connectivity test"
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user