diff --git a/README.md b/README.md index 135f11e..54ddaef 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,7 @@ vector_db = VectorDatabaseService( # Add image vector point_id = vector_db.add_image_vector( image_id="img_123", - vector=[0.1, 0.2, ...], # 512-dimensional vector + vector=[0.1, 0.2, ...], # 1408-dimensional vector metadata={"filename": "image.jpg", "size": 1024} ) diff --git a/deployment/cleanup-images.sh b/deployment/cleanup-images.sh deleted file mode 100644 index 5475721..0000000 --- a/deployment/cleanup-images.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -e - -# Container Image Cleanup Script -# This script cleans up container images from Google Container Registry -# Images are not managed by Terraform, so this provides a manual cleanup option - -PROJECT_ID=$(gcloud config get-value project) -IMAGE_NAME="sereact-api" - -if [ -z "$PROJECT_ID" ]; then - echo "ERROR: No Google Cloud project is set. Run 'gcloud config set project YOUR_PROJECT_ID'" - exit 1 -fi - -echo "Cleaning up container images for project: $PROJECT_ID" -echo "Image repository: gcr.io/$PROJECT_ID/$IMAGE_NAME" -echo "" - -# Check if repository exists -if ! gcloud container images list-tags "gcr.io/$PROJECT_ID/$IMAGE_NAME" > /dev/null 2>&1; then - echo "No container images found for $IMAGE_NAME" - exit 0 -fi - -echo "Found container images. Listing current images:" -gcloud container images list-tags "gcr.io/$PROJECT_ID/$IMAGE_NAME" -echo "" - -read -p "Do you want to delete ALL images for $IMAGE_NAME? (yes/no): " confirm - -if [ "$confirm" != "yes" ]; then - echo "Cleanup cancelled." - exit 0 -fi - -echo "Deleting container images..." - -# Get all image digests and delete them -DIGESTS=$(gcloud container images list-tags "gcr.io/$PROJECT_ID/$IMAGE_NAME" --format="get(digest)" --filter="tags:*" 2>/dev/null || true) -UNTAGGED_DIGESTS=$(gcloud container images list-tags "gcr.io/$PROJECT_ID/$IMAGE_NAME" --format="get(digest)" --filter="-tags:*" 2>/dev/null || true) - -# Delete tagged images -if [ ! -z "$DIGESTS" ]; then - echo "Deleting tagged images..." - for digest in $DIGESTS; do - gcloud container images delete "gcr.io/$PROJECT_ID/$IMAGE_NAME@$digest" --force-delete-tags --quiet || echo "Failed to delete $digest" - done -fi - -# Delete untagged images -if [ ! -z "$UNTAGGED_DIGESTS" ]; then - echo "Deleting untagged images..." - for digest in $UNTAGGED_DIGESTS; do - gcloud container images delete "gcr.io/$PROJECT_ID/$IMAGE_NAME@$digest" --quiet || echo "Failed to delete $digest" - done -fi - -echo "Container image cleanup completed." -echo "" -echo "Note: The repository gcr.io/$PROJECT_ID/$IMAGE_NAME may still exist but should be empty." -echo "You can verify with: gcloud container images list-tags gcr.io/$PROJECT_ID/$IMAGE_NAME" \ No newline at end of file diff --git a/deployment/cloud-function/test_vertex_ai_embeddings.py b/deployment/cloud-function/test_vertex_ai_embeddings.py deleted file mode 100644 index eef647e..0000000 --- a/deployment/cloud-function/test_vertex_ai_embeddings.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -""" -Simple test script to verify Vertex AI multimodal embeddings work correctly. -Run this script to test the embedding generation before deploying. -""" - -import os -import sys -import logging -from PIL import Image -import io -import numpy as np - -# Add the current directory to the path so we can import main -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def create_test_image(): - """Create a simple test image""" - # Create a simple 100x100 RGB image with a red square - img = Image.new('RGB', (100, 100), color='white') - pixels = img.load() - - # Draw a red square in the center - for i in range(25, 75): - for j in range(25, 75): - pixels[i, j] = (255, 0, 0) # Red - - # Convert to bytes - img_bytes = io.BytesIO() - img.save(img_bytes, format='PNG') - return img_bytes.getvalue() - -def test_vertex_ai_embeddings(): - """Test Vertex AI embedding generation""" - try: - # Set required environment variables for testing - os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ.get('GOOGLE_CLOUD_PROJECT', 'your-project-id') - os.environ['VERTEX_AI_LOCATION'] = os.environ.get('VERTEX_AI_LOCATION', 'us-central1') - - # Import the function we want to test - from main import generate_image_embeddings - - # Create test image - logger.info("Creating test image...") - test_image_data = create_test_image() - logger.info(f"Created test image with {len(test_image_data)} bytes") - - # Generate embeddings - logger.info("Generating embeddings using Vertex AI...") - embeddings = generate_image_embeddings(test_image_data) - - if embeddings is None: - logger.error("Failed to generate embeddings!") - return False - - # Verify embeddings - logger.info(f"Generated embeddings with shape: {embeddings.shape}") - logger.info(f"Embeddings dtype: {embeddings.dtype}") - logger.info(f"Embeddings range: [{embeddings.min():.4f}, {embeddings.max():.4f}]") - logger.info(f"Embeddings norm: {np.linalg.norm(embeddings):.4f}") - - # Basic validation - assert isinstance(embeddings, np.ndarray), "Embeddings should be numpy array" - assert embeddings.dtype == np.float32, "Embeddings should be float32" - assert len(embeddings.shape) == 1, "Embeddings should be 1D array" - assert embeddings.shape[0] == 1408, f"Expected 1408 dimensions, got {embeddings.shape[0]}" - - # Check if normalized (should be close to 1.0) - norm = np.linalg.norm(embeddings) - assert 0.9 <= norm <= 1.1, f"Embeddings should be normalized, norm is {norm}" - - logger.info("✅ All tests passed! Vertex AI embeddings are working correctly.") - return True - - except ImportError as e: - logger.error(f"Import error: {e}") - logger.error("Make sure you have installed the required dependencies:") - logger.error("pip install google-cloud-aiplatform") - return False - except Exception as e: - logger.error(f"Test failed with error: {e}") - return False - -if __name__ == "__main__": - logger.info("Testing Vertex AI multimodal embeddings...") - - # Check if required environment variables are set - project_id = os.environ.get('GOOGLE_CLOUD_PROJECT') - if not project_id: - logger.error("Please set GOOGLE_CLOUD_PROJECT environment variable") - logger.error("Example: export GOOGLE_CLOUD_PROJECT=your-project-id") - sys.exit(1) - - logger.info(f"Using project: {project_id}") - - success = test_vertex_ai_embeddings() - - if success: - logger.info("🎉 Test completed successfully!") - sys.exit(0) - else: - logger.error("❌ Test failed!") - sys.exit(1) \ No newline at end of file diff --git a/deployment/fix-cloud-run-deployment.sh b/deployment/fix-cloud-run-deployment.sh deleted file mode 100644 index 2d159c6..0000000 --- a/deployment/fix-cloud-run-deployment.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash -set -e - -# Configuration -PROJECT_ID="gen-lang-client-0424120530" -REGION="us-central1" -SERVICE_NAME="sereact" - -echo "=== Cloud Run Deployment Fix Script ===" -echo "Project: $PROJECT_ID" -echo "Region: $REGION" -echo "Service: $SERVICE_NAME" -echo "" - -# Function to check if gcloud is authenticated -check_auth() { - if ! gcloud auth list --filter=status:ACTIVE --format="value(account)" | grep -q .; then - echo "ERROR: No active gcloud authentication found." - echo "Please run: gcloud auth login" - exit 1 - fi -} - -# Function to clean up problematic revisions -cleanup_revisions() { - echo "Step 1: Cleaning up problematic revisions..." - - # Get all revisions for the service - REVISIONS=$(gcloud run revisions list --service=$SERVICE_NAME --region=$REGION --project=$PROJECT_ID --format="value(metadata.name)" 2>/dev/null || echo "") - - if [ -z "$REVISIONS" ]; then - echo "No revisions found or service doesn't exist." - return 0 - fi - - echo "Found revisions:" - echo "$REVISIONS" - echo "" - - # Delete old revisions (keep the latest one for now) - REVISION_COUNT=$(echo "$REVISIONS" | wc -l) - if [ $REVISION_COUNT -gt 1 ]; then - echo "Deleting old revisions to prevent conflicts..." - echo "$REVISIONS" | head -n -1 | while read revision; do - if [ ! -z "$revision" ]; then - echo "Deleting revision: $revision" - gcloud run revisions delete "$revision" --region=$REGION --project=$PROJECT_ID --quiet || echo "Failed to delete $revision (may be in use)" - fi - done - fi -} - -# Function to force a new deployment -force_redeploy() { - echo "Step 2: Forcing a new deployment with Terraform..." - - cd "$(dirname "$0")/terraform" - - # Taint the Cloud Run service to force recreation - echo "Tainting Cloud Run service to force recreation..." - terraform taint google_cloud_run_service.sereact || echo "Service not in state or already tainted" - - # Apply the configuration - echo "Applying Terraform configuration..." - terraform apply -auto-approve - - cd - > /dev/null -} - -# Function to verify deployment -verify_deployment() { - echo "Step 3: Verifying deployment..." - - # Wait a moment for the service to be ready - sleep 10 - - # Check service status - SERVICE_URL=$(gcloud run services describe $SERVICE_NAME --region=$REGION --project=$PROJECT_ID --format="value(status.url)" 2>/dev/null || echo "") - - if [ ! -z "$SERVICE_URL" ]; then - echo "✅ Service deployed successfully!" - echo "Service URL: $SERVICE_URL" - - # Test the service - echo "Testing service health..." - if curl -s -o /dev/null -w "%{http_code}" "$SERVICE_URL/health" | grep -q "200"; then - echo "✅ Service is responding correctly!" - else - echo "⚠️ Service deployed but health check failed. Check logs:" - echo "gcloud logging read \"resource.type=cloud_run_revision AND resource.labels.service_name=$SERVICE_NAME\" --limit=10 --project=$PROJECT_ID" - fi - else - echo "❌ Service deployment failed." - exit 1 - fi -} - -# Main execution -main() { - echo "Starting Cloud Run deployment fix..." - echo "" - - check_auth - cleanup_revisions - force_redeploy - verify_deployment - - echo "" - echo "=== Deployment Fix Complete ===" - echo "If you continue to have issues, try:" - echo "1. Building a new image: ./deployment/deploy.sh --deploy --build" - echo "2. Checking logs: gcloud logging read \"resource.type=cloud_run_revision AND resource.labels.service_name=$SERVICE_NAME\" --limit=10 --project=$PROJECT_ID" -} - -# Help function -show_help() { - echo "Usage: $0 [--help]" - echo "" - echo "This script fixes Cloud Run deployment issues by:" - echo "1. Cleaning up problematic revisions" - echo "2. Forcing a new deployment with Terraform" - echo "3. Verifying the deployment" - echo "" - echo "Options:" - echo " --help Show this help message" -} - -# Parse arguments -if [ "$1" = "--help" ]; then - show_help - exit 0 -fi - -# Run main function -main \ No newline at end of file diff --git a/deployment/terraform/scripts/install_qdrant.sh b/deployment/terraform/scripts/install_qdrant.sh index dc94a02..b8fbc22 100644 --- a/deployment/terraform/scripts/install_qdrant.sh +++ b/deployment/terraform/scripts/install_qdrant.sh @@ -172,7 +172,7 @@ curl -X PUT "http://localhost:6333/collections/image_vectors" \ -H "Content-Type: application/json" \ -d '{ "vectors": { - "size": 512, + "size": 1408, "distance": "Cosine" }, "optimizers_config": { diff --git a/deployment/terraform/terraform.tfstate b/deployment/terraform/terraform.tfstate index fb755f2..e4f22d7 100644 --- a/deployment/terraform/terraform.tfstate +++ b/deployment/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 425, + "serial": 438, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { "cloud_function_name": { @@ -13,11 +13,11 @@ "type": "string" }, "cloud_run_qdrant_host": { - "value": "34.71.6.1", + "value": "35.193.174.125", "type": "string" }, "cloud_run_qdrant_host_internal": { - "value": "10.128.0.6", + "value": "10.128.0.7", "type": "string" }, "cloud_run_url": { @@ -32,8 +32,8 @@ "value": { "cloud_run_url": "https://sereact-p64zpdtkta-uc.a.run.app", "firestore_database": "sereact-imagedb", - "qdrant_endpoint": "http://34.71.6.1:6333", - "qdrant_host_ip": "34.71.6.1", + "qdrant_endpoint": "http://35.193.174.125:6333", + "qdrant_host_ip": "35.193.174.125", "static_ip_enabled": false, "storage_bucket": "sereact-images" }, @@ -62,11 +62,11 @@ "type": "string" }, "qdrant_grpc_endpoint": { - "value": "http://34.71.6.1:6334", + "value": "http://35.193.174.125:6334", "type": "string" }, "qdrant_http_endpoint": { - "value": "http://34.71.6.1:6333", + "value": "http://35.193.174.125:6333", "type": "string" }, "storage_bucket_name": { @@ -74,11 +74,11 @@ "type": "string" }, "vector_db_vm_external_ip": { - "value": "34.71.6.1", + "value": "35.193.174.125", "type": "string" }, "vector_db_vm_internal_ip": { - "value": "10.128.0.6", + "value": "10.128.0.7", "type": "string" }, "vector_db_vm_name": { @@ -98,16 +98,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "bfc4b3b9de401cd15676a09a067a8e4095b0bf4e", - "output_base64sha256": "cXx9sC1kIbTDG7BlKAtf3FUasHLZ/wZPzVoyBvt8p9Q=", - "output_base64sha512": "dcntRZ4Hz4dfBBj7YVsTzx+SEAqCXZCD8TAAh8cr5xa3uT2Lsmtf8zpxpyQEeMlsCNaF8dUohGQ7BD9LJYigPw==", + "id": "88ee03db0f4c7023c0c620449e167ad27074fdd0", + "output_base64sha256": "0p558sP6ikbyrfmva7zGOYklnR/4VRPD1zcl8HZcv8A=", + "output_base64sha512": "mal2zoxqjg5lZYruPmffQdDqY9FJONPc5Wnu41NP07LOj/tC+sJAAeQ7tmU0mq8h6SfQE6wwxFeYJuEO1y2xLg==", "output_file_mode": null, - "output_md5": "46efa1aee5386e2f244b597289c7c4ba", + "output_md5": "58a2b7fe53bb2c8c921405cc965d635c", "output_path": "./function-source.zip", - "output_sha": "bfc4b3b9de401cd15676a09a067a8e4095b0bf4e", - "output_sha256": "717c7db02d6421b4c31bb065280b5fdc551ab072d9ff064fcd5a3206fb7ca7d4", - "output_sha512": "75c9ed459e07cf875f0418fb615b13cf1f92100a825d9083f1300087c72be716b7b93d8bb26b5ff33a71a7240478c96c08d685f1d52884643b043f4b2588a03f", - "output_size": 6734, + "output_sha": "88ee03db0f4c7023c0c620449e167ad27074fdd0", + "output_sha256": "d29e79f2c3fa8a46f2adf9af6bbcc63989259d1ff85513c3d73725f0765cbfc0", + "output_sha512": "99a976ce8c6a8e0e65658aee3e67df41d0ea63d14938d3dce569eee3534fd3b2ce8ffb42fac24001e43bb665349aaf21e927d013ac30c4579826e10ed72db12e", + "output_size": 69765973, "source": [], "source_content": null, "source_content_filename": null, @@ -172,7 +172,7 @@ "effective_annotations": { "run.googleapis.com/ingress": "all", "run.googleapis.com/ingress-status": "all", - "run.googleapis.com/operation-id": "7a9a82f0-4bc9-4fe6-af40-ddee543c0536", + "run.googleapis.com/operation-id": "0f195b05-99ac-4d28-b5fe-2d3dea289124", "run.googleapis.com/urls": "[\"https://sereact-761163285547.us-central1.run.app\",\"https://sereact-p64zpdtkta-uc.a.run.app\"]", "serving.knative.dev/creator": "johnpccd3@gmail.com", "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" @@ -182,14 +182,14 @@ "goog-terraform-provisioned": "true" }, "generation": 1, - "labels": null, + "labels": {}, "namespace": "gen-lang-client-0424120530", - "resource_version": "AAY18uNZfE8", + "resource_version": "AAY189oNgAQ", "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/sereact", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "dfae23a0-4b71-40aa-baa8-64f5b842501a" + "uid": "20e61eb3-6217-40e8-8ae5-45111d31bbda" } ], "name": "sereact", @@ -216,14 +216,14 @@ "type": "RoutesReady" } ], - "latest_created_revision_name": "sereact-00001-58h", - "latest_ready_revision_name": "sereact-00001-58h", + "latest_created_revision_name": "sereact-00001-2lz", + "latest_ready_revision_name": "sereact-00001-2lz", "observed_generation": 1, "traffic": [ { "latest_revision": true, "percent": 100, - "revision_name": "sereact-00001-58h", + "revision_name": "sereact-00001-2lz", "tag": "", "url": "" } @@ -256,8 +256,8 @@ "container_concurrency": 80, "containers": [ { - "args": null, - "command": null, + "args": [], + "command": [], "env": [ { "name": "API_KEY_SECRET", @@ -291,7 +291,7 @@ }, { "name": "QDRANT_HOST", - "value": "34.71.6.1", + "value": "35.193.174.125", "value_from": [] }, { @@ -337,7 +337,7 @@ "cpu": "1", "memory": "1Gi" }, - "requests": null + "requests": {} } ], "startup_probe": [ @@ -359,7 +359,7 @@ "working_dir": "" } ], - "node_selector": null, + "node_selector": {}, "service_account_name": "761163285547-compute@developer.gserviceaccount.com", "serving_state": "", "timeout_seconds": 300, @@ -440,7 +440,7 @@ "schema_version": 0, "attributes": { "condition": [], - "etag": "BwY18uO4z7A=", + "etag": "BwY189qg+AA=", "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact/roles/run.invoker/allUsers", "location": "us-central1", "member": "allUsers", @@ -474,7 +474,7 @@ "automatic_update_policy": [ {} ], - "build": "projects/761163285547/locations/us-central1/builds/47fecac1-7233-42af-ad0e-2a5a9c66e282", + "build": "projects/761163285547/locations/us-central1/builds/aab08c74-df86-4cd7-9176-4ff267cab3e6", "docker_repository": "projects/gen-lang-client-0424120530/locations/us-central1/repositories/gcf-artifacts", "entry_point": "process_image_embedding", "environment_variables": {}, @@ -487,8 +487,8 @@ "storage_source": [ { "bucket": "gen-lang-client-0424120530-cloud-function-source", - "generation": 1748123369545880, - "object": "function-source-46efa1aee5386e2f244b597289c7c4ba.zip" + "generation": 1748171376287077, + "object": "function-source-58a2b7fe53bb2c8c921405cc965d635c.zip" } ] } @@ -508,13 +508,13 @@ "pubsub_topic": "projects/gen-lang-client-0424120530/topics/image-processing-topic", "retry_policy": "RETRY_POLICY_RETRY", "service_account_email": "761163285547-compute@developer.gserviceaccount.com", - "trigger": "projects/gen-lang-client-0424120530/locations/us-central1/triggers/process-image-embedding-422683", + "trigger": "projects/gen-lang-client-0424120530/locations/us-central1/triggers/process-image-embedding-013009", "trigger_region": "us-central1" } ], "id": "projects/gen-lang-client-0424120530/locations/us-central1/functions/process-image-embedding", "kms_key_name": "", - "labels": {}, + "labels": null, "location": "us-central1", "name": "process-image-embedding", "project": "gen-lang-client-0424120530", @@ -534,7 +534,7 @@ "PROJECT_ID": "gen-lang-client-0424120530", "QDRANT_API_KEY": "", "QDRANT_COLLECTION": "image_vectors", - "QDRANT_HOST": "34.71.6.1", + "QDRANT_HOST": "35.193.174.125", "QDRANT_HTTPS": "false", "QDRANT_PORT": "6333", "VERTEX_AI_LOCATION": "us-central1" @@ -559,7 +559,7 @@ "goog-terraform-provisioned": "true" }, "timeouts": null, - "update_time": "2025-05-25T09:26:49.473142736Z", + "update_time": "2025-05-25T11:13:04.212724797Z", "url": "https://us-central1-gen-lang-client-0424120530.cloudfunctions.net/process-image-embedding" }, "sensitive_attributes": [ @@ -711,7 +711,7 @@ "can_ip_forward": false, "confidential_instance_config": [], "cpu_platform": "Intel Broadwell", - "creation_timestamp": "2025-05-24T14:09:30.477-07:00", + "creation_timestamp": "2025-05-25T03:58:31.230-07:00", "current_status": "RUNNING", "deletion_protection": false, "description": "", @@ -724,21 +724,21 @@ "hostname": "", "id": "projects/gen-lang-client-0424120530/zones/us-central1-a/instances/sereact-vector-db", "instance_encryption_key": [], - "instance_id": "1246596410843827045", + "instance_id": "3665939918038714681", "key_revocation_action_type": "", "label_fingerprint": "vezUS-42LLM=", "labels": {}, "machine_type": "e2-standard-2", "metadata": {}, - "metadata_fingerprint": "cE8FbgySELs=", - "metadata_startup_script": "#!/bin/bash\n\n# Qdrant Vector Database Installation Script\n# This script installs and configures Qdrant on Ubuntu 22.04\n\nset -e\n\n# Update system packages\napt-get update\napt-get upgrade -y\n\n# Install required packages\napt-get install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates\n\n# Install Docker\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg\necho \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list \u003e /dev/null\napt-get update\napt-get install -y docker-ce docker-ce-cli containerd.io\n\n# Start and enable Docker\nsystemctl start docker\nsystemctl enable docker\n\n# Create qdrant user and directories\nuseradd -r -s /bin/false qdrant || true\nmkdir -p /opt/qdrant/storage\nmkdir -p /opt/qdrant/config\nchown -R qdrant:qdrant /opt/qdrant\n\n# Create Qdrant configuration file\ncat \u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_cors: true\n\nstorage:\n storage_path: /qdrant/storage\n snapshots_path: /qdrant/snapshots\n on_disk_payload: true\n\ncluster:\n enabled: false\n\ntelemetry:\n disabled: true\n\nlog_level: INFO\nEOF\n\n# Create API key configuration if provided\nif [ -n \"\" ] \u0026\u0026 [ \"\" != \"\" ]; then\ncat \u003e\u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\n\nservice:\n api_key: \"\"\nEOF\nfi\n\n# Create systemd service for Qdrant\ncat \u003e /etc/systemd/system/qdrant.service \u003c\u003c EOF\n[Unit]\nDescription=Qdrant Vector Database\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nType=simple\nUser=root\nExecStartPre=-/usr/bin/docker stop qdrant\nExecStartPre=-/usr/bin/docker rm qdrant\nExecStart=/usr/bin/docker run --name qdrant \\\n -p 6333:6333 \\\n -p 6334:6334 \\\n -v /opt/qdrant/storage:/qdrant/storage:z \\\n -v /opt/qdrant/config/config.yaml:/qdrant/config/production.yaml:z \\\n qdrant/qdrant:latest\nExecStop=/usr/bin/docker stop qdrant\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n# Pull Qdrant Docker image\ndocker pull qdrant/qdrant:latest\n\n# Enable and start Qdrant service\nsystemctl daemon-reload\nsystemctl enable qdrant\nsystemctl start qdrant\n\n# Install monitoring tools\napt-get install -y htop iotop nethogs\n\n# Create a simple health check script\ncat \u003e /opt/qdrant/health_check.sh \u003c\u003c 'EOF'\n#!/bin/bash\nresponse=$(curl -s -o /dev/null -w \"%{http_code}\" http://localhost:6333/health)\nif [ \"$response\" = \"200\" ]; then\n echo \"Qdrant is healthy\"\n exit 0\nelse\n echo \"Qdrant is not responding properly (HTTP $response)\"\n exit 1\nfi\nEOF\n\nchmod +x /opt/qdrant/health_check.sh\n\n# Set up log rotation for Docker logs\ncat \u003e /etc/logrotate.d/docker \u003c\u003c EOF\n/var/lib/docker/containers/*/*.log {\n rotate 7\n daily\n compress\n size=1M\n missingok\n delaycompress\n copytruncate\n}\nEOF\n\n# Configure firewall (ufw)\nufw --force enable\nufw allow ssh\nufw allow 6333/tcp # Qdrant HTTP API\nufw allow 6334/tcp # Qdrant gRPC API\n\n# Create a simple backup script\ncat \u003e /opt/qdrant/backup.sh \u003c\u003c 'EOF'\n#!/bin/bash\nBACKUP_DIR=\"/opt/qdrant/backups\"\nDATE=$(date +%Y%m%d_%H%M%S)\nmkdir -p $BACKUP_DIR\n\n# Create snapshot via API\ncurl -X POST \"http://localhost:6333/snapshots\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"snapshot_name\": \"backup_'$DATE'\"}'\n\n# Copy storage directory\ntar -czf $BACKUP_DIR/qdrant_storage_$DATE.tar.gz -C /opt/qdrant storage/\n\n# Keep only last 7 backups\nfind $BACKUP_DIR -name \"*.tar.gz\" -mtime +7 -delete\n\necho \"Backup completed: $DATE\"\nEOF\n\nchmod +x /opt/qdrant/backup.sh\n\n# Set up daily backup cron job\necho \"0 2 * * * root /opt/qdrant/backup.sh \u003e\u003e /var/log/qdrant_backup.log 2\u003e\u00261\" \u003e\u003e /etc/crontab\n\n# Wait for Qdrant to be ready\necho \"Waiting for Qdrant to start...\"\nfor i in {1..30}; do\n if curl -s http://localhost:6333/health \u003e /dev/null; then\n echo \"Qdrant is ready!\"\n break\n fi\n echo \"Waiting... ($i/30)\"\n sleep 10\ndone\n\n# Create a default collection for image vectors\ncurl -X PUT \"http://localhost:6333/collections/image_vectors\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"vectors\": {\n \"size\": 512,\n \"distance\": \"Cosine\"\n },\n \"optimizers_config\": {\n \"default_segment_number\": 2\n },\n \"replication_factor\": 1\n }'\n\necho \"Qdrant installation and configuration completed!\"\necho \"Qdrant is accessible at:\"\necho \" HTTP API: http://$(curl -s ifconfig.me):6333\"\necho \" gRPC API: http://$(curl -s ifconfig.me):6334\"\necho \"Health check: /opt/qdrant/health_check.sh\"\necho \"Backup script: /opt/qdrant/backup.sh\" ", + "metadata_fingerprint": "nyOcq7EkvZA=", + "metadata_startup_script": "#!/bin/bash\n\n# Qdrant Vector Database Installation Script\n# This script installs and configures Qdrant on Ubuntu 22.04\n\nset -e\n\n# Update system packages\napt-get update\napt-get upgrade -y\n\n# Install required packages\napt-get install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates\n\n# Install Docker\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg\necho \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list \u003e /dev/null\napt-get update\napt-get install -y docker-ce docker-ce-cli containerd.io\n\n# Start and enable Docker\nsystemctl start docker\nsystemctl enable docker\n\n# Create qdrant user and directories\nuseradd -r -s /bin/false qdrant || true\nmkdir -p /opt/qdrant/storage\nmkdir -p /opt/qdrant/config\nchown -R qdrant:qdrant /opt/qdrant\n\n# Create Qdrant configuration file\ncat \u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_cors: true\n\nstorage:\n storage_path: /qdrant/storage\n snapshots_path: /qdrant/snapshots\n on_disk_payload: true\n\ncluster:\n enabled: false\n\ntelemetry:\n disabled: true\n\nlog_level: INFO\nEOF\n\n# Create API key configuration if provided\nif [ -n \"\" ] \u0026\u0026 [ \"\" != \"\" ]; then\ncat \u003e\u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\n\nservice:\n api_key: \"\"\nEOF\nfi\n\n# Create systemd service for Qdrant\ncat \u003e /etc/systemd/system/qdrant.service \u003c\u003c EOF\n[Unit]\nDescription=Qdrant Vector Database\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nType=simple\nUser=root\nExecStartPre=-/usr/bin/docker stop qdrant\nExecStartPre=-/usr/bin/docker rm qdrant\nExecStart=/usr/bin/docker run --name qdrant \\\n -p 6333:6333 \\\n -p 6334:6334 \\\n -v /opt/qdrant/storage:/qdrant/storage:z \\\n -v /opt/qdrant/config/config.yaml:/qdrant/config/production.yaml:z \\\n qdrant/qdrant:latest\nExecStop=/usr/bin/docker stop qdrant\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n# Pull Qdrant Docker image\ndocker pull qdrant/qdrant:latest\n\n# Enable and start Qdrant service\nsystemctl daemon-reload\nsystemctl enable qdrant\nsystemctl start qdrant\n\n# Install monitoring tools\napt-get install -y htop iotop nethogs\n\n# Create a simple health check script\ncat \u003e /opt/qdrant/health_check.sh \u003c\u003c 'EOF'\n#!/bin/bash\nresponse=$(curl -s -o /dev/null -w \"%{http_code}\" http://localhost:6333/health)\nif [ \"$response\" = \"200\" ]; then\n echo \"Qdrant is healthy\"\n exit 0\nelse\n echo \"Qdrant is not responding properly (HTTP $response)\"\n exit 1\nfi\nEOF\n\nchmod +x /opt/qdrant/health_check.sh\n\n# Set up log rotation for Docker logs\ncat \u003e /etc/logrotate.d/docker \u003c\u003c EOF\n/var/lib/docker/containers/*/*.log {\n rotate 7\n daily\n compress\n size=1M\n missingok\n delaycompress\n copytruncate\n}\nEOF\n\n# Configure firewall (ufw)\nufw --force enable\nufw allow ssh\nufw allow 6333/tcp # Qdrant HTTP API\nufw allow 6334/tcp # Qdrant gRPC API\n\n# Create a simple backup script\ncat \u003e /opt/qdrant/backup.sh \u003c\u003c 'EOF'\n#!/bin/bash\nBACKUP_DIR=\"/opt/qdrant/backups\"\nDATE=$(date +%Y%m%d_%H%M%S)\nmkdir -p $BACKUP_DIR\n\n# Create snapshot via API\ncurl -X POST \"http://localhost:6333/snapshots\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"snapshot_name\": \"backup_'$DATE'\"}'\n\n# Copy storage directory\ntar -czf $BACKUP_DIR/qdrant_storage_$DATE.tar.gz -C /opt/qdrant storage/\n\n# Keep only last 7 backups\nfind $BACKUP_DIR -name \"*.tar.gz\" -mtime +7 -delete\n\necho \"Backup completed: $DATE\"\nEOF\n\nchmod +x /opt/qdrant/backup.sh\n\n# Set up daily backup cron job\necho \"0 2 * * * root /opt/qdrant/backup.sh \u003e\u003e /var/log/qdrant_backup.log 2\u003e\u00261\" \u003e\u003e /etc/crontab\n\n# Wait for Qdrant to be ready\necho \"Waiting for Qdrant to start...\"\nfor i in {1..30}; do\n if curl -s http://localhost:6333/health \u003e /dev/null; then\n echo \"Qdrant is ready!\"\n break\n fi\n echo \"Waiting... ($i/30)\"\n sleep 10\ndone\n\n# Create a default collection for image vectors\ncurl -X PUT \"http://localhost:6333/collections/image_vectors\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"vectors\": {\n \"size\": 1408,\n \"distance\": \"Cosine\"\n },\n \"optimizers_config\": {\n \"default_segment_number\": 2\n },\n \"replication_factor\": 1\n }'\n\necho \"Qdrant installation and configuration completed!\"\necho \"Qdrant is accessible at:\"\necho \" HTTP API: http://$(curl -s ifconfig.me):6333\"\necho \" gRPC API: http://$(curl -s ifconfig.me):6334\"\necho \"Health check: /opt/qdrant/health_check.sh\"\necho \"Backup script: /opt/qdrant/backup.sh\" ", "min_cpu_platform": "", "name": "sereact-vector-db", "network_interface": [ { "access_config": [ { - "nat_ip": "34.71.6.1", + "nat_ip": "35.193.174.125", "network_tier": "PREMIUM", "public_ptr_domain_name": "" } @@ -751,7 +751,7 @@ "name": "nic0", "network": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/global/networks/default", "network_attachment": "", - "network_ip": "10.128.0.6", + "network_ip": "10.128.0.7", "nic_type": "", "queue_count": 0, "stack_type": "IPV4_ONLY", @@ -812,18 +812,7 @@ [ { "type": "get_attr", - "value": "boot_disk" - }, - { - "type": "index", - "value": { - "value": 0, - "type": "number" - } - }, - { - "type": "get_attr", - "value": "disk_encryption_key_raw" + "value": "metadata_startup_script" } ], [ @@ -846,7 +835,18 @@ [ { "type": "get_attr", - "value": "metadata_startup_script" + "value": "boot_disk" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "disk_encryption_key_raw" } ] ], @@ -875,8 +875,8 @@ "database_edition": "STANDARD", "delete_protection_state": "DELETE_PROTECTION_DISABLED", "deletion_policy": "ABANDON", - "earliest_version_time": "2025-05-25T08:59:14.308781Z", - "etag": "IMCn9pGuvo0DMKrW4vCEvY0D", + "earliest_version_time": "2025-05-25T10:09:32.175339Z", + "etag": "IMC5lO29vo0DMKrW4vCEvY0D", "id": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", "key_prefix": "", "location_id": "us-central1", @@ -1514,21 +1514,21 @@ "content_encoding": "", "content_language": "", "content_type": "application/zip", - "crc32c": "kTROsA==", + "crc32c": "eCjQFg==", "customer_encryption": [], - "detect_md5hash": "Ru+hruU4bi8kS1lyicfEug==", + "detect_md5hash": "WKK3/lO7LIySFAXMll1jXA==", "event_based_hold": false, - "generation": 1748126317190781, - "id": "gen-lang-client-0424120530-cloud-function-source-function-source-46efa1aee5386e2f244b597289c7c4ba.zip", + "generation": 1748170673167525, + "id": "gen-lang-client-0424120530-cloud-function-source-function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", "kms_key_name": "", - "md5hash": "Ru+hruU4bi8kS1lyicfEug==", - "md5hexhash": "46efa1aee5386e2f244b597289c7c4ba", - "media_link": "https://storage.googleapis.com/download/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-46efa1aee5386e2f244b597289c7c4ba.zip?generation=1748126317190781\u0026alt=media", + "md5hash": "WKK3/lO7LIySFAXMll1jXA==", + "md5hexhash": "58a2b7fe53bb2c8c921405cc965d635c", + "media_link": "https://storage.googleapis.com/download/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-58a2b7fe53bb2c8c921405cc965d635c.zip?generation=1748170673167525\u0026alt=media", "metadata": {}, - "name": "function-source-46efa1aee5386e2f244b597289c7c4ba.zip", - "output_name": "function-source-46efa1aee5386e2f244b597289c7c4ba.zip", + "name": "function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", + "output_name": "function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", "retention": [], - "self_link": "https://www.googleapis.com/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-46efa1aee5386e2f244b597289c7c4ba.zip", + "self_link": "https://www.googleapis.com/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", "source": "./function-source.zip", "storage_class": "STANDARD", "temporary_hold": false, diff --git a/deployment/terraform/terraform.tfstate.backup b/deployment/terraform/terraform.tfstate.backup index 8e9163a..de5103c 100644 --- a/deployment/terraform/terraform.tfstate.backup +++ b/deployment/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 422, + "serial": 436, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { "cloud_function_name": { @@ -13,11 +13,11 @@ "type": "string" }, "cloud_run_qdrant_host": { - "value": "34.71.6.1", + "value": "35.193.174.125", "type": "string" }, "cloud_run_qdrant_host_internal": { - "value": "10.128.0.6", + "value": "10.128.0.7", "type": "string" }, "cloud_run_url": { @@ -32,8 +32,8 @@ "value": { "cloud_run_url": "https://sereact-p64zpdtkta-uc.a.run.app", "firestore_database": "sereact-imagedb", - "qdrant_endpoint": "http://34.71.6.1:6333", - "qdrant_host_ip": "34.71.6.1", + "qdrant_endpoint": "http://35.193.174.125:6333", + "qdrant_host_ip": "35.193.174.125", "static_ip_enabled": false, "storage_bucket": "sereact-images" }, @@ -62,11 +62,11 @@ "type": "string" }, "qdrant_grpc_endpoint": { - "value": "http://34.71.6.1:6334", + "value": "http://35.193.174.125:6334", "type": "string" }, "qdrant_http_endpoint": { - "value": "http://34.71.6.1:6333", + "value": "http://35.193.174.125:6333", "type": "string" }, "storage_bucket_name": { @@ -74,11 +74,11 @@ "type": "string" }, "vector_db_vm_external_ip": { - "value": "34.71.6.1", + "value": "35.193.174.125", "type": "string" }, "vector_db_vm_internal_ip": { - "value": "10.128.0.6", + "value": "10.128.0.7", "type": "string" }, "vector_db_vm_name": { @@ -98,16 +98,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "bfc4b3b9de401cd15676a09a067a8e4095b0bf4e", - "output_base64sha256": "cXx9sC1kIbTDG7BlKAtf3FUasHLZ/wZPzVoyBvt8p9Q=", - "output_base64sha512": "dcntRZ4Hz4dfBBj7YVsTzx+SEAqCXZCD8TAAh8cr5xa3uT2Lsmtf8zpxpyQEeMlsCNaF8dUohGQ7BD9LJYigPw==", + "id": "88ee03db0f4c7023c0c620449e167ad27074fdd0", + "output_base64sha256": "0p558sP6ikbyrfmva7zGOYklnR/4VRPD1zcl8HZcv8A=", + "output_base64sha512": "mal2zoxqjg5lZYruPmffQdDqY9FJONPc5Wnu41NP07LOj/tC+sJAAeQ7tmU0mq8h6SfQE6wwxFeYJuEO1y2xLg==", "output_file_mode": null, - "output_md5": "46efa1aee5386e2f244b597289c7c4ba", + "output_md5": "58a2b7fe53bb2c8c921405cc965d635c", "output_path": "./function-source.zip", - "output_sha": "bfc4b3b9de401cd15676a09a067a8e4095b0bf4e", - "output_sha256": "717c7db02d6421b4c31bb065280b5fdc551ab072d9ff064fcd5a3206fb7ca7d4", - "output_sha512": "75c9ed459e07cf875f0418fb615b13cf1f92100a825d9083f1300087c72be716b7b93d8bb26b5ff33a71a7240478c96c08d685f1d52884643b043f4b2588a03f", - "output_size": 6734, + "output_sha": "88ee03db0f4c7023c0c620449e167ad27074fdd0", + "output_sha256": "d29e79f2c3fa8a46f2adf9af6bbcc63989259d1ff85513c3d73725f0765cbfc0", + "output_sha512": "99a976ce8c6a8e0e65658aee3e67df41d0ea63d14938d3dce569eee3534fd3b2ce8ffb42fac24001e43bb665349aaf21e927d013ac30c4579826e10ed72db12e", + "output_size": 69765973, "source": [], "source_content": null, "source_content_filename": null, @@ -172,7 +172,7 @@ "effective_annotations": { "run.googleapis.com/ingress": "all", "run.googleapis.com/ingress-status": "all", - "run.googleapis.com/operation-id": "23c7f059-a4af-4cc2-95a4-f0fe54d0d7e6", + "run.googleapis.com/operation-id": "0f195b05-99ac-4d28-b5fe-2d3dea289124", "run.googleapis.com/urls": "[\"https://sereact-761163285547.us-central1.run.app\",\"https://sereact-p64zpdtkta-uc.a.run.app\"]", "serving.knative.dev/creator": "johnpccd3@gmail.com", "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" @@ -181,15 +181,15 @@ "cloud.googleapis.com/location": "us-central1", "goog-terraform-provisioned": "true" }, - "generation": 2, - "labels": {}, + "generation": 1, + "labels": null, "namespace": "gen-lang-client-0424120530", - "resource_version": "AAY18rUlZh4", + "resource_version": "AAY189oNgAQ", "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/sereact", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "d5532269-ab10-4b77-b90f-698306bf0919" + "uid": "20e61eb3-6217-40e8-8ae5-45111d31bbda" } ], "name": "sereact", @@ -216,14 +216,14 @@ "type": "RoutesReady" } ], - "latest_created_revision_name": "sereact-00002-q2g", - "latest_ready_revision_name": "sereact-00002-q2g", - "observed_generation": 2, + "latest_created_revision_name": "sereact-00001-2lz", + "latest_ready_revision_name": "sereact-00001-2lz", + "observed_generation": 1, "traffic": [ { "latest_revision": true, "percent": 100, - "revision_name": "sereact-00002-q2g", + "revision_name": "sereact-00001-2lz", "tag": "", "url": "" } @@ -256,8 +256,8 @@ "container_concurrency": 80, "containers": [ { - "args": [], - "command": [], + "args": null, + "command": null, "env": [ { "name": "API_KEY_SECRET", @@ -291,7 +291,7 @@ }, { "name": "QDRANT_HOST", - "value": "34.71.6.1", + "value": "35.193.174.125", "value_from": [] }, { @@ -337,7 +337,7 @@ "cpu": "1", "memory": "1Gi" }, - "requests": {} + "requests": null } ], "startup_probe": [ @@ -359,7 +359,7 @@ "working_dir": "" } ], - "node_selector": {}, + "node_selector": null, "service_account_name": "761163285547-compute@developer.gserviceaccount.com", "serving_state": "", "timeout_seconds": 300, @@ -440,7 +440,7 @@ "schema_version": 0, "attributes": { "condition": [], - "etag": "BwY16UdHJ00=", + "etag": "BwY189qg+AA=", "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact/roles/run.invoker/allUsers", "location": "us-central1", "member": "allUsers", @@ -474,7 +474,7 @@ "automatic_update_policy": [ {} ], - "build": "projects/761163285547/locations/us-central1/builds/47fecac1-7233-42af-ad0e-2a5a9c66e282", + "build": "projects/761163285547/locations/us-central1/builds/c627da1f-c247-4d17-8ea8-5c03b518b1aa", "docker_repository": "projects/gen-lang-client-0424120530/locations/us-central1/repositories/gcf-artifacts", "entry_point": "process_image_embedding", "environment_variables": {}, @@ -488,7 +488,7 @@ { "bucket": "gen-lang-client-0424120530-cloud-function-source", "generation": 1748123369545880, - "object": "function-source-46efa1aee5386e2f244b597289c7c4ba.zip" + "object": "function-source-58a2b7fe53bb2c8c921405cc965d635c.zip" } ] } @@ -534,7 +534,7 @@ "PROJECT_ID": "gen-lang-client-0424120530", "QDRANT_API_KEY": "", "QDRANT_COLLECTION": "image_vectors", - "QDRANT_HOST": "34.71.6.1", + "QDRANT_HOST": "35.193.174.125", "QDRANT_HTTPS": "false", "QDRANT_PORT": "6333", "VERTEX_AI_LOCATION": "us-central1" @@ -554,12 +554,12 @@ "vpc_connector_egress_settings": "" } ], - "state": "ACTIVE", + "state": "DEPLOYING", "terraform_labels": { "goog-terraform-provisioned": "true" }, "timeouts": null, - "update_time": "2025-05-25T09:26:49.473142736Z", + "update_time": "2025-05-25T11:06:59.358560449Z", "url": "https://us-central1-gen-lang-client-0424120530.cloudfunctions.net/process-image-embedding" }, "sensitive_attributes": [ @@ -603,6 +603,13 @@ } ] }, + { + "mode": "managed", + "type": "google_compute_address", + "name": "vector_db_static_ip", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [] + }, { "mode": "managed", "type": "google_compute_firewall", @@ -711,7 +718,7 @@ "can_ip_forward": false, "confidential_instance_config": [], "cpu_platform": "Intel Broadwell", - "creation_timestamp": "2025-05-24T14:09:30.477-07:00", + "creation_timestamp": "2025-05-25T03:58:31.230-07:00", "current_status": "RUNNING", "deletion_protection": false, "description": "", @@ -724,21 +731,21 @@ "hostname": "", "id": "projects/gen-lang-client-0424120530/zones/us-central1-a/instances/sereact-vector-db", "instance_encryption_key": [], - "instance_id": "1246596410843827045", + "instance_id": "3665939918038714681", "key_revocation_action_type": "", "label_fingerprint": "vezUS-42LLM=", "labels": {}, "machine_type": "e2-standard-2", "metadata": {}, - "metadata_fingerprint": "cE8FbgySELs=", - "metadata_startup_script": "#!/bin/bash\n\n# Qdrant Vector Database Installation Script\n# This script installs and configures Qdrant on Ubuntu 22.04\n\nset -e\n\n# Update system packages\napt-get update\napt-get upgrade -y\n\n# Install required packages\napt-get install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates\n\n# Install Docker\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg\necho \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list \u003e /dev/null\napt-get update\napt-get install -y docker-ce docker-ce-cli containerd.io\n\n# Start and enable Docker\nsystemctl start docker\nsystemctl enable docker\n\n# Create qdrant user and directories\nuseradd -r -s /bin/false qdrant || true\nmkdir -p /opt/qdrant/storage\nmkdir -p /opt/qdrant/config\nchown -R qdrant:qdrant /opt/qdrant\n\n# Create Qdrant configuration file\ncat \u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_cors: true\n\nstorage:\n storage_path: /qdrant/storage\n snapshots_path: /qdrant/snapshots\n on_disk_payload: true\n\ncluster:\n enabled: false\n\ntelemetry:\n disabled: true\n\nlog_level: INFO\nEOF\n\n# Create API key configuration if provided\nif [ -n \"\" ] \u0026\u0026 [ \"\" != \"\" ]; then\ncat \u003e\u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\n\nservice:\n api_key: \"\"\nEOF\nfi\n\n# Create systemd service for Qdrant\ncat \u003e /etc/systemd/system/qdrant.service \u003c\u003c EOF\n[Unit]\nDescription=Qdrant Vector Database\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nType=simple\nUser=root\nExecStartPre=-/usr/bin/docker stop qdrant\nExecStartPre=-/usr/bin/docker rm qdrant\nExecStart=/usr/bin/docker run --name qdrant \\\n -p 6333:6333 \\\n -p 6334:6334 \\\n -v /opt/qdrant/storage:/qdrant/storage:z \\\n -v /opt/qdrant/config/config.yaml:/qdrant/config/production.yaml:z \\\n qdrant/qdrant:latest\nExecStop=/usr/bin/docker stop qdrant\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n# Pull Qdrant Docker image\ndocker pull qdrant/qdrant:latest\n\n# Enable and start Qdrant service\nsystemctl daemon-reload\nsystemctl enable qdrant\nsystemctl start qdrant\n\n# Install monitoring tools\napt-get install -y htop iotop nethogs\n\n# Create a simple health check script\ncat \u003e /opt/qdrant/health_check.sh \u003c\u003c 'EOF'\n#!/bin/bash\nresponse=$(curl -s -o /dev/null -w \"%{http_code}\" http://localhost:6333/health)\nif [ \"$response\" = \"200\" ]; then\n echo \"Qdrant is healthy\"\n exit 0\nelse\n echo \"Qdrant is not responding properly (HTTP $response)\"\n exit 1\nfi\nEOF\n\nchmod +x /opt/qdrant/health_check.sh\n\n# Set up log rotation for Docker logs\ncat \u003e /etc/logrotate.d/docker \u003c\u003c EOF\n/var/lib/docker/containers/*/*.log {\n rotate 7\n daily\n compress\n size=1M\n missingok\n delaycompress\n copytruncate\n}\nEOF\n\n# Configure firewall (ufw)\nufw --force enable\nufw allow ssh\nufw allow 6333/tcp # Qdrant HTTP API\nufw allow 6334/tcp # Qdrant gRPC API\n\n# Create a simple backup script\ncat \u003e /opt/qdrant/backup.sh \u003c\u003c 'EOF'\n#!/bin/bash\nBACKUP_DIR=\"/opt/qdrant/backups\"\nDATE=$(date +%Y%m%d_%H%M%S)\nmkdir -p $BACKUP_DIR\n\n# Create snapshot via API\ncurl -X POST \"http://localhost:6333/snapshots\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"snapshot_name\": \"backup_'$DATE'\"}'\n\n# Copy storage directory\ntar -czf $BACKUP_DIR/qdrant_storage_$DATE.tar.gz -C /opt/qdrant storage/\n\n# Keep only last 7 backups\nfind $BACKUP_DIR -name \"*.tar.gz\" -mtime +7 -delete\n\necho \"Backup completed: $DATE\"\nEOF\n\nchmod +x /opt/qdrant/backup.sh\n\n# Set up daily backup cron job\necho \"0 2 * * * root /opt/qdrant/backup.sh \u003e\u003e /var/log/qdrant_backup.log 2\u003e\u00261\" \u003e\u003e /etc/crontab\n\n# Wait for Qdrant to be ready\necho \"Waiting for Qdrant to start...\"\nfor i in {1..30}; do\n if curl -s http://localhost:6333/health \u003e /dev/null; then\n echo \"Qdrant is ready!\"\n break\n fi\n echo \"Waiting... ($i/30)\"\n sleep 10\ndone\n\n# Create a default collection for image vectors\ncurl -X PUT \"http://localhost:6333/collections/image_vectors\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"vectors\": {\n \"size\": 512,\n \"distance\": \"Cosine\"\n },\n \"optimizers_config\": {\n \"default_segment_number\": 2\n },\n \"replication_factor\": 1\n }'\n\necho \"Qdrant installation and configuration completed!\"\necho \"Qdrant is accessible at:\"\necho \" HTTP API: http://$(curl -s ifconfig.me):6333\"\necho \" gRPC API: http://$(curl -s ifconfig.me):6334\"\necho \"Health check: /opt/qdrant/health_check.sh\"\necho \"Backup script: /opt/qdrant/backup.sh\" ", + "metadata_fingerprint": "nyOcq7EkvZA=", + "metadata_startup_script": "#!/bin/bash\n\n# Qdrant Vector Database Installation Script\n# This script installs and configures Qdrant on Ubuntu 22.04\n\nset -e\n\n# Update system packages\napt-get update\napt-get upgrade -y\n\n# Install required packages\napt-get install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates\n\n# Install Docker\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg\necho \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list \u003e /dev/null\napt-get update\napt-get install -y docker-ce docker-ce-cli containerd.io\n\n# Start and enable Docker\nsystemctl start docker\nsystemctl enable docker\n\n# Create qdrant user and directories\nuseradd -r -s /bin/false qdrant || true\nmkdir -p /opt/qdrant/storage\nmkdir -p /opt/qdrant/config\nchown -R qdrant:qdrant /opt/qdrant\n\n# Create Qdrant configuration file\ncat \u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_cors: true\n\nstorage:\n storage_path: /qdrant/storage\n snapshots_path: /qdrant/snapshots\n on_disk_payload: true\n\ncluster:\n enabled: false\n\ntelemetry:\n disabled: true\n\nlog_level: INFO\nEOF\n\n# Create API key configuration if provided\nif [ -n \"\" ] \u0026\u0026 [ \"\" != \"\" ]; then\ncat \u003e\u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\n\nservice:\n api_key: \"\"\nEOF\nfi\n\n# Create systemd service for Qdrant\ncat \u003e /etc/systemd/system/qdrant.service \u003c\u003c EOF\n[Unit]\nDescription=Qdrant Vector Database\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nType=simple\nUser=root\nExecStartPre=-/usr/bin/docker stop qdrant\nExecStartPre=-/usr/bin/docker rm qdrant\nExecStart=/usr/bin/docker run --name qdrant \\\n -p 6333:6333 \\\n -p 6334:6334 \\\n -v /opt/qdrant/storage:/qdrant/storage:z \\\n -v /opt/qdrant/config/config.yaml:/qdrant/config/production.yaml:z \\\n qdrant/qdrant:latest\nExecStop=/usr/bin/docker stop qdrant\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n# Pull Qdrant Docker image\ndocker pull qdrant/qdrant:latest\n\n# Enable and start Qdrant service\nsystemctl daemon-reload\nsystemctl enable qdrant\nsystemctl start qdrant\n\n# Install monitoring tools\napt-get install -y htop iotop nethogs\n\n# Create a simple health check script\ncat \u003e /opt/qdrant/health_check.sh \u003c\u003c 'EOF'\n#!/bin/bash\nresponse=$(curl -s -o /dev/null -w \"%{http_code}\" http://localhost:6333/health)\nif [ \"$response\" = \"200\" ]; then\n echo \"Qdrant is healthy\"\n exit 0\nelse\n echo \"Qdrant is not responding properly (HTTP $response)\"\n exit 1\nfi\nEOF\n\nchmod +x /opt/qdrant/health_check.sh\n\n# Set up log rotation for Docker logs\ncat \u003e /etc/logrotate.d/docker \u003c\u003c EOF\n/var/lib/docker/containers/*/*.log {\n rotate 7\n daily\n compress\n size=1M\n missingok\n delaycompress\n copytruncate\n}\nEOF\n\n# Configure firewall (ufw)\nufw --force enable\nufw allow ssh\nufw allow 6333/tcp # Qdrant HTTP API\nufw allow 6334/tcp # Qdrant gRPC API\n\n# Create a simple backup script\ncat \u003e /opt/qdrant/backup.sh \u003c\u003c 'EOF'\n#!/bin/bash\nBACKUP_DIR=\"/opt/qdrant/backups\"\nDATE=$(date +%Y%m%d_%H%M%S)\nmkdir -p $BACKUP_DIR\n\n# Create snapshot via API\ncurl -X POST \"http://localhost:6333/snapshots\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"snapshot_name\": \"backup_'$DATE'\"}'\n\n# Copy storage directory\ntar -czf $BACKUP_DIR/qdrant_storage_$DATE.tar.gz -C /opt/qdrant storage/\n\n# Keep only last 7 backups\nfind $BACKUP_DIR -name \"*.tar.gz\" -mtime +7 -delete\n\necho \"Backup completed: $DATE\"\nEOF\n\nchmod +x /opt/qdrant/backup.sh\n\n# Set up daily backup cron job\necho \"0 2 * * * root /opt/qdrant/backup.sh \u003e\u003e /var/log/qdrant_backup.log 2\u003e\u00261\" \u003e\u003e /etc/crontab\n\n# Wait for Qdrant to be ready\necho \"Waiting for Qdrant to start...\"\nfor i in {1..30}; do\n if curl -s http://localhost:6333/health \u003e /dev/null; then\n echo \"Qdrant is ready!\"\n break\n fi\n echo \"Waiting... ($i/30)\"\n sleep 10\ndone\n\n# Create a default collection for image vectors\ncurl -X PUT \"http://localhost:6333/collections/image_vectors\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"vectors\": {\n \"size\": 1408,\n \"distance\": \"Cosine\"\n },\n \"optimizers_config\": {\n \"default_segment_number\": 2\n },\n \"replication_factor\": 1\n }'\n\necho \"Qdrant installation and configuration completed!\"\necho \"Qdrant is accessible at:\"\necho \" HTTP API: http://$(curl -s ifconfig.me):6333\"\necho \" gRPC API: http://$(curl -s ifconfig.me):6334\"\necho \"Health check: /opt/qdrant/health_check.sh\"\necho \"Backup script: /opt/qdrant/backup.sh\" ", "min_cpu_platform": "", "name": "sereact-vector-db", "network_interface": [ { "access_config": [ { - "nat_ip": "34.71.6.1", + "nat_ip": "35.193.174.125", "network_tier": "PREMIUM", "public_ptr_domain_name": "" } @@ -751,7 +758,7 @@ "name": "nic0", "network": "https://www.googleapis.com/compute/v1/projects/gen-lang-client-0424120530/global/networks/default", "network_attachment": "", - "network_ip": "10.128.0.6", + "network_ip": "10.128.0.7", "nic_type": "", "queue_count": 0, "stack_type": "IPV4_ONLY", @@ -875,8 +882,8 @@ "database_edition": "STANDARD", "delete_protection_state": "DELETE_PROTECTION_DISABLED", "deletion_policy": "ABANDON", - "earliest_version_time": "2025-05-25T08:56:42.972923Z", - "etag": "IMzA4cmtvo0DMKrW4vCEvY0D", + "earliest_version_time": "2025-05-25T10:07:57.305684Z", + "etag": "IIOH9r+9vo0DMKrW4vCEvY0D", "id": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", "key_prefix": "", "location_id": "us-central1", @@ -1514,21 +1521,21 @@ "content_encoding": "", "content_language": "", "content_type": "application/zip", - "crc32c": "kTROsA==", + "crc32c": "eCjQFg==", "customer_encryption": [], - "detect_md5hash": "Ru+hruU4bi8kS1lyicfEug==", + "detect_md5hash": "WKK3/lO7LIySFAXMll1jXA==", "event_based_hold": false, - "generation": 1748126317190781, - "id": "gen-lang-client-0424120530-cloud-function-source-function-source-46efa1aee5386e2f244b597289c7c4ba.zip", + "generation": 1748170673167525, + "id": "gen-lang-client-0424120530-cloud-function-source-function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", "kms_key_name": "", - "md5hash": "Ru+hruU4bi8kS1lyicfEug==", - "md5hexhash": "46efa1aee5386e2f244b597289c7c4ba", - "media_link": "https://storage.googleapis.com/download/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-46efa1aee5386e2f244b597289c7c4ba.zip?generation=1748126317190781\u0026alt=media", + "md5hash": "WKK3/lO7LIySFAXMll1jXA==", + "md5hexhash": "58a2b7fe53bb2c8c921405cc965d635c", + "media_link": "https://storage.googleapis.com/download/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-58a2b7fe53bb2c8c921405cc965d635c.zip?generation=1748170673167525\u0026alt=media", "metadata": {}, - "name": "function-source-46efa1aee5386e2f244b597289c7c4ba.zip", - "output_name": "function-source-46efa1aee5386e2f244b597289c7c4ba.zip", + "name": "function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", + "output_name": "function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", "retention": [], - "self_link": "https://www.googleapis.com/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-46efa1aee5386e2f244b597289c7c4ba.zip", + "self_link": "https://www.googleapis.com/storage/v1/b/gen-lang-client-0424120530-cloud-function-source/o/function-source-58a2b7fe53bb2c8c921405cc965d635c.zip", "source": "./function-source.zip", "storage_class": "STANDARD", "temporary_hold": false, diff --git a/requirements.txt b/requirements.txt index 15372a5..23a9e1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ google-cloud-storage==2.12.0 google-cloud-vision==3.4.5 google-cloud-firestore==2.11.1 google-cloud-pubsub==2.18.4 +google-cloud-aiplatform==1.38.0 python-multipart==0.0.6 python-jose==3.3.0 passlib==1.7.4 diff --git a/scripts/push.sh b/scripts/push.sh deleted file mode 100644 index dc102b0..0000000 --- a/scripts/push.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -e - -# Source the build environment to reuse variables -source "$(dirname "$0")/build.sh" - -# Push the Docker image to the registry -echo "Pushing image: ${FULL_IMAGE_NAME} to registry..." -docker push "${FULL_IMAGE_NAME}" -echo "Image pushed successfully" - -echo "" -echo "Image pushed to: ${FULL_IMAGE_NAME}" -echo "" \ No newline at end of file