From bfa1292b054044ae762266ef02f182169b8c1171 Mon Sep 17 00:00:00 2001 From: johnpccd Date: Sat, 24 May 2025 20:16:39 +0200 Subject: [PATCH] cp --- deployment/terraform/terraform.tfstate | 207 +++++++++++++++--- deployment/terraform/terraform.tfstate.backup | 158 +++++++++++-- 2 files changed, 314 insertions(+), 51 deletions(-) diff --git a/deployment/terraform/terraform.tfstate b/deployment/terraform/terraform.tfstate index 12f5e7b..98a27cd 100644 --- a/deployment/terraform/terraform.tfstate +++ b/deployment/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 254, + "serial": 260, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { "cloud_run_qdrant_host": { @@ -12,10 +12,39 @@ "value": "10.128.0.4", "type": "string" }, + "cloud_run_url": { + "value": "https://sereact-p64zpdtkta-uc.a.run.app", + "type": "string" + }, "container_registry_url": { "value": "gcr.io/gen-lang-client-0424120530/sereact", "type": "string" }, + "deployment_summary": { + "value": { + "cloud_run_url": "https://sereact-p64zpdtkta-uc.a.run.app", + "firestore_database": "sereact-imagedb", + "qdrant_endpoint": "http://34.71.6.1:6333", + "qdrant_host_ip": "34.71.6.1", + "static_ip_enabled": false, + "storage_bucket": "sereact-images" + }, + "type": [ + "object", + { + "cloud_run_url": "string", + "firestore_database": "string", + "qdrant_endpoint": "string", + "qdrant_host_ip": "string", + "static_ip_enabled": "bool", + "storage_bucket": "string" + } + ] + }, + "firestore_database_id": { + "value": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", + "type": "string" + }, "pubsub_dlq_topic_name": { "value": "image-processing-topic-dlq", "type": "string" @@ -94,7 +123,6 @@ "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", "instances": [ { - "status": "tainted", "schema_version": 2, "attributes": { "autogenerate_revision_name": false, @@ -106,25 +134,67 @@ "run.googleapis.com/ingress": "all" }, "effective_annotations": { - "run.googleapis.com/ingress": "all" + "run.googleapis.com/ingress": "all", + "run.googleapis.com/ingress-status": "all", + "run.googleapis.com/operation-id": "5c612fa4-f8c2-4757-88a1-c0c2fe67d0ad", + "run.googleapis.com/urls": "[\"https://sereact-761163285547.us-central1.run.app\",\"https://sereact-p64zpdtkta-uc.a.run.app\"]", + "serving.knative.dev/creator": "johnpccd3@gmail.com", + "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" }, "effective_labels": { + "cloud.googleapis.com/location": "us-central1", "goog-terraform-provisioned": "true" }, - "generation": 0, + "generation": 1, "labels": null, - "namespace": "", - "resource_version": "", - "self_link": "", + "namespace": "gen-lang-client-0424120530", + "resource_version": "AAY15bM0evU", + "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/sereact", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "" + "uid": "f67eed78-59c9-4bb6-a118-fb940f206d09" } ], "name": "sereact", "project": "gen-lang-client-0424120530", - "status": null, + "status": [ + { + "conditions": [ + { + "message": "", + "reason": "", + "status": "True", + "type": "Ready" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "ConfigurationsReady" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "RoutesReady" + } + ], + "latest_created_revision_name": "sereact-00001-rsq", + "latest_ready_revision_name": "sereact-00001-rsq", + "observed_generation": 1, + "traffic": [ + { + "latest_revision": true, + "percent": 100, + "revision_name": "sereact-00001-rsq", + "tag": "", + "url": "" + } + ], + "url": "https://sereact-p64zpdtkta-uc.a.run.app" + } + ], "template": [ { "metadata": [ @@ -133,7 +203,9 @@ "autoscaling.knative.dev/maxScale": "10" }, "generation": 0, - "labels": {}, + "labels": { + "run.googleapis.com/startupProbeType": "Default" + }, "name": "", "namespace": "", "resource_version": "", @@ -143,7 +215,7 @@ ], "spec": [ { - "container_concurrency": 0, + "container_concurrency": 80, "containers": [ { "args": null, @@ -202,7 +274,7 @@ "ports": [ { "container_port": 8000, - "name": "", + "name": "http1", "protocol": "" } ], @@ -215,15 +287,29 @@ "requests": null } ], - "startup_probe": [], + "startup_probe": [ + { + "failure_threshold": 1, + "grpc": [], + "http_get": [], + "initial_delay_seconds": 0, + "period_seconds": 240, + "tcp_socket": [ + { + "port": 8000 + } + ], + "timeout_seconds": 240 + } + ], "volume_mounts": [], "working_dir": "" } ], "node_selector": null, - "service_account_name": "", + "service_account_name": "761163285547-compute@developer.gserviceaccount.com", "serving_state": "", - "timeout_seconds": 0, + "timeout_seconds": 300, "volumes": [] } ] @@ -291,6 +377,36 @@ } ] }, + { + "mode": "managed", + "type": "google_cloud_run_service_iam_member", + "name": "public_access", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwY15bNaQMM=", + "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact/roles/run.invoker/allUsers", + "location": "us-central1", + "member": "allUsers", + "project": "gen-lang-client-0424120530", + "role": "roles/run.invoker", + "service": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "google_cloud_run_service.sereact", + "google_compute_address.vector_db_static_ip", + "google_compute_instance.vector_db_vm", + "google_project_service.services", + "google_service_account.vector_db_sa" + ] + } + ] + }, { "mode": "managed", "type": "google_compute_address", @@ -507,7 +623,18 @@ [ { "type": "get_attr", - "value": "metadata_startup_script" + "value": "boot_disk" + }, + { + "type": "index", + "value": { + "value": 0, + "type": "number" + } + }, + { + "type": "get_attr", + "value": "disk_encryption_key_raw" } ], [ @@ -530,18 +657,7 @@ [ { "type": "get_attr", - "value": "boot_disk" - }, - { - "type": "index", - "value": { - "value": 0, - "type": "number" - } - }, - { - "type": "get_attr", - "value": "disk_encryption_key_raw" + "value": "metadata_startup_script" } ] ], @@ -554,6 +670,41 @@ } ] }, + { + "mode": "managed", + "type": "google_firestore_database", + "name": "database", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "app_engine_integration_mode": "DISABLED", + "cmek_config": [], + "concurrency_mode": "PESSIMISTIC", + "create_time": "", + "database_edition": "STANDARD", + "delete_protection_state": "DELETE_PROTECTION_DISABLED", + "deletion_policy": "ABANDON", + "earliest_version_time": "2025-05-24T17:16:21.411338Z", + "etag": "IM/W7rHbvI0DMLa2lbPIvI0D", + "id": "projects/gen-lang-client-0424120530/databases/sereact-imagedb", + "key_prefix": "", + "location_id": "us-central1", + "name": "sereact-imagedb", + "point_in_time_recovery_enablement": "POINT_IN_TIME_RECOVERY_DISABLED", + "project": "gen-lang-client-0424120530", + "timeouts": null, + "type": "FIRESTORE_NATIVE", + "uid": "30634721-c8c7-4673-b605-c1a91dff3eab", + "update_time": "", + "version_retention_period": "3600s" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMCJ9" + } + ] + }, { "mode": "managed", "type": "google_project_service", diff --git a/deployment/terraform/terraform.tfstate.backup b/deployment/terraform/terraform.tfstate.backup index 45ae0dc..0f12ea7 100644 --- a/deployment/terraform/terraform.tfstate.backup +++ b/deployment/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.10.1", - "serial": 250, + "serial": 259, "lineage": "a183cd95-f987-8698-c6dd-84e933c394a5", "outputs": { "cloud_run_qdrant_host": { @@ -12,10 +12,35 @@ "value": "10.128.0.4", "type": "string" }, + "cloud_run_url": { + "value": "https://sereact-p64zpdtkta-uc.a.run.app", + "type": "string" + }, "container_registry_url": { "value": "gcr.io/gen-lang-client-0424120530/sereact", "type": "string" }, + "deployment_summary": { + "value": { + "cloud_run_url": "https://sereact-p64zpdtkta-uc.a.run.app", + "firestore_database": "sereact-imagedb", + "qdrant_endpoint": "http://34.71.6.1:6333", + "qdrant_host_ip": "34.71.6.1", + "static_ip_enabled": false, + "storage_bucket": "sereact-images" + }, + "type": [ + "object", + { + "cloud_run_url": "string", + "firestore_database": "string", + "qdrant_endpoint": "string", + "qdrant_host_ip": "string", + "static_ip_enabled": "bool", + "storage_bucket": "string" + } + ] + }, "pubsub_dlq_topic_name": { "value": "image-processing-topic-dlq", "type": "string" @@ -94,7 +119,6 @@ "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", "instances": [ { - "status": "tainted", "schema_version": 2, "attributes": { "autogenerate_revision_name": false, @@ -106,25 +130,67 @@ "run.googleapis.com/ingress": "all" }, "effective_annotations": { - "run.googleapis.com/ingress": "all" + "run.googleapis.com/ingress": "all", + "run.googleapis.com/ingress-status": "all", + "run.googleapis.com/operation-id": "5c612fa4-f8c2-4757-88a1-c0c2fe67d0ad", + "run.googleapis.com/urls": "[\"https://sereact-761163285547.us-central1.run.app\",\"https://sereact-p64zpdtkta-uc.a.run.app\"]", + "serving.knative.dev/creator": "johnpccd3@gmail.com", + "serving.knative.dev/lastModifier": "johnpccd3@gmail.com" }, "effective_labels": { + "cloud.googleapis.com/location": "us-central1", "goog-terraform-provisioned": "true" }, - "generation": 0, + "generation": 1, "labels": null, - "namespace": "", - "resource_version": "", - "self_link": "", + "namespace": "gen-lang-client-0424120530", + "resource_version": "AAY15bM0evU", + "self_link": "/apis/serving.knative.dev/v1/namespaces/761163285547/services/sereact", "terraform_labels": { "goog-terraform-provisioned": "true" }, - "uid": "" + "uid": "f67eed78-59c9-4bb6-a118-fb940f206d09" } ], "name": "sereact", "project": "gen-lang-client-0424120530", - "status": null, + "status": [ + { + "conditions": [ + { + "message": "", + "reason": "", + "status": "True", + "type": "Ready" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "ConfigurationsReady" + }, + { + "message": "", + "reason": "", + "status": "True", + "type": "RoutesReady" + } + ], + "latest_created_revision_name": "sereact-00001-rsq", + "latest_ready_revision_name": "sereact-00001-rsq", + "observed_generation": 1, + "traffic": [ + { + "latest_revision": true, + "percent": 100, + "revision_name": "sereact-00001-rsq", + "tag": "", + "url": "" + } + ], + "url": "https://sereact-p64zpdtkta-uc.a.run.app" + } + ], "template": [ { "metadata": [ @@ -133,7 +199,9 @@ "autoscaling.knative.dev/maxScale": "10" }, "generation": 0, - "labels": {}, + "labels": { + "run.googleapis.com/startupProbeType": "Default" + }, "name": "", "namespace": "", "resource_version": "", @@ -143,7 +211,7 @@ ], "spec": [ { - "container_concurrency": 0, + "container_concurrency": 80, "containers": [ { "args": null, @@ -202,7 +270,7 @@ "ports": [ { "container_port": 8000, - "name": "", + "name": "http1", "protocol": "" } ], @@ -215,15 +283,29 @@ "requests": null } ], - "startup_probe": [], + "startup_probe": [ + { + "failure_threshold": 1, + "grpc": [], + "http_get": [], + "initial_delay_seconds": 0, + "period_seconds": 240, + "tcp_socket": [ + { + "port": 8000 + } + ], + "timeout_seconds": 240 + } + ], "volume_mounts": [], "working_dir": "" } ], "node_selector": null, - "service_account_name": "", + "service_account_name": "761163285547-compute@developer.gserviceaccount.com", "serving_state": "", - "timeout_seconds": 0, + "timeout_seconds": 300, "volumes": [] } ] @@ -291,6 +373,36 @@ } ] }, + { + "mode": "managed", + "type": "google_cloud_run_service_iam_member", + "name": "public_access", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwY15bNaQMM=", + "id": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact/roles/run.invoker/allUsers", + "location": "us-central1", + "member": "allUsers", + "project": "gen-lang-client-0424120530", + "role": "roles/run.invoker", + "service": "v1/projects/gen-lang-client-0424120530/locations/us-central1/services/sereact" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "google_cloud_run_service.sereact", + "google_compute_address.vector_db_static_ip", + "google_compute_instance.vector_db_vm", + "google_project_service.services", + "google_service_account.vector_db_sa" + ] + } + ] + }, { "mode": "managed", "type": "google_compute_address", @@ -334,9 +446,9 @@ "0.0.0.0/0", "10.0.0.0/8" ], - "source_service_accounts": null, - "source_tags": null, - "target_service_accounts": null, + "source_service_accounts": [], + "source_tags": [], + "target_service_accounts": [], "target_tags": [ "qdrant" ], @@ -387,7 +499,7 @@ "labels": {}, "provisioned_iops": 0, "provisioned_throughput": 0, - "resource_manager_tags": null, + "resource_manager_tags": {}, "resource_policies": [], "size": 50, "snapshot": "", @@ -422,9 +534,9 @@ "instance_id": "8076617322369889915", "key_revocation_action_type": "", "label_fingerprint": "vezUS-42LLM=", - "labels": null, + "labels": {}, "machine_type": "e2-standard-2", - "metadata": null, + "metadata": {}, "metadata_fingerprint": "cE8FbgySELs=", "metadata_startup_script": "#!/bin/bash\n\n# Qdrant Vector Database Installation Script\n# This script installs and configures Qdrant on Ubuntu 22.04\n\nset -e\n\n# Update system packages\napt-get update\napt-get upgrade -y\n\n# Install required packages\napt-get install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates\n\n# Install Docker\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg\necho \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | tee /etc/apt/sources.list.d/docker.list \u003e /dev/null\napt-get update\napt-get install -y docker-ce docker-ce-cli containerd.io\n\n# Start and enable Docker\nsystemctl start docker\nsystemctl enable docker\n\n# Create qdrant user and directories\nuseradd -r -s /bin/false qdrant || true\nmkdir -p /opt/qdrant/storage\nmkdir -p /opt/qdrant/config\nchown -R qdrant:qdrant /opt/qdrant\n\n# Create Qdrant configuration file\ncat \u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\nservice:\n host: 0.0.0.0\n http_port: 6333\n grpc_port: 6334\n enable_cors: true\n\nstorage:\n storage_path: /qdrant/storage\n snapshots_path: /qdrant/snapshots\n on_disk_payload: true\n\ncluster:\n enabled: false\n\ntelemetry:\n disabled: true\n\nlog_level: INFO\nEOF\n\n# Create API key configuration if provided\nif [ -n \"\" ] \u0026\u0026 [ \"\" != \"\" ]; then\ncat \u003e\u003e /opt/qdrant/config/config.yaml \u003c\u003c EOF\n\nservice:\n api_key: \"\"\nEOF\nfi\n\n# Create systemd service for Qdrant\ncat \u003e /etc/systemd/system/qdrant.service \u003c\u003c EOF\n[Unit]\nDescription=Qdrant Vector Database\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nType=simple\nUser=root\nExecStartPre=-/usr/bin/docker stop qdrant\nExecStartPre=-/usr/bin/docker rm qdrant\nExecStart=/usr/bin/docker run --name qdrant \\\n -p 6333:6333 \\\n -p 6334:6334 \\\n -v /opt/qdrant/storage:/qdrant/storage:z \\\n -v /opt/qdrant/config/config.yaml:/qdrant/config/production.yaml:z \\\n qdrant/qdrant:latest\nExecStop=/usr/bin/docker stop qdrant\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n# Pull Qdrant Docker image\ndocker pull qdrant/qdrant:latest\n\n# Enable and start Qdrant service\nsystemctl daemon-reload\nsystemctl enable qdrant\nsystemctl start qdrant\n\n# Install monitoring tools\napt-get install -y htop iotop nethogs\n\n# Create a simple health check script\ncat \u003e /opt/qdrant/health_check.sh \u003c\u003c 'EOF'\n#!/bin/bash\nresponse=$(curl -s -o /dev/null -w \"%{http_code}\" http://localhost:6333/health)\nif [ \"$response\" = \"200\" ]; then\n echo \"Qdrant is healthy\"\n exit 0\nelse\n echo \"Qdrant is not responding properly (HTTP $response)\"\n exit 1\nfi\nEOF\n\nchmod +x /opt/qdrant/health_check.sh\n\n# Set up log rotation for Docker logs\ncat \u003e /etc/logrotate.d/docker \u003c\u003c EOF\n/var/lib/docker/containers/*/*.log {\n rotate 7\n daily\n compress\n size=1M\n missingok\n delaycompress\n copytruncate\n}\nEOF\n\n# Configure firewall (ufw)\nufw --force enable\nufw allow ssh\nufw allow 6333/tcp # Qdrant HTTP API\nufw allow 6334/tcp # Qdrant gRPC API\n\n# Create a simple backup script\ncat \u003e /opt/qdrant/backup.sh \u003c\u003c 'EOF'\n#!/bin/bash\nBACKUP_DIR=\"/opt/qdrant/backups\"\nDATE=$(date +%Y%m%d_%H%M%S)\nmkdir -p $BACKUP_DIR\n\n# Create snapshot via API\ncurl -X POST \"http://localhost:6333/snapshots\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"snapshot_name\": \"backup_'$DATE'\"}'\n\n# Copy storage directory\ntar -czf $BACKUP_DIR/qdrant_storage_$DATE.tar.gz -C /opt/qdrant storage/\n\n# Keep only last 7 backups\nfind $BACKUP_DIR -name \"*.tar.gz\" -mtime +7 -delete\n\necho \"Backup completed: $DATE\"\nEOF\n\nchmod +x /opt/qdrant/backup.sh\n\n# Set up daily backup cron job\necho \"0 2 * * * root /opt/qdrant/backup.sh \u003e\u003e /var/log/qdrant_backup.log 2\u003e\u00261\" \u003e\u003e /etc/crontab\n\n# Wait for Qdrant to be ready\necho \"Waiting for Qdrant to start...\"\nfor i in {1..30}; do\n if curl -s http://localhost:6333/health \u003e /dev/null; then\n echo \"Qdrant is ready!\"\n break\n fi\n echo \"Waiting... ($i/30)\"\n sleep 10\ndone\n\n# Create a default collection for image vectors\ncurl -X PUT \"http://localhost:6333/collections/image_vectors\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"vectors\": {\n \"size\": 512,\n \"distance\": \"Cosine\"\n },\n \"optimizers_config\": {\n \"default_segment_number\": 2\n },\n \"replication_factor\": 1\n }'\n\necho \"Qdrant installation and configuration completed!\"\necho \"Qdrant is accessible at:\"\necho \" HTTP API: http://$(curl -s ifconfig.me):6333\"\necho \" gRPC API: http://$(curl -s ifconfig.me):6334\"\necho \"Health check: /opt/qdrant/health_check.sh\"\necho \"Backup script: /opt/qdrant/backup.sh\" ", "min_cpu_platform": "", @@ -458,7 +570,7 @@ "params": [], "project": "gen-lang-client-0424120530", "reservation_affinity": [], - "resource_policies": null, + "resource_policies": [], "scheduling": [ { "automatic_restart": true, @@ -961,7 +1073,7 @@ } ], "id": "sereact-images", - "labels": null, + "labels": {}, "lifecycle_rule": [], "location": "US-CENTRAL1", "logging": [],