#!/bin/bash # Comprehensive Infrastructure Testing Script for Phase 1 # Tests all deployed resources and connectivity set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PHASE1_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Test results TESTS_PASSED=0 TESTS_FAILED=0 TESTS_SKIPPED=0 # Function to print test result print_test() { local status=$1 local message=$2 case $status in PASS) echo -e "${GREEN}✓${NC} $message" ((TESTS_PASSED++)) ;; FAIL) echo -e "${RED}✗${NC} $message" ((TESTS_FAILED++)) ;; SKIP) echo -e "${YELLOW}⊘${NC} $message" ((TESTS_SKIPPED++)) ;; INFO) echo -e "${BLUE}ℹ${NC} $message" ;; esac } echo "==========================================" echo "Phase 1 Infrastructure Testing" echo "==========================================" echo "" # Test 1: Terraform State echo "1. Testing Terraform State..." cd "$PHASE1_DIR" if terraform state list > /dev/null 2>&1; then RESOURCE_COUNT=$(terraform state list 2>/dev/null | wc -l) if [ "$RESOURCE_COUNT" -ge 100 ]; then print_test PASS "Terraform state accessible ($RESOURCE_COUNT resources)" else print_test FAIL "Terraform state has fewer resources than expected ($RESOURCE_COUNT)" fi else print_test FAIL "Terraform state not accessible" fi # Test 2: Terraform Outputs echo "" echo "2. Testing Terraform Outputs..." if terraform output -json > /dev/null 2>&1; then OUTPUTS=$(terraform output -json 2>/dev/null) if echo "$OUTPUTS" | jq -e '.nginx_proxy' > /dev/null 2>&1; then print_test PASS "Terraform outputs accessible" else print_test FAIL "Terraform outputs incomplete" fi else print_test FAIL "Terraform outputs not accessible" fi # Test 3: Nginx Proxy VM echo "" echo "3. Testing Nginx Proxy VM..." NGINX_IP=$(terraform output -json 2>/dev/null | jq -r '.nginx_proxy.value.public_ip // empty' || echo "") if [ -n "$NGINX_IP" ] && [ "$NGINX_IP" != "null" ]; then print_test PASS "Nginx proxy IP: $NGINX_IP" # Test SSH connectivity (non-blocking) if timeout 5 bash -c "echo > /dev/tcp/$NGINX_IP/22" 2>/dev/null; then print_test PASS "Nginx proxy SSH port (22) accessible" else print_test SKIP "Nginx proxy SSH port test (may require VPN/Bastion)" fi # Test HTTP/HTTPS ports if timeout 5 bash -c "echo > /dev/tcp/$NGINX_IP/80" 2>/dev/null; then print_test PASS "Nginx proxy HTTP port (80) accessible" else print_test INFO "Nginx proxy HTTP port (80) not yet accessible (Nginx may not be configured)" fi if timeout 5 bash -c "echo > /dev/tcp/$NGINX_IP/443" 2>/dev/null; then print_test PASS "Nginx proxy HTTPS port (443) accessible" else print_test INFO "Nginx proxy HTTPS port (443) not yet accessible (SSL not configured)" fi else print_test FAIL "Nginx proxy IP not found" fi # Test 4: Backend VMs echo "" echo "4. Testing Backend VMs..." BACKEND_VMS=$(terraform output -json 2>/dev/null | jq -r '.phase1_us_regions.value | to_entries[] | "\(.key):\(.value.private_ips[0])"' || echo "") BACKEND_COUNT=0 if [ -n "$BACKEND_VMS" ]; then while IFS=: read -r region ip; do if [ -n "$ip" ] && [ "$ip" != "null" ]; then print_test PASS "Backend VM $region: $ip" ((BACKEND_COUNT++)) # Test SSH connectivity (non-blocking, will likely fail without VPN) if timeout 3 bash -c "echo > /dev/tcp/$ip/22" 2>/dev/null; then print_test PASS " → SSH port (22) accessible" else print_test SKIP " → SSH port (22) requires VPN/Bastion (expected for private IPs)" fi # Test RPC ports (non-blocking) if timeout 3 bash -c "echo > /dev/tcp/$ip/8545" 2>/dev/null; then print_test PASS " → RPC HTTP port (8545) accessible" else print_test INFO " → RPC HTTP port (8545) not accessible (Besu may not be running)" fi if timeout 3 bash -c "echo > /dev/tcp/$ip/8546" 2>/dev/null; then print_test PASS " → RPC WebSocket port (8546) accessible" else print_test INFO " → RPC WebSocket port (8546) not accessible (Besu may not be running)" fi fi done <<< "$BACKEND_VMS" if [ "$BACKEND_COUNT" -eq 5 ]; then print_test PASS "All 5 backend VMs found" else print_test FAIL "Expected 5 backend VMs, found $BACKEND_COUNT" fi else print_test FAIL "Backend VMs not found in outputs" fi # Test 5: Azure Resource Groups echo "" echo "5. Testing Azure Resource Groups..." if command -v az > /dev/null 2>&1; then RGS=$(az group list --query "[?contains(name, 'az-p-') && contains(name, '-rg-comp-001')].name" -o tsv 2>/dev/null || echo "") RG_COUNT=$(echo "$RGS" | grep -c . || echo "0") if [ "$RG_COUNT" -ge 6 ]; then print_test PASS "Resource groups found: $RG_COUNT" echo "$RGS" | while read -r rg; do if [ -n "$rg" ]; then print_test INFO " → $rg" fi done else print_test SKIP "Resource groups check (may require Azure CLI authentication)" fi else print_test SKIP "Azure CLI not available" fi # Test 6: Storage Accounts echo "" echo "6. Testing Storage Accounts..." STORAGE_OUTPUT=$(terraform output -json 2>/dev/null | jq -r '.storage_accounts.value.boot_diagnostics // {}' || echo "{}") if [ "$STORAGE_OUTPUT" != "{}" ]; then STORAGE_COUNT=$(echo "$STORAGE_OUTPUT" | jq -r 'length' || echo "0") if [ "$STORAGE_COUNT" -ge 5 ]; then print_test PASS "Boot diagnostics storage accounts: $STORAGE_COUNT" else print_test INFO "Storage accounts found: $STORAGE_COUNT" fi else print_test SKIP "Storage accounts check" fi # Test 7: Key Vault echo "" echo "7. Testing Key Vault..." KEY_VAULT=$(terraform output -json 2>/dev/null | jq -r '.key_vault_name.value // empty' || echo "") if [ -n "$KEY_VAULT" ] && [ "$KEY_VAULT" != "null" ]; then print_test PASS "Key Vault: $KEY_VAULT" if command -v az > /dev/null 2>&1; then if az keyvault show --name "$KEY_VAULT" > /dev/null 2>&1; then print_test PASS "Key Vault accessible via Azure CLI" else print_test SKIP "Key Vault access check (may require authentication)" fi fi else print_test FAIL "Key Vault not found" fi # Test 8: Monitoring Resources echo "" echo "8. Testing Monitoring Resources..." MONITORING=$(terraform output -json 2>/dev/null | jq -r '.monitoring.value // {}' || echo "{}") if [ "$MONITORING" != "{}" ]; then MONITORING_COUNT=$(echo "$MONITORING" | jq -r 'length' || echo "0") if [ "$MONITORING_COUNT" -ge 5 ]; then print_test PASS "Log Analytics Workspaces: $MONITORING_COUNT" else print_test INFO "Monitoring resources found: $MONITORING_COUNT" fi else print_test SKIP "Monitoring resources check" fi # Test 9: Backup Resources echo "" echo "9. Testing Backup Resources..." BACKUPS=$(terraform output -json 2>/dev/null | jq -r '.backups.value // {}' || echo "{}") if [ "$BACKUPS" != "{}" ]; then BACKUP_COUNT=$(echo "$BACKUPS" | jq -r 'length' || echo "0") if [ "$BACKUP_COUNT" -ge 5 ]; then print_test PASS "Recovery Services Vaults: $BACKUP_COUNT" else print_test INFO "Backup resources found: $BACKUP_COUNT" fi else print_test SKIP "Backup resources check" fi # Test 10: Network Connectivity (if accessible) echo "" echo "10. Testing Network Connectivity..." if [ -n "$NGINX_IP" ] && [ "$NGINX_IP" != "null" ]; then # Test internet connectivity from Nginx proxy (if we can reach it) print_test INFO "Network connectivity tests require SSH access to VMs" print_test INFO "Run these tests after SSH access is configured:" print_test INFO " - ping tests between VMs" print_test INFO " - DNS resolution tests" print_test INFO " - Service connectivity tests" fi # Test 11: Cloudflare Credentials echo "" echo "11. Testing Cloudflare Credentials..." ENV_FILE="$PROJECT_ROOT/.env" if [ -f "$ENV_FILE" ]; then if grep -q "CLOUDFLARE_ZONE_ID" "$ENV_FILE" && \ grep -q "CLOUDFLARE_ACCOUNT_ID" "$ENV_FILE" && \ grep -q "CLOUDFLARE_API_TOKEN" "$ENV_FILE"; then print_test PASS "Cloudflare credentials found in .env" # Test API token (if curl available) if command -v curl > /dev/null 2>&1; then source <(grep -v '^#' "$ENV_FILE" | grep CLOUDFLARE_ | xargs) if [ -n "${CLOUDFLARE_API_TOKEN:-}" ] && [ -n "${CLOUDFLARE_ZONE_ID:-}" ]; then API_TEST=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}" \ -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \ -H "Content-Type: application/json" 2>/dev/null | jq -r '.success // false' || echo "false") if [ "$API_TEST" = "true" ]; then print_test PASS "Cloudflare API token valid" else print_test FAIL "Cloudflare API token invalid or insufficient permissions" fi fi fi else print_test FAIL "Cloudflare credentials incomplete in .env" fi else print_test SKIP "Cloudflare credentials (.env file not found)" fi # Test 12: Scripts Availability echo "" echo "12. Testing Scripts Availability..." SCRIPTS=( "setup-cloudflare-tunnel.sh" "setup-besu-node.sh" "update-nginx-backends.sh" "setup-cloudflare-tunnel-backend.sh" "automated-cloudflare-setup.sh" ) for script in "${SCRIPTS[@]}"; do if [ -f "$SCRIPT_DIR/$script" ] && [ -x "$SCRIPT_DIR/$script" ]; then print_test PASS "Script available and executable: $script" else print_test FAIL "Script missing or not executable: $script" fi done # Test 13: Documentation echo "" echo "13. Testing Documentation..." DOCS=( "NEXT_STEPS_COMPLETE.md" "DEPLOYMENT_VERIFICATION.md" "CLOUDFLARE_CREDENTIALS.md" "CLOUDFLARE_AUTOMATION.md" "SETUP_COMPLETE_FINAL.md" ) for doc in "${DOCS[@]}"; do if [ -f "$PHASE1_DIR/$doc" ]; then print_test PASS "Documentation available: $doc" else print_test INFO "Documentation missing: $doc" fi done # Summary echo "" echo "==========================================" echo "Test Summary" echo "==========================================" echo -e "${GREEN}Passed:${NC} $TESTS_PASSED" echo -e "${RED}Failed:${NC} $TESTS_FAILED" echo -e "${YELLOW}Skipped:${NC} $TESTS_SKIPPED" echo "" if [ $TESTS_FAILED -eq 0 ]; then echo -e "${GREEN}✓ All critical tests passed!${NC}" exit 0 else echo -e "${RED}✗ Some tests failed. Review output above.${NC}" exit 1 fi