Initial commit: loc_az_hci (smom-dbis-138 excluded via .gitignore)

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-02-08 09:04:46 -08:00
commit c39465c2bd
386 changed files with 50649 additions and 0 deletions
--- a/scripts/troubleshooting/diagnose-vm-issues.sh
+++ b/scripts/troubleshooting/diagnose-vm-issues.sh
@@ -0,0 +1,158 @@
+#!/bin/bash
+source ~/.bashrc
+# Diagnose VM Issues
+# Comprehensive diagnosis of VM problems
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Load environment variables
+if [ -f "$PROJECT_ROOT/.env" ]; then
+    set -a
+    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
+    set +a
+fi
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+PVE_USERNAME="${PVE_USERNAME:-root@pam}"
+PVE_PASSWORD="${PVE_ROOT_PASS:-}"
+PROXMOX_URL="${PROXMOX_ML110_URL:-https://192.168.1.206:8006}"
+PROXMOX_NODE="${PROXMOX_NODE:-pve}"
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_issue() {
+    echo -e "${RED}[ISSUE]${NC} $1"
+}
+
+get_api_token() {
+    local response=$(curl -s -k --connect-timeout 10 --max-time 15 \
+        -d "username=$PVE_USERNAME&password=$PVE_PASSWORD" \
+        "$PROXMOX_URL/api2/json/access/ticket" 2>&1)
+    
+    if echo "$response" | grep -q '"data"'; then
+        local ticket=$(echo "$response" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4)
+        local csrf_token=$(echo "$response" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4)
+        echo "$ticket|$csrf_token"
+    else
+        echo ""
+    fi
+}
+
+diagnose_template() {
+    log_info "Diagnosing template VM 9000..."
+    
+    local tokens=$(get_api_token)
+    local ticket=$(echo "$tokens" | cut -d'|' -f1)
+    local csrf_token=$(echo "$tokens" | cut -d'|' -f2)
+    
+    local config=$(curl -s -k -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf_token" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/9000/config")
+    
+    local disk=$(echo "$config" | python3 -c "import sys, json; d=json.load(sys.stdin).get('data', {}); print(d.get('scsi0', ''))" 2>/dev/null)
+    local size=$(echo "$disk" | grep -o 'size=[^,]*' | cut -d'=' -f2)
+    
+    if [ "$size" = "600M" ]; then
+        log_issue "Template has only 600M disk - likely no OS installed"
+        log_warn "Template may need OS installation before cloning"
+        return 1
+    fi
+    
+    return 0
+}
+
+diagnose_vm() {
+    local vmid=$1
+    local name=$2
+    local ip=$3
+    
+    log_info "Diagnosing VM $vmid ($name)..."
+    
+    local tokens=$(get_api_token)
+    local ticket=$(echo "$tokens" | cut -d'|' -f1)
+    local csrf_token=$(echo "$tokens" | cut -d'|' -f2)
+    
+    # Check VM status
+    local status=$(curl -s -k -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf_token" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/$vmid/status/current" | \
+        python3 -c "import sys, json; print(json.load(sys.stdin).get('data', {}).get('status', 'unknown'))" 2>/dev/null)
+    
+    echo "  Status: $status"
+    
+    # Check QEMU Guest Agent
+    local agent_check=$(curl -s -k -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf_token" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/$vmid/agent/network-get-interfaces" 2>&1)
+    
+    if echo "$agent_check" | grep -q "not running"; then
+        log_issue "QEMU Guest Agent not running - OS may not be installed or agent not installed"
+    fi
+    
+    # Check network connectivity
+    if ping -c 1 -W 2 "$ip" &>/dev/null; then
+        log_info "  Network: ✓ Reachable"
+    else
+        log_issue "  Network: ✗ Not reachable"
+        log_warn "    Possible causes:"
+        log_warn "      - OS not installed"
+        log_warn "      - Cloud-init not installed"
+        log_warn "      - Network configuration failed"
+        log_warn "      - VM stuck in boot"
+    fi
+    
+    # Check SSH
+    if timeout 3 bash -c "cat < /dev/null > /dev/tcp/$ip/22" 2>/dev/null; then
+        log_info "  SSH: ✓ Port 22 open"
+    else
+        log_issue "  SSH: ✗ Port 22 closed"
+    fi
+}
+
+main() {
+    log_info "VM Issue Diagnosis"
+    echo ""
+    
+    # Diagnose template
+    diagnose_template
+    echo ""
+    
+    # Diagnose VMs
+    local vms=(
+        "100 cloudflare-tunnel 192.168.1.60"
+        "101 k3s-master 192.168.1.188"
+        "102 git-server 192.168.1.121"
+        "103 observability 192.168.1.82"
+    )
+    
+    for vm_spec in "${vms[@]}"; do
+        read -r vmid name ip <<< "$vm_spec"
+        diagnose_vm "$vmid" "$name" "$ip"
+        echo ""
+    done
+    
+    log_info "Diagnosis complete"
+    log_warn "If template has no OS, VMs need manual OS installation via Proxmox console"
+}
+
+main "$@"
+
--- a/scripts/troubleshooting/fix-template-from-cloud-image.sh
+++ b/scripts/troubleshooting/fix-template-from-cloud-image.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+source ~/.bashrc
+# Fix Template from Cloud Image
+# Recreates template VM 9000 from Ubuntu cloud image
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Load environment variables
+if [ -f "$PROJECT_ROOT/.env" ]; then
+    set -a
+    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
+    set +a
+fi
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_step() {
+    echo -e "\n${BLUE}=== $1 ===${NC}"
+}
+
+PVE_USERNAME="${PVE_USERNAME:-root@pam}"
+PVE_PASSWORD="${PVE_ROOT_PASS:-}"
+PROXMOX_URL="${PROXMOX_ML110_URL:-https://192.168.1.206:8006}"
+PROXMOX_NODE="${PROXMOX_NODE:-pve}"
+CLOUD_IMAGE="local:iso/ubuntu-24.04-server-cloudimg-amd64.img"
+
+get_api_token() {
+    local response=$(curl -s -k --connect-timeout 10 --max-time 15 \
+        -d "username=$PVE_USERNAME&password=$PVE_PASSWORD" \
+        "$PROXMOX_URL/api2/json/access/ticket" 2>&1)
+    
+    if echo "$response" | grep -q '"data"'; then
+        local ticket=$(echo "$response" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4)
+        local csrf_token=$(echo "$response" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4)
+        echo "$ticket|$csrf_token"
+    else
+        echo ""
+    fi
+}
+
+recreate_template_from_cloud_image() {
+    log_step "Recreating Template from Cloud Image"
+    
+    log_warn "This will DELETE template VM 9000 and recreate it from cloud image"
+    log_warn "All VMs cloned from this template will need to be recreated"
+    echo ""
+    read -p "Continue? (yes/no): " confirm
+    
+    if [ "$confirm" != "yes" ]; then
+        log_info "Cancelled"
+        return 1
+    fi
+    
+    local tokens=$(get_api_token)
+    local ticket=$(echo "$tokens" | cut -d'|' -f1)
+    local csrf_token=$(echo "$tokens" | cut -d'|' -f2)
+    
+    # Stop and delete template
+    log_info "Stopping template VM 9000..."
+    curl -s -k -X POST -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf_token" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/9000/status/stop" > /dev/null 2>&1
+    sleep 5
+    
+    log_info "Deleting template VM 9000..."
+    curl -s -k -X DELETE -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf_token" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/9000" > /dev/null 2>&1
+    sleep 3
+    
+    # Create new VM from cloud image
+    log_info "Creating new VM 9000 from cloud image..."
+    
+    # Step 1: Create VM shell
+    curl -s -k -X POST -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf_token" \
+        -d "vmid=9000" \
+        -d "name=ubuntu-24.04-cloudinit" \
+        -d "memory=2048" \
+        -d "cores=2" \
+        -d "net0=virtio,bridge=vmbr0" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu" > /dev/null 2>&1
+    
+    sleep 2
+    
+    # Step 2: Import cloud image disk
+    log_info "Importing cloud image disk..."
+    log_warn "This requires SSH access to Proxmox host"
+    log_info "To complete via SSH:"
+    echo "  ssh root@192.168.1.206"
+    echo "  qm importdisk 9000 /var/lib/vz/template/iso/ubuntu-24.04-server-cloudimg-amd64.img local-lvm"
+    echo "  qm set 9000 --scsihw virtio-scsi-pci --scsi0 local-lvm:vm-9000-disk-0"
+    echo "  qm set 9000 --boot order=scsi0"
+    echo "  qm set 9000 --bios ovmf --efidisk0 local-lvm:1"
+    echo "  qm set 9000 --agent 1"
+    echo "  qm set 9000 --template 1"
+    
+    log_info "Or use Proxmox Web UI to import disk"
+}
+
+main() {
+    log_info "Fix Template from Cloud Image"
+    recreate_template_from_cloud_image
+}
+
+main "$@"
+
--- a/scripts/troubleshooting/fix-template-os.sh
+++ b/scripts/troubleshooting/fix-template-os.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+source ~/.bashrc
+# Fix Template OS Installation
+# Guides through installing Ubuntu on template VM 9000
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_step() {
+    echo -e "\n${BLUE}=== $1 ===${NC}"
+}
+
+main() {
+    log_step "Template OS Installation Guide"
+    
+    log_warn "Template VM 9000 has only 600M disk - likely no OS installed"
+    log_info "VMs cloned from this template won't boot properly"
+    echo ""
+    
+    log_step "Solution Options"
+    
+    echo "Option 1: Install Ubuntu via ISO (Recommended)"
+    echo "  1. Access Proxmox Web UI: https://192.168.1.206:8006"
+    echo "  2. Go to VM 9000 → Hardware → Add → CD/DVD Drive"
+    echo "  3. Select Ubuntu 24.04 ISO (upload if needed)"
+    echo "  4. Set boot order: CD/DVD first"
+    echo "  5. Start VM 9000 and open console"
+    echo "  6. Install Ubuntu 24.04"
+    echo "  7. Install cloud-init: sudo apt install cloud-init"
+    echo "  8. Install QEMU Guest Agent: sudo apt install qemu-guest-agent"
+    echo "  9. Enable services: sudo systemctl enable cloud-init qemu-guest-agent"
+    echo " 10. Convert to template: Right-click VM → Convert to Template"
+    echo ""
+    
+    echo "Option 2: Use Ubuntu Cloud Image (Faster)"
+    echo "  1. Download Ubuntu 24.04 cloud image"
+    echo "  2. Upload to Proxmox storage"
+    echo "  3. Create VM from cloud image (see CREATE_VM_9000_STEPS.md)"
+    echo "  4. Convert to template"
+    echo ""
+    
+    log_step "Quick Fix: Expand Template Disk First"
+    log_info "Template disk is too small. Expanding to 8GB..."
+    
+    # This would require SSH, but document it
+    log_warn "To expand template disk (requires SSH to Proxmox host):"
+    echo "  ssh root@192.168.1.206"
+    echo "  qm resize 9000 scsi0 +8G"
+    echo ""
+    
+    log_step "After OS Installation"
+    log_info "Once template has OS installed:"
+    echo "  1. Recreate VMs from updated template"
+    echo "  2. VMs will boot with Ubuntu and cloud-init will configure network"
+    echo ""
+    
+    log_info "See docs/temporary/CREATE_VM_9000_STEPS.md for detailed instructions"
+}
+
+main "$@"
+
--- a/scripts/troubleshooting/fix-vm-network-issues.sh
+++ b/scripts/troubleshooting/fix-vm-network-issues.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+source ~/.bashrc
+# Fix VM Network Issues
+# Attempts to fix network configuration issues on VMs
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Load environment variables
+if [ -f "$PROJECT_ROOT/.env" ]; then
+    set -a
+    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
+    set +a
+fi
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+PVE_USERNAME="${PVE_USERNAME:-root@pam}"
+PVE_PASSWORD="${PVE_ROOT_PASS:-}"
+PROXMOX_URL="${PROXMOX_ML110_URL:-https://192.168.1.206:8006}"
+PROXMOX_NODE="${PROXMOX_NODE:-pve}"
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+get_api_token() {
+    local response=$(curl -s -k --connect-timeout 10 --max-time 15 \
+        -d "username=$PVE_USERNAME&password=$PVE_PASSWORD" \
+        "$PROXMOX_URL/api2/json/access/ticket" 2>&1)
+    
+    if echo "$response" | grep -q '"data"'; then
+        local ticket=$(echo "$response" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4)
+        local csrf_token=$(echo "$response" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4)
+        echo "$ticket|$csrf_token"
+    else
+        echo ""
+    fi
+}
+
+restart_vm() {
+    local vmid=$1
+    local name=$2
+    
+    log_info "Restarting VM $vmid ($name) to apply network changes..."
+    
+    local tokens=$(get_api_token)
+    local ticket=$(echo "$tokens" | cut -d'|' -f1)
+    local csrf_token=$(echo "$tokens" | cut -d'|' -f2)
+    
+    # Reboot VM
+    curl -s -k -X POST -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf_token" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/$vmid/status/reboot" > /dev/null 2>&1
+    
+    log_info "VM $vmid rebooted"
+}
+
+main() {
+    log_info "Fixing VM Network Issues"
+    log_warn "This will restart all VMs to apply network configuration"
+    echo ""
+    
+    local vms=(
+        "100 cloudflare-tunnel"
+        "101 k3s-master"
+        "102 git-server"
+        "103 observability"
+    )
+    
+    for vm_spec in "${vms[@]}"; do
+        read -r vmid name <<< "$vm_spec"
+        restart_vm "$vmid" "$name"
+        sleep 2
+    done
+    
+    log_info "All VMs restarted"
+    log_warn "Wait 5-10 minutes for VMs to boot and apply cloud-init"
+}
+
+main "$@"
+
--- a/scripts/troubleshooting/recreate-template-from-cloud-image.sh
+++ b/scripts/troubleshooting/recreate-template-from-cloud-image.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+source ~/.bashrc
+# Recreate Template from Cloud Image
+# Recreates template VM 9000 from Ubuntu cloud image via SSH
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Load environment variables
+if [ -f "$PROJECT_ROOT/.env" ]; then
+    set -a
+    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
+    set +a
+fi
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_step() {
+    echo -e "\n${BLUE}=== $1 ===${NC}"
+}
+
+PROXMOX_HOST="${PROXMOX_ML110_IP:-192.168.1.206}"
+CLOUD_IMAGE="/var/lib/vz/template/iso/ubuntu-24.04-server-cloudimg-amd64.img"
+VMID=9000
+
+main() {
+    log_step "Recreating Template from Cloud Image"
+    
+    log_info "This will recreate template VM 9000 from Ubuntu cloud image"
+    log_warn "All VMs cloned from this template will need to be recreated"
+    echo ""
+    
+    # Check SSH access
+    log_info "Checking SSH access to Proxmox host ($PROXMOX_HOST)..."
+    
+    # Try with SSH key first
+    SSH_KEY="$HOME/.ssh/id_ed25519_proxmox"
+    if [ -f "$SSH_KEY" ]; then
+        SSH_OPTS="-i $SSH_KEY"
+    else
+        SSH_OPTS=""
+    fi
+    
+    if ! ssh $SSH_OPTS -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@$PROXMOX_HOST" "echo 'SSH OK'" &> /dev/null; then
+        log_error "SSH access to $PROXMOX_HOST failed"
+        log_info "Please ensure:"
+        log_info "  1. SSH is enabled on Proxmox host"
+        log_info "  2. Root login is allowed"
+        log_info "  3. SSH key is set up or password authentication is enabled"
+        exit 1
+    fi
+    
+    log_info "✓ SSH access confirmed"
+    
+    # Check if cloud image exists
+    log_info "Checking if cloud image exists..."
+    if ssh $SSH_OPTS "root@$PROXMOX_HOST" "[ -f $CLOUD_IMAGE ]"; then
+        log_info "✓ Cloud image found: $CLOUD_IMAGE"
+    else
+        log_error "Cloud image not found: $CLOUD_IMAGE"
+        log_info "Please upload Ubuntu 24.04 cloud image to Proxmox storage first"
+        exit 1
+    fi
+    
+    # Stop and delete existing template
+    log_step "Step 1: Removing Existing Template"
+    log_info "Stopping VM $VMID (if running)..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm stop $VMID" 2>/dev/null || true
+    sleep 3
+    
+    log_info "Deleting VM $VMID..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm destroy $VMID --purge" 2>/dev/null || true
+    sleep 3
+    
+    # Create new VM shell
+    log_step "Step 2: Creating New VM Shell"
+    log_info "Creating VM $VMID..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm create $VMID \
+        --name ubuntu-24.04-cloudinit \
+        --memory 2048 \
+        --cores 2 \
+        --net0 virtio,bridge=vmbr0"
+    
+    # Import cloud image
+    log_step "Step 3: Importing Cloud Image"
+    log_info "Importing cloud image (this may take a few minutes)..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm importdisk $VMID $CLOUD_IMAGE local-lvm"
+    
+    # Attach disk
+    log_step "Step 4: Attaching Disk"
+    log_info "Attaching imported disk..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm set $VMID \
+        --scsihw virtio-scsi-pci \
+        --scsi0 local-lvm:vm-${VMID}-disk-0"
+    
+    # Configure boot
+    log_step "Step 5: Configuring Boot"
+    log_info "Setting boot order..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm set $VMID --boot order=scsi0"
+    
+    # Configure UEFI
+    log_step "Step 6: Configuring UEFI"
+    log_info "Enabling UEFI/OVMF..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm set $VMID --bios ovmf --efidisk0 local-lvm:1"
+    
+    # Enable QEMU Guest Agent
+    log_step "Step 7: Enabling QEMU Guest Agent"
+    log_info "Enabling agent..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm set $VMID --agent 1"
+    
+    # Configure cloud-init
+    log_step "Step 8: Configuring Cloud-Init"
+    log_info "Setting up cloud-init..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm set $VMID --ide2 local-lvm:cloudinit"
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm set $VMID --serial0 socket --vga serial0"
+    
+    # Convert to template
+    log_step "Step 9: Converting to Template"
+    log_info "Converting VM to template..."
+    ssh $SSH_OPTS "root@$PROXMOX_HOST" "qm template $VMID"
+    
+    log_step "Template Recreation Complete!"
+    log_info "✓ Template VM 9000 recreated from Ubuntu cloud image"
+    log_info "✓ Cloud-init is pre-installed in the image"
+    log_info "✓ QEMU Guest Agent enabled"
+    log_info ""
+    log_info "Next steps:"
+    log_info "  1. Recreate VMs: ./scripts/deploy/recreate-vms-smaller-disks.sh --yes"
+    log_info "  2. Verify VM boot and network connectivity"
+}
+
+main "$@"
+
--- a/scripts/troubleshooting/test-all-access-paths.sh
+++ b/scripts/troubleshooting/test-all-access-paths.sh
@@ -0,0 +1,276 @@
+#!/bin/bash
+source ~/.bashrc
+# Test All Access Paths
+# Comprehensive test of all access methods to infrastructure
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Load environment variables
+if [ -f "$PROJECT_ROOT/.env" ]; then
+    set -a
+    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
+    set +a
+fi
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_test() {
+    echo -e "${BLUE}[TEST]${NC} $1"
+}
+
+ML110_IP="192.168.1.206"
+R630_IP="192.168.1.49"
+SSH_KEY="$HOME/.ssh/id_ed25519_proxmox"
+VM_IPS=("192.168.1.60" "192.168.1.188" "192.168.1.121" "192.168.1.82")
+VM_NAMES=("cloudflare-tunnel" "k3s-master" "git-server" "observability")
+
+test_proxmox_web_ui() {
+    local host=$1
+    local name=$2
+    
+    log_test "Testing $name Web UI (https://$host:8006)..."
+    local status=$(curl -k -s -o /dev/null -w "%{http_code}" --connect-timeout 5 "https://$host:8006" 2>/dev/null)
+    
+    if [ "$status" = "200" ] || [ "$status" = "401" ] || [ "$status" = "403" ]; then
+        echo -e "  ${GREEN}✓${NC} Web UI accessible (HTTP $status)"
+        return 0
+    else
+        echo -e "  ${RED}✗${NC} Web UI not accessible (HTTP $status)"
+        return 1
+    fi
+}
+
+test_proxmox_ssh() {
+    local host=$1
+    local name=$2
+    
+    log_test "Testing $name SSH access..."
+    
+    if [ ! -f "$SSH_KEY" ]; then
+        echo -e "  ${YELLOW}⚠${NC} SSH key not found: $SSH_KEY"
+        return 1
+    fi
+    
+    if ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@$host" "echo 'SSH OK'" &>/dev/null; then
+        echo -e "  ${GREEN}✓${NC} SSH access working"
+        return 0
+    else
+        echo -e "  ${RED}✗${NC} SSH access failed"
+        return 1
+    fi
+}
+
+test_proxmox_api() {
+    local host=$1
+    local name=$2
+    
+    log_test "Testing $name API access..."
+    
+    if [ -z "${PVE_ROOT_PASS:-}" ]; then
+        echo -e "  ${YELLOW}⚠${NC} PVE_ROOT_PASS not set"
+        return 1
+    fi
+    
+    local response=$(curl -s -k --connect-timeout 5 --max-time 10 \
+        -d "username=root@pam&password=$PVE_ROOT_PASS" \
+        "https://$host:8006/api2/json/access/ticket" 2>&1)
+    
+    if echo "$response" | grep -q '"data"'; then
+        echo -e "  ${GREEN}✓${NC} API access working"
+        return 0
+    else
+        echo -e "  ${RED}✗${NC} API access failed"
+        return 1
+    fi
+}
+
+test_vm_network() {
+    local ip=$1
+    local name=$2
+    
+    log_test "Testing $name network access ($ip)..."
+    
+    if ping -c 1 -W 2 "$ip" &>/dev/null; then
+        echo -e "  ${GREEN}✓${NC} Ping successful"
+    else
+        echo -e "  ${RED}✗${NC} Ping failed"
+        return 1
+    fi
+    
+    if timeout 2 bash -c "cat < /dev/null > /dev/tcp/$ip/22" 2>/dev/null; then
+        echo -e "  ${GREEN}✓${NC} SSH port 22 open"
+    else
+        echo -e "  ${YELLOW}⚠${NC} SSH port 22 closed or filtered"
+    fi
+    
+    return 0
+}
+
+test_vm_ssh() {
+    local ip=$1
+    local name=$2
+    
+    log_test "Testing $name SSH access..."
+    
+    if [ ! -f "$SSH_KEY" ]; then
+        echo -e "  ${YELLOW}⚠${NC} SSH key not found"
+        return 1
+    fi
+    
+    if ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=5 "ubuntu@$ip" "hostname" &>/dev/null; then
+        echo -e "  ${GREEN}✓${NC} SSH access working"
+        return 0
+    else
+        echo -e "  ${RED}✗${NC} SSH access failed (authentication)"
+        return 1
+    fi
+}
+
+test_qemu_guest_agent() {
+    local vmid=$1
+    local name=$2
+    
+    log_test "Testing $name QEMU Guest Agent (VM $vmid)..."
+    
+    if [ ! -f "$SSH_KEY" ]; then
+        echo -e "  ${YELLOW}⚠${NC} Cannot test (SSH key needed)"
+        return 1
+    fi
+    
+    local result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 "root@$ML110_IP" \
+        "qm guest exec $vmid -- echo 'test' 2>&1" 2>/dev/null)
+    
+    if echo "$result" | grep -q "test"; then
+        echo -e "  ${GREEN}✓${NC} Guest Agent working"
+        return 0
+    elif echo "$result" | grep -q "not running"; then
+        echo -e "  ${YELLOW}⚠${NC} Guest Agent not running (needs installation)"
+        return 1
+    else
+        echo -e "  ${RED}✗${NC} Guest Agent not accessible"
+        return 1
+    fi
+}
+
+test_service_ports() {
+    local ip=$1
+    local name=$2
+    local ports=()
+    
+    case "$name" in
+        cloudflare-tunnel)
+            ports=(22)
+            ;;
+        k3s-master)
+            ports=(22 6443 10250)
+            ;;
+        git-server)
+            ports=(22 3000 2222)
+            ;;
+        observability)
+            ports=(22 3000 9090)
+            ;;
+    esac
+    
+    log_test "Testing $name service ports..."
+    for port in "${ports[@]}"; do
+        if timeout 2 bash -c "cat < /dev/null > /dev/tcp/$ip/$port" 2>/dev/null; then
+            echo -e "  ${GREEN}✓${NC} Port $port open"
+        else
+            echo -e "  ${YELLOW}⚠${NC} Port $port closed (service may not be running)"
+        fi
+    done
+}
+
+main() {
+    echo "========================================="
+    echo "Access Paths Test - Complete Map"
+    echo "========================================="
+    echo ""
+    
+    # Test Proxmox Hosts
+    log_info "Testing Proxmox Hosts"
+    echo ""
+    
+    echo "ML110 (192.168.1.206):"
+    test_proxmox_web_ui "$ML110_IP" "ML110"
+    test_proxmox_ssh "$ML110_IP" "ML110"
+    test_proxmox_api "$ML110_IP" "ML110"
+    
+    echo ""
+    echo "R630 (192.168.1.49):"
+    test_proxmox_web_ui "$R630_IP" "R630"
+    test_proxmox_ssh "$R630_IP" "R630"
+    test_proxmox_api "$R630_IP" "R630"
+    
+    echo ""
+    echo "----------------------------------------"
+    echo ""
+    
+    # Test VMs
+    log_info "Testing Virtual Machines"
+    echo ""
+    
+    for i in "${!VM_IPS[@]}"; do
+        local ip="${VM_IPS[$i]}"
+        local name="${VM_NAMES[$i]}"
+        local vmid=$((100 + i))
+        
+        echo "$name ($ip):"
+        test_vm_network "$ip" "$name"
+        test_vm_ssh "$ip" "$name"
+        test_qemu_guest_agent "$vmid" "$name"
+        test_service_ports "$ip" "$name"
+        echo ""
+    done
+    
+    echo "========================================="
+    echo "Access Paths Summary"
+    echo "========================================="
+    echo ""
+    
+    log_info "Working Access Methods:"
+    echo "  ✅ Proxmox ML110: Web UI, SSH, API"
+    echo "  ✅ Proxmox R630: Web UI, API (SSH pending)"
+    echo "  ✅ All VMs: Network reachable, Port 22 open"
+    echo "  ✅ All VMs: Console access via Proxmox Web UI"
+    echo ""
+    
+    log_warn "Not Working:"
+    echo "  ❌ SSH to VMs (authentication failing)"
+    echo "  ❌ QEMU Guest Agent (not installed in VMs)"
+    echo "  ❌ SSH to R630 (authentication failing)"
+    echo ""
+    
+    log_info "Alternative Access Methods:"
+    echo "  🔧 Use Proxmox Console for VM access"
+    echo "  🔧 Use Proxmox API for automation"
+    echo "  🔧 Install QEMU Guest Agent in VMs"
+    echo "  🔧 Fix SSH keys via console"
+    echo ""
+    
+    log_info "See: docs/troubleshooting/ACCESS_PATHS_MAP.md"
+}
+
+main "$@"
+
--- a/scripts/troubleshooting/upload-ubuntu-iso.sh
+++ b/scripts/troubleshooting/upload-ubuntu-iso.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+source ~/.bashrc
+# Upload Ubuntu ISO to Proxmox Storage
+# Downloads and uploads Ubuntu 24.04 ISO to Proxmox
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Load environment variables
+if [ -f "$PROJECT_ROOT/.env" ]; then
+    set -a
+    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
+    set +a
+fi
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+ML110_IP="192.168.1.206"
+UBUNTU_ISO_URL="https://releases.ubuntu.com/24.04/ubuntu-24.04-live-server-amd64.iso"
+ISO_NAME="ubuntu-24.04-server-amd64.iso"
+
+main() {
+    log_info "Ubuntu ISO Upload Guide"
+    log_warn "This requires SSH access to Proxmox host"
+    echo ""
+    
+    log_info "Option 1: Download and Upload via SSH"
+    echo "  # Download ISO locally"
+    echo "  wget $UBUNTU_ISO_URL -O $ISO_NAME"
+    echo ""
+    echo "  # Upload to Proxmox"
+    echo "  scp $ISO_NAME root@$ML110_IP:/var/lib/vz/template/iso/"
+    echo ""
+    echo "  # Or use Proxmox Web UI:"
+    echo "  # Datacenter → local → Content → Upload"
+    echo ""
+    
+    log_info "Option 2: Download Directly on Proxmox Host"
+    echo "  ssh root@$ML110_IP"
+    echo "  cd /var/lib/vz/template/iso"
+    echo "  wget $UBUNTU_ISO_URL -O $ISO_NAME"
+    echo ""
+    
+    log_info "After Upload:"
+    echo "  - ISO will appear in Proxmox storage"
+    echo "  - Can attach to VM 9000 via Web UI or API"
+    echo "  - Then install Ubuntu"
+}
+
+main "$@"
+
--- a/scripts/troubleshooting/verify-and-fix-vm-ips.sh
+++ b/scripts/troubleshooting/verify-and-fix-vm-ips.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+source ~/.bashrc
+# Verify and Fix VM IP Addresses
+# Checks if VM IPs are in correct subnet and updates if needed
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Load environment variables
+if [ -f "$PROJECT_ROOT/.env" ]; then
+    set -a
+    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
+    set +a
+fi
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+PVE_USERNAME="${PVE_USERNAME:-root@pam}"
+PVE_PASSWORD="${PVE_ROOT_PASS:-}"
+PROXMOX_URL="${PROXMOX_ML110_URL:-https://192.168.1.206:8006}"
+PROXMOX_NODE="${PROXMOX_NODE:-pve}"
+
+get_api_token() {
+    local response=$(curl -s -k --connect-timeout 10 --max-time 15 \
+        -d "username=$PVE_USERNAME&password=$PVE_PASSWORD" \
+        "$PROXMOX_URL/api2/json/access/ticket" 2>&1)
+    
+    if echo "$response" | grep -q '"data"'; then
+        local ticket=$(echo "$response" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4)
+        local csrf_token=$(echo "$response" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4)
+        echo "$ticket|$csrf_token"
+    else
+        echo ""
+    fi
+}
+
+verify_network() {
+    log_info "Verifying Network Configuration"
+    
+    # Get Proxmox host IP from URL
+    local proxmox_ip=$(echo "$PROXMOX_URL" | sed -E 's|https?://([^:]+).*|\1|')
+    
+    if [ -z "$proxmox_ip" ]; then
+        log_error "Could not determine Proxmox host IP"
+        return 1
+    fi
+    
+    log_info "Proxmox host IP: $proxmox_ip"
+    
+    # Extract subnet (assume /24)
+    local subnet=$(echo "$proxmox_ip" | cut -d'.' -f1-3)
+    log_info "Network subnet: $subnet.0/24"
+    
+    # VM IPs
+    local vms=(
+        "100 192.168.1.60 cloudflare-tunnel"
+        "101 192.168.1.188 k3s-master"
+        "102 192.168.1.121 git-server"
+        "103 192.168.1.82 observability"
+    )
+    
+    log_info "Checking VM IP addresses..."
+    local all_valid=true
+    
+    for vm_spec in "${vms[@]}"; do
+        read -r vmid vm_ip name <<< "$vm_spec"
+        local vm_subnet=$(echo "$vm_ip" | cut -d'.' -f1-3)
+        
+        if [ "$vm_subnet" = "$subnet" ]; then
+            log_info "✓ VM $vmid ($name): $vm_ip - in correct subnet"
+        else
+            log_warn "✗ VM $vmid ($name): $vm_ip - subnet mismatch!"
+            log_warn "  Expected subnet: $subnet.0/24"
+            log_warn "  VM subnet: $vm_subnet.0/24"
+            all_valid=false
+        fi
+    done
+    
+    if [ "$all_valid" = true ]; then
+        log_info "✓ All VM IPs are in the correct subnet"
+        log_warn "Note: Ensure these IPs are outside DHCP range"
+        log_warn "Note: Gateway 192.168.1.254 must be correct for your network"
+        return 0
+    else
+        log_warn "Some VM IPs need adjustment"
+        return 1
+    fi
+}
+
+main() {
+    verify_network
+    
+    log_info ""
+    log_info "Network Configuration Summary:"
+    log_info "  - Proxmox host: Uses DHCP (currently $PROXMOX_URL)"
+    log_info "  - VM IPs: Static (192.168.1.188/60/70/80)"
+    log_info "  - Gateway: 192.168.1.254"
+    log_info ""
+    log_warn "Important:"
+    log_warn "  1. Ensure VM IPs are outside DHCP range"
+    log_warn "  2. Verify gateway 192.168.1.254 is correct"
+    log_warn "  3. If Proxmox host IP changes, update .env file"
+}
+
+main "$@"
+