Initial commit: loc_az_hci (smom-dbis-138 excluded via .gitignore)

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-02-08 09:04:46 -08:00
commit c39465c2bd
386 changed files with 50649 additions and 0 deletions
--- a/scripts/vm-management/monitor/check-and-recreate.sh
+++ b/scripts/vm-management/monitor/check-and-recreate.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+source ~/.bashrc
+# Check Template Status and Guide Recreation
+# This script checks if template exists and guides the process
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_step() {
+    echo -e "${BLUE}[STEP]${NC} $1"
+}
+
+log_header() {
+    echo -e "${CYAN}========================================${NC}"
+    echo -e "${CYAN}$1${NC}"
+    echo -e "${CYAN}========================================${NC}"
+}
+
+# Load environment variables
+if [ -f .env ]; then
+    set -a
+    source <(grep -v '^#' .env | grep -v '^$' | sed 's/#.*$//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | grep '=')
+    set +a
+fi
+
+PVE_USERNAME="${PVE_USERNAME:-root@pam}"
+PVE_PASSWORD="${PVE_ROOT_PASS:-}"
+PROXMOX_URL="https://192.168.1.206:8006"
+PROXMOX_NODE="pve"
+TEMPLATE_ID=9000
+
+# Check if template exists
+check_template() {
+    local response=$(curl -k -s -d "username=$PVE_USERNAME&password=$PVE_PASSWORD" \
+        "$PROXMOX_URL/api2/json/access/ticket" 2>/dev/null)
+    
+    local ticket=$(echo "$response" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4)
+    local csrf=$(echo "$response" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4)
+    
+    if [ -z "$ticket" ] || [ -z "$csrf" ]; then
+        return 1
+    fi
+    
+    local config=$(curl -k -s \
+        -H "Cookie: PVEAuthCookie=$ticket" \
+        -H "CSRFPreventionToken: $csrf" \
+        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/$TEMPLATE_ID/config" 2>&1)
+    
+    if echo "$config" | grep -q '"name"'; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+main() {
+    log_header "Template Status Check"
+    echo ""
+    
+    if [ -z "$PVE_PASSWORD" ]; then
+        log_error "PVE_ROOT_PASS not set in .env"
+        exit 1
+    fi
+    
+    log_step "Checking if template $TEMPLATE_ID exists..."
+    
+    if check_template; then
+        log_info "✓ Template $TEMPLATE_ID exists!"
+        echo ""
+        log_step "Template is ready. Proceeding with VM recreation..."
+        echo ""
+        
+        # Run recreation script
+        export SSH_KEY="$HOME/.ssh/id_rsa"
+        export SSH_USER="ubuntu"
+        ./scripts/recreate-vms-from-template.sh
+        
+    else
+        log_warn "Template $TEMPLATE_ID does not exist yet"
+        echo ""
+        log_info "You need to create the template first:"
+        echo ""
+        log_step "Quick Steps:"
+        echo "  1. Upload cloud image to Proxmox:"
+        echo "     • Proxmox Web UI → Storage → local → Upload"
+        echo "     • File: ./downloads/ubuntu-24.04-server-cloudimg-amd64.img"
+        echo ""
+        echo "  2. Create VM 9000 from image:"
+        echo "     • Create VM (ID: 9000, Name: ubuntu-24.04-cloudinit)"
+        echo "     • Import disk from uploaded image"
+        echo "     • Configure Cloud-Init with SSH key"
+        echo ""
+        echo "  3. Convert to template:"
+        echo "     • Right-click VM 9000 → Convert to Template"
+        echo ""
+        log_info "See: QUICK_TEMPLATE_GUIDE.md for detailed instructions"
+        echo ""
+        log_info "After creating template, run this script again:"
+        echo "  ./scripts/check-and-recreate.sh"
+        echo ""
+        log_info "Or run directly:"
+        echo "  ./scripts/recreate-vms-from-template.sh"
+    fi
+}
+
+main "$@"
+
--- a/scripts/vm-management/monitor/check-vm-disk-sizes.sh
+++ b/scripts/vm-management/monitor/check-vm-disk-sizes.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+source ~/.bashrc
+# Check VM Disk Sizes and Configuration
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_step() {
+    echo -e "${BLUE}[STEP]${NC} $1"
+}
+
+# Load environment variables
+if [ -f .env ]; then
+    set -a
+    source <(grep -v '^#' .env | grep -v '^$' | sed 's/#.*$//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | grep '=')
+    set +a
+fi
+
+PVE_USERNAME="${PVE_USERNAME:-root@pam}"
+PVE_PASSWORD="${PVE_ROOT_PASS:-}"
+PROXMOX_URL="https://192.168.1.206:8006"
+PROXMOX_NODE="pve"
+
+main() {
+    echo "========================================="
+    echo "VM Disk Size Configuration"
+    echo "========================================="
+    echo ""
+    
+    if [ -z "$PVE_PASSWORD" ]; then
+        echo "Error: PVE_ROOT_PASS not set in .env"
+        exit 1
+    fi
+    
+    # Get authentication ticket
+    local response=$(curl -k -s -d "username=$PVE_USERNAME&password=$PVE_PASSWORD" \
+        "$PROXMOX_URL/api2/json/access/ticket")
+    
+    local ticket=$(echo "$response" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4)
+    local csrf=$(echo "$response" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4)
+    
+    if [ -z "$ticket" ] || [ -z "$csrf" ]; then
+        echo "Error: Failed to authenticate"
+        exit 1
+    fi
+    
+    echo "VM Disk Configuration:"
+    echo ""
+    
+    declare -A VMS=(
+        [100]="cloudflare-tunnel:40G"
+        [101]="k3s-master:80G"
+        [102]="git-server:100G"
+        [103]="observability:200G"
+    )
+    
+    for vmid in 100 101 102 103; do
+        IFS=':' read -r name expected_size <<< "${VMS[$vmid]}"
+        
+        local config=$(curl -k -s \
+            -H "Cookie: PVEAuthCookie=$ticket" \
+            -H "CSRFPreventionToken: $csrf" \
+            "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/$vmid/config")
+        
+        local scsi0=$(echo "$config" | grep -o '"scsi0":"[^"]*' | cut -d'"' -f4)
+        local actual_size=$(echo "$scsi0" | grep -o 'size=[0-9]*G' | cut -d'=' -f2 || echo "Unknown")
+        
+        echo "VM $vmid - $name:"
+        echo "  Expected: $expected_size"
+        echo "  Actual:   $actual_size"
+        echo "  Device:   $scsi0"
+        
+        if [ "$actual_size" = "$expected_size" ]; then
+            log_info "  ✓ Disk size matches"
+        else
+            echo "  ⚠ Size mismatch or not found"
+        fi
+        echo ""
+    done
+    
+    echo "========================================="
+    echo "Installation Tips:"
+    echo "========================================="
+    echo ""
+    echo "During Ubuntu installation:"
+    echo "  • Select 'Custom storage layout'"
+    echo "  • Choose the disk matching your VM size"
+    echo "  • Ignore the CD-ROM (ISO, ~3GB)"
+    echo "  • Use entire disk or create partitions"
+    echo ""
+    echo "See: UBUNTU_INSTALL_DISK_SELECTION.md for details"
+}
+
+main "$@"
+
--- a/scripts/vm-management/monitor/check-vm-readiness.sh
+++ b/scripts/vm-management/monitor/check-vm-readiness.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+source ~/.bashrc
+# Check VM Readiness - Verify VMs are ready for SSH and task execution
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+declare -A VMS=(
+    [100]="cloudflare-tunnel:192.168.1.60"
+    [101]="k3s-master:192.168.1.188"
+    [102]="git-server:192.168.1.121"
+    [103]="observability:192.168.1.82"
+)
+
+SSH_USER="${SSH_USER:-ubuntu}"
+SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_rsa}"
+
+check_vm() {
+    local vmid=$1
+    local name=$2
+    local ip=$3
+    
+    echo -n "VM $vmid ($name) at $ip: "
+    
+    # Check ping
+    if ping -c 1 -W 2 "$ip" > /dev/null 2>&1; then
+        echo -n "✓ Reachable, "
+        
+        # Check SSH
+        if ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no -i "$SSH_KEY" "${SSH_USER}@${ip}" "echo 'OK'" > /dev/null 2>&1; then
+            echo "✓ SSH Ready"
+            return 0
+        else
+            echo "⚠ SSH Not Ready"
+            return 1
+        fi
+    else
+        echo "✗ Not Reachable"
+        return 1
+    fi
+}
+
+main() {
+    echo "========================================="
+    echo "VM Readiness Check"
+    echo "========================================="
+    echo ""
+    
+    if [ ! -f "$SSH_KEY" ]; then
+        log_error "SSH key not found: $SSH_KEY"
+        log_info "Available keys:"
+        ls -1 ~/.ssh/id_* 2>/dev/null | grep -v ".pub" || echo "  None found"
+        exit 1
+    fi
+    
+    log_info "Using SSH key: $SSH_KEY"
+    echo ""
+    
+    local all_ready=true
+    
+    for vmid in 100 101 102 103; do
+        IFS=':' read -r name ip <<< "${VMS[$vmid]}"
+        if ! check_vm "$vmid" "$name" "$ip"; then
+            all_ready=false
+        fi
+    done
+    
+    echo ""
+    if [ "$all_ready" = true ]; then
+        log_info "✓ All VMs are ready!"
+        log_info "You can now run: ./scripts/complete-all-vm-tasks.sh"
+    else
+        log_warn "⚠ Some VMs are not ready yet"
+        log_info "Wait for Ubuntu installation to complete"
+        log_info "Then run this script again to check readiness"
+    fi
+}
+
+main "$@"
+
--- a/scripts/vm-management/monitor/check-vm-status.sh
+++ b/scripts/vm-management/monitor/check-vm-status.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+source ~/.bashrc
+# Check VM Status and Verify Prerequisites Before Next Steps
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_step() {
+    echo -e "${BLUE}[STEP]${NC} $1"
+}
+
+log_header() {
+    echo -e "${CYAN}========================================${NC}"
+    echo -e "${CYAN}$1${NC}"
+    echo -e "${CYAN}========================================${NC}"
+}
+
+# Check VM connectivity
+check_vm_connectivity() {
+    local ip=$1
+    local name=$2
+    
+    log_info "Checking $name ($ip)..."
+    
+    # Ping test
+    if ping -c 1 -W 2 "$ip" >/dev/null 2>&1; then
+        log_info "✓ $name is reachable"
+        
+        # Check if SSH is available
+        if timeout 2 bash -c "echo >/dev/tcp/$ip/22" 2>/dev/null; then
+            log_info "✓ SSH port (22) is open"
+            
+            # Try to check if Ubuntu is installed
+            if ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no "ubuntu@$ip" "lsb_release -d 2>/dev/null || echo 'OS check failed'" 2>/dev/null | grep -q "Ubuntu"; then
+                log_info "✓ Ubuntu is installed"
+                return 0
+            else
+                log_warn "✗ Ubuntu installation not verified (may need manual check)"
+                return 1
+            fi
+        else
+            log_warn "✗ SSH not available yet (OS may still be installing)"
+            return 1
+        fi
+    else
+        log_warn "✗ $name is not reachable"
+        return 1
+    fi
+}
+
+# VM configurations
+declare -A VMS=(
+    ["100"]="cloudflare-tunnel:192.168.1.60:scripts/setup-cloudflare-tunnel.sh"
+    ["101"]="k3s-master:192.168.1.188:scripts/setup-k3s.sh"
+    ["102"]="git-server:192.168.1.121:scripts/setup-git-server.sh"
+    ["103"]="observability:192.168.1.82:scripts/setup-observability.sh"
+)
+
+main() {
+    log_header "VM Status Check - Prerequisites Verification"
+    echo ""
+    
+    log_step "Checking VM Connectivity and OS Installation"
+    echo ""
+    
+    local all_ready=true
+    
+    for vmid in "${!VMS[@]}"; do
+        IFS=':' read -r name ip script <<< "${VMS[$vmid]}"
+        echo "--- $name (ID: $vmid) ---"
+        
+        if check_vm_connectivity "$ip" "$name"; then
+            log_info "✓ $name is ready for setup"
+        else
+            log_warn "✗ $name is not ready yet"
+            all_ready=false
+        fi
+        echo ""
+    done
+    
+    log_header "Status Summary"
+    echo ""
+    
+    if [ "$all_ready" = true ]; then
+        log_info "✅ All VMs are ready for setup scripts!"
+        echo ""
+        log_info "Next: Run setup scripts on each VM:"
+        for vmid in "${!VMS[@]}"; do
+            IFS=':' read -r name ip script <<< "${VMS[$vmid]}"
+            echo "  - $name: ssh ubuntu@$ip 'sudo bash $script'"
+        done
+    else
+        log_warn "⚠️  Some VMs are not ready yet"
+        echo ""
+        log_info "Please complete Ubuntu installation on all VMs first:"
+        echo "  1. Access Proxmox Web UI: https://192.168.1.206:8006"
+        echo "  2. Open console for each VM"
+        echo "  3. Complete Ubuntu 24.04 installation"
+        echo "  4. Configure static IP addresses"
+        echo "  5. Run this script again to verify"
+    fi
+    
+    echo ""
+}
+
+main "$@"
+
--- a/scripts/vm-management/monitor/monitor-and-complete.sh
+++ b/scripts/vm-management/monitor/monitor-and-complete.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+source ~/.bashrc
+# Monitor VMs and Automatically Complete Tasks When Ready
+# This script continuously checks VM readiness and runs complete-all-vm-tasks.sh when ready
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+log_step() {
+    echo -e "${BLUE}[STEP]${NC} $1"
+}
+
+declare -A VMS=(
+    [100]="cloudflare-tunnel:192.168.1.60"
+    [101]="k3s-master:192.168.1.188"
+    [102]="git-server:192.168.1.121"
+    [103]="observability:192.168.1.82"
+)
+
+SSH_USER="${SSH_USER:-ubuntu}"
+SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_rsa}"
+CHECK_INTERVAL=30  # Check every 30 seconds
+MAX_WAIT=3600      # Maximum wait time: 1 hour
+
+check_all_vms_ready() {
+    local all_ready=true
+    
+    for vmid in 100 101 102 103; do
+        IFS=':' read -r name ip <<< "${VMS[$vmid]}"
+        
+        # Check ping
+        if ! ping -c 1 -W 2 "$ip" > /dev/null 2>&1; then
+            all_ready=false
+            return 1
+        fi
+        
+        # Check SSH
+        if ! ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no -i "$SSH_KEY" "${SSH_USER}@${ip}" "echo 'OK'" > /dev/null 2>&1; then
+            all_ready=false
+            return 1
+        fi
+    done
+    
+    return 0
+}
+
+main() {
+    echo "========================================="
+    echo "VM Monitor - Auto-Complete Tasks"
+    echo "========================================="
+    echo ""
+    log_info "Monitoring VMs for readiness..."
+    log_info "Will automatically run tasks when all VMs are ready"
+    log_info "Checking every $CHECK_INTERVAL seconds"
+    log_info "Maximum wait: $MAX_WAIT seconds (1 hour)"
+    echo ""
+    
+    if [ ! -f "$SSH_KEY" ]; then
+        log_error "SSH key not found: $SSH_KEY"
+        exit 1
+    fi
+    
+    local start_time=$(date +%s)
+    local check_count=0
+    
+    while true; do
+        check_count=$((check_count + 1))
+        local elapsed=$(($(date +%s) - start_time))
+        
+        if [ $elapsed -gt $MAX_WAIT ]; then
+            log_error "Maximum wait time exceeded"
+            exit 1
+        fi
+        
+        echo -n "[Check $check_count] $(date '+%H:%M:%S') - "
+        
+        if check_all_vms_ready; then
+            echo ""
+            log_info "✓ All VMs are ready!"
+            echo ""
+            log_step "Running complete-all-vm-tasks.sh..."
+            echo ""
+            
+            export SSH_KEY="$SSH_KEY"
+            export SSH_USER="$SSH_USER"
+            ./scripts/complete-all-vm-tasks.sh
+            
+            exit $?
+        else
+            echo "VMs not ready yet... (elapsed: ${elapsed}s)"
+            sleep $CHECK_INTERVAL
+        fi
+    done
+}
+
+main "$@"
+