Add Oracle Aggregator and CCIP Integration

- Introduced Aggregator.sol for Chainlink-compatible oracle functionality, including round-based updates and access control. - Added OracleWithCCIP.sol to extend Aggregator with CCIP cross-chain messaging capabilities. - Created .gitmodules to include OpenZeppelin contracts as a submodule. - Developed a comprehensive deployment guide in NEXT_STEPS_COMPLETE_GUIDE.md for Phase 2 and smart contract deployment. - Implemented Vite configuration for the orchestration portal, supporting both Vue and React frameworks. - Added server-side logic for the Multi-Cloud Orchestration Portal, including API endpoints for environment management and monitoring. - Created scripts for resource import and usage validation across non-US regions. - Added tests for CCIP error handling and integration to ensure robust functionality. - Included various new files and directories for the orchestration portal and deployment scripts.
2025-12-12 14:57:48 -08:00
parent a1466e4005
commit 1fb7266469
1720 changed files with 241279 additions and 16 deletions
--- a/scripts/key-management/azure-keyvault-setup.sh
+++ b/scripts/key-management/azure-keyvault-setup.sh
@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Setup Azure Key Vault for key storage
+# This script creates an Azure Key Vault and sets up access policies
+#
+# NOTE: For production, consider using the enhanced Key Vault module with RBAC
+# See terraform/modules/keyvault-enhanced/ for Well-Architected Framework implementation
+# See docs/AZURE_WELL_ARCHITECTED_REVIEW.md for details
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+SCRIPT_NAME="azure-keyvault-setup.sh"
+SCRIPT_DESC="Set up Azure Key Vaults with RBAC/access policies and AKS managed identity access"
+SCRIPT_USAGE="${SCRIPT_NAME} [--region <name>] [--dry-run] [--help]"
+SCRIPT_OPTIONS="--region <name>   Limit setup to a specific region\n--dry-run         Print actions without executing\n--help            Show help"
+SCRIPT_REQUIREMENTS="Azure CLI (ensure_azure_cli), permissions to manage Key Vaults"
+handle_help "${1:-}"
+
+# Initialize
+SUBSCRIPTION_ID="$(get_subscription_id)"
+ensure_azure_cli || exit 1
+set_subscription "$SUBSCRIPTION_ID" || true
+
+# Configuration
+ENVIRONMENT="${ENVIRONMENT:-prod}"
+RESOURCE_GROUP="${RESOURCE_GROUP:-rg-${ENVIRONMENT}-security-001}"
+KEY_VAULT_NAME="${KEY_VAULT_NAME:-kv-${ENVIRONMENT}-secrets-001}"
+LOCATION="${LOCATION:-westeurope}"
+USE_RBAC="${USE_RBAC:-false}"  # Set to true to use RBAC instead of access policies
+
+log_section "SETTING UP AZURE KEY VAULT"
+log_info "Vault: $KEY_VAULT_NAME"
+
+# CLI and login ensured by library above
+
+# Create resource group if it doesn't exist
+az group create --name "$RESOURCE_GROUP" --location "$LOCATION" || true
+
+# Create Key Vault
+log_info "Creating Key Vault: $KEY_VAULT_NAME in resource group: $RESOURCE_GROUP"
+
+az keyvault create \
+    --name "$KEY_VAULT_NAME" \
+    --resource-group "$RESOURCE_GROUP" \
+    --location "$LOCATION" \
+    --enable-soft-delete true \
+    --enable-purge-protection $([ "$ENVIRONMENT" == "prod" ] && echo "true" || echo "false") \
+    --retention-days $([ "$ENVIRONMENT" == "prod" ] && echo "90" || echo "7") \
+    --sku standard
+
+# Configure network access (restrict in production)
+if [ "$ENVIRONMENT" == "prod" ]; then
+    log_info "Configuring network restrictions for production..."
+    # Default action: Deny (restrict access)
+    az keyvault update \
+        --name "$KEY_VAULT_NAME" \
+        --resource-group "$RESOURCE_GROUP" \
+        --default-action Deny \
+        --bypass AzureServices
+else
+    log_info "Using permissive network access for non-production environment..."
+    # Default action: Allow (permissive for dev/test)
+    az keyvault update \
+        --name "$KEY_VAULT_NAME" \
+        --resource-group "$RESOURCE_GROUP" \
+        --default-action Allow \
+        --bypass AzureServices
+fi
+
+# Configure access (RBAC or Access Policies)
+if [ "$USE_RBAC" == "true" ]; then
+    log_info "Enabling RBAC authorization..."
+    az keyvault update \
+        --name "$KEY_VAULT_NAME" \
+        --resource-group "$RESOURCE_GROUP" \
+        --enable-rbac-authorization true
+    
+    # Get current user object ID
+    CURRENT_USER_OBJECT_ID=$(az ad signed-in-user show --query id -o tsv)
+    
+    # Assign Key Vault Administrator role
+    az role assignment create \
+        --role "Key Vault Administrator" \
+        --assignee "$CURRENT_USER_OBJECT_ID" \
+        --scope "/subscriptions/$(az account show --query id -o tsv)/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.KeyVault/vaults/$KEY_VAULT_NAME"
+else
+    log_info "Using access policies (legacy method)..."
+    # Get current user principal
+    CURRENT_USER=$(az account show --query user.name -o tsv)
+    
+    # Set access policy for current user
+    az keyvault set-policy \
+        --name "$KEY_VAULT_NAME" \
+        --upn "$CURRENT_USER" \
+        --secret-permissions get list set delete \
+        --key-permissions get list create import
+fi
+
+# Get AKS managed identity (if AKS exists)
+AKS_CLUSTER_NAME="${AKS_CLUSTER_NAME:-defi-oracle-aks}"
+AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-$RESOURCE_GROUP}"
+
+if az aks show --name "$AKS_CLUSTER_NAME" --resource-group "$AKS_RESOURCE_GROUP" &> /dev/null; then
+    log_info "Configuring AKS managed identity access..."
+    
+    # Get AKS node resource group
+    NODE_RESOURCE_GROUP=$(az aks show \
+        --name "$AKS_CLUSTER_NAME" \
+        --resource-group "$AKS_RESOURCE_GROUP" \
+        --query nodeResourceGroup -o tsv)
+    
+    # Get AKS managed identity
+    AKS_IDENTITY_ID=$(az aks show \
+        --name "$AKS_CLUSTER_NAME" \
+        --resource-group "$AKS_RESOURCE_GROUP" \
+        --query identity.principalId -o tsv)
+    
+    if [ -n "$AKS_IDENTITY_ID" ]; then
+        az keyvault set-policy \
+            --name "$KEY_VAULT_NAME" \
+            --object-id "$AKS_IDENTITY_ID" \
+            --secret-permissions get list \
+            --key-permissions get list
+    fi
+fi
+
+log_success "Azure Key Vault setup complete!"
+echo "Key Vault Name: $KEY_VAULT_NAME"
+echo "Resource Group: $RESOURCE_GROUP"
+echo "Location: $LOCATION"
+echo ""
+echo "To store a key:"
+echo "  az keyvault secret set --vault-name $KEY_VAULT_NAME --name validator-key-1 --value <key>"
+echo ""
+echo "To list all secrets:"
+echo "  az keyvault secret list --vault-name $KEY_VAULT_NAME"
+echo ""
+echo "To retrieve a secret:"
+echo "  az keyvault secret show --vault-name $KEY_VAULT_NAME --name validator-key-1 --query value -o tsv"
+
--- a/scripts/key-management/check-keyvault-status.sh
+++ b/scripts/key-management/check-keyvault-status.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# Check Key Vault deployment status
+# REFACTORED - Uses common libraries
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+
+# Initialize
+SUBSCRIPTION_ID="$(get_subscription_id)"
+ensure_azure_cli || exit 1
+set_subscription "$SUBSCRIPTION_ID" || true
+
+log_section "CHECKING KEY VAULT DEPLOYMENT STATUS (36 REGIONS)"
+
+log_info "Subscription: $SUBSCRIPTION_ID"
+echo ""
+
+# Expected Key Vault naming pattern: az-p-{region_code}-kv-secrets-001 (with dashes)
+# Region codes are standardized to exactly 3 characters
+# This matches Resource Group naming: az-p-{code}-rg-sec-001
+# Some existing Key Vaults may use: azp{code}kvsecrets001 (legacy, no dashes, old codes)
+REGIONS=($(get_all_regions))
+
+log_subsection "KEY VAULT STATUS BY REGION"
+
+EXISTING_COUNT=0
+MISSING_COUNT=0
+MISSING_REGIONS=()
+
+for region_info in "${REGIONS[@]}"; do
+    REGION_NAME="${region_info%%:*}"
+    REGION_CODE="${region_info##*:}"
+    
+    # Use library function if available, otherwise extract from string
+    if [ -z "$REGION_CODE" ]; then
+        REGION_CODE=$(get_region_code "$REGION_NAME")
+    fi
+    
+    # Try both naming patterns (standard with dashes, legacy without)
+    KV_NAME_STANDARD="az-p-${REGION_CODE}-kv-secrets-001"  # Standard (with dashes, 3-char code, matches RG)
+    KV_NAME_LEGACY="azp${REGION_CODE}kvsecrets001"  # Legacy (no dashes, may use old codes)
+    KV_FOUND=""
+    KV_NAME=""
+    
+    # Prefer standard naming, but check legacy if standard not found
+    if az keyvault show --name "$KV_NAME_STANDARD" --query id &> /dev/null; then
+        KV_FOUND="$KV_NAME_STANDARD"
+        KV_NAME="$KV_NAME_STANDARD"
+    elif az keyvault show --name "$KV_NAME_LEGACY" --query id &> /dev/null; then
+        KV_FOUND="$KV_NAME_LEGACY"
+        KV_NAME="$KV_NAME_LEGACY"
+    fi
+    
+    if [ -n "$KV_FOUND" ]; then
+        RG=$(az keyvault show --name "$KV_NAME" --query resourceGroup -o tsv 2>/dev/null)
+        echo "✅ $REGION_NAME: $KV_NAME (RG: $RG)"
+        ((EXISTING_COUNT++))
+    else
+        echo "❌ $REGION_NAME: $KV_NAME_STANDARD or $KV_NAME_LEGACY (NOT FOUND)"
+        ((MISSING_COUNT++))
+        MISSING_REGIONS+=("$REGION_NAME:$REGION_CODE")
+    fi
+done
+
+echo ""
+echo "=" | awk '{printf "%-64s\n", ""}'
+echo "📊 SUMMARY"
+echo "=" | awk '{printf "%-64s\n", ""}'
+echo ""
+
+echo "Existing Key Vaults: $EXISTING_COUNT/36"
+echo "Missing Key Vaults: $MISSING_COUNT/36"
+echo ""
+
+if [ $MISSING_COUNT -gt 0 ]; then
+    echo "Missing regions:"
+    for region_info in "${MISSING_REGIONS[@]}"; do
+        echo "  • ${region_info%%:*}"
+    done
+    echo ""
+    echo "⚠️  Key Vaults need to be deployed via Terraform"
+    exit 1
+else
+    echo "✅ All Key Vaults are deployed"
+    exit 0
+fi
+
--- a/scripts/key-management/generate-oracle-keys.sh
+++ b/scripts/key-management/generate-oracle-keys.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Generate oracle keys for EthSigner
+# Oracle keys are used to sign transactions for oracle updates
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+KEYS_DIR="$PROJECT_ROOT/keys/oracle"
+
+NUM_ORACLES=${1:-1}
+PASSWORD_FILE="${2:-$PROJECT_ROOT/keys/.password}"
+
+echo "Generating $NUM_ORACLES oracle keys..."
+
+# Create password file if it doesn't exist
+if [ ! -f "$PASSWORD_FILE" ]; then
+    mkdir -p "$(dirname "$PASSWORD_FILE")"
+    openssl rand -base64 32 > "$PASSWORD_FILE"
+    chmod 600 "$PASSWORD_FILE"
+    echo "Created password file: $PASSWORD_FILE"
+fi
+
+# Generate keys
+for i in $(seq 1 $NUM_ORACLES); do
+    ORACLE_DIR="$KEYS_DIR/oracle-$i"
+    mkdir -p "$ORACLE_DIR"
+    
+    # Generate private key
+    PRIVATE_KEY=$(openssl rand -hex 32)
+    echo "$PRIVATE_KEY" > "$ORACLE_DIR/key.priv"
+    chmod 600 "$ORACLE_DIR/key.priv"
+    
+    # Create keystore for EthSigner
+    if command -v ethsigner &> /dev/null; then
+        ethsigner --data-path="$ORACLE_DIR" \
+            account import --private-key-file="$ORACLE_DIR/key.priv" \
+            --password-file="$PASSWORD_FILE" 2>/dev/null || true
+    fi
+    
+    echo "Generated oracle $i key: $ORACLE_DIR/key.priv"
+done
+
+echo "Oracle keys generated in: $KEYS_DIR"
+echo "Password file: $PASSWORD_FILE"
+echo "IMPORTANT: Store keys securely in Azure Key Vault for production."
+
--- a/scripts/key-management/generate-validator-keys.sh
+++ b/scripts/key-management/generate-validator-keys.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Generate validator keys for IBFT 2.0
+# This script generates validator keypairs for Besu IBFT 2.0 consensus
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+KEYS_DIR="$PROJECT_ROOT/keys/validators"
+
+NUM_VALIDATORS=${1:-4}
+PASSWORD_FILE="${2:-$PROJECT_ROOT/keys/.password}"
+
+echo "Generating $NUM_VALIDATORS validator keys..."
+
+# Create password file if it doesn't exist
+if [ ! -f "$PASSWORD_FILE" ]; then
+    mkdir -p "$(dirname "$PASSWORD_FILE")"
+    openssl rand -base64 32 > "$PASSWORD_FILE"
+    chmod 600 "$PASSWORD_FILE"
+    echo "Created password file: $PASSWORD_FILE"
+fi
+
+# Generate keys
+for i in $(seq 1 $NUM_VALIDATORS); do
+    VALIDATOR_DIR="$KEYS_DIR/validator-$i"
+    mkdir -p "$VALIDATOR_DIR"
+    
+    # Generate private key
+    PRIVATE_KEY=$(openssl rand -hex 32)
+    echo "$PRIVATE_KEY" > "$VALIDATOR_DIR/key.priv"
+    chmod 600 "$VALIDATOR_DIR/key.priv"
+    
+    # Create keystore using Besu if available
+    if command -v besu &> /dev/null; then
+        echo "$(cat "$PASSWORD_FILE")" | besu --data-path="$VALIDATOR_DIR" \
+            account import --private-key="$VALIDATOR_DIR/key.priv" \
+            --password-file=<(echo "$(cat "$PASSWORD_FILE")") 2>/dev/null || true
+    fi
+    
+    echo "Generated validator $i key: $VALIDATOR_DIR/key.priv"
+done
+
+echo "Validator keys generated in: $KEYS_DIR"
+echo "Password file: $PASSWORD_FILE"
+echo "IMPORTANT: Store keys securely. For production, use Azure Key Vault or HSM."
+
--- a/scripts/key-management/grant-keyvault-permissions-parallel.sh
+++ b/scripts/key-management/grant-keyvault-permissions-parallel.sh
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+
+# Grant Key Vault permissions in parallel for faster execution
+# Handles both access policies and RBAC-enabled vaults
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Your AAD object ID
+OBJECT_ID="5c40d456-49d2-4f2a-b35c-66255ca33b04"
+
+# Email for logging
+USER_EMAIL="admin@absoluterealms.org"
+
+# Subscription ID
+SUBSCRIPTION_ID="fc08d829-4f14-413d-ab27-ce024425db0b"
+
+echo "╔════════════════════════════════════════════════════════════════╗"
+echo "║     GRANTING KEY VAULT PERMISSIONS - PARALLEL EXECUTION         ║"
+echo "╚════════════════════════════════════════════════════════════════╝"
+
+echo "User: $USER_EMAIL"
+echo "Object ID: $OBJECT_ID"
+echo "Subscription: $SUBSCRIPTION_ID"
+
+# Set subscription
+az account set --subscription "$SUBSCRIPTION_ID" > /dev/null 2>&1
+
+echo "Processing subscription: $SUBSCRIPTION_ID"
+
+# Get all Key Vault names
+VAULTS=$(az keyvault list --query "[].name" -o tsv 2>/dev/null)
+
+if [ -z "$VAULTS" ]; then
+    echo "❌ No Key Vaults found"
+    exit 1
+fi
+
+TOTAL=$(echo "$VAULTS" | wc -l)
+echo "Total Key Vaults: $TOTAL"
+
+# Function to grant permissions for a single vault
+grant_permissions() {
+    local kv_name="$1"
+    local object_id="$2"
+    local subscription_id="$3"
+    
+    # Get resource group
+    local kv_rg=$(az keyvault show --name "$kv_name" --query "resourceGroup" -o tsv 2>/dev/null)
+    
+    if [ -z "$kv_rg" ]; then
+        echo "❌ $kv_name: Could not get resource group"
+        return 1
+    fi
+    
+    # Check if RBAC-enabled
+    local is_rbac=$(az keyvault show --name "$kv_name" --query "properties.enableRbacAuthorization" -o tsv 2>/dev/null)
+    
+    if [ "$is_rbac" = "true" ]; then
+        # Use RBAC role assignment
+        if az role assignment create \
+            --role "Key Vault Secrets Officer" \
+            --assignee "$object_id" \
+            --scope "/subscriptions/$subscription_id/resourceGroups/$kv_rg/providers/Microsoft.KeyVault/vaults/$kv_name" \
+            > /dev/null 2>&1; then
+            echo "✅ $kv_name: RBAC role assigned"
+            return 0
+        else
+            echo "❌ $kv_name: Failed to assign RBAC role"
+            return 1
+        fi
+    else
+        # Use access policy
+        if az keyvault set-policy \
+            --name "$kv_name" \
+            --object-id "$object_id" \
+            --secret-permissions get list set delete backup restore recover purge \
+            > /dev/null 2>&1; then
+            echo "✅ $kv_name: Access policy updated"
+            return 0
+        else
+            echo "❌ $kv_name: Failed to update access policy"
+            return 1
+        fi
+    fi
+}
+
+export -f grant_permissions
+export OBJECT_ID
+export SUBSCRIPTION_ID
+
+echo "Granting permissions (parallel execution)..."
+
+# Process in parallel (max 5 concurrent)
+SUCCESS_COUNT=0
+FAILED_COUNT=0
+
+while IFS= read -r kv_name; do
+    if grant_permissions "$kv_name" "$OBJECT_ID" "$SUBSCRIPTION_ID"; then
+        ((SUCCESS_COUNT++))
+    else
+        ((FAILED_COUNT++))
+    fi
+done < <(echo "$VAULTS" | xargs -P 5 -I {} bash -c 'grant_permissions "$@"' _ {})
+
+echo "======================================================================"
+echo "📊 SUMMARY"
+echo "======================================================================"
+echo "Total Key Vaults processed: $TOTAL"
+echo "✅ Success: $SUCCESS_COUNT"
+echo "❌ Failed: $FAILED_COUNT"
+
+if [ $FAILED_COUNT -eq 0 ]; then
+    echo "✅ All permissions granted successfully"
+    exit 0
+else
+    echo "⚠️  Some permissions failed - check errors above"
+    exit 1
+fi
+
--- a/scripts/key-management/grant-keyvault-permissions.sh
+++ b/scripts/key-management/grant-keyvault-permissions.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+# Grant Key Vault permissions using the provided Azure CLI method
+# Handles both access policies and RBAC-enabled vaults
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Your AAD object ID
+OBJECT_ID="5c40d456-49d2-4f2a-b35c-66255ca33b04"
+
+# Email for logging
+USER_EMAIL="admin@absoluterealms.org"
+
+# Subscription ID
+SUBSCRIPTION_ID="fc08d829-4f14-413d-ab27-ce024425db0b"
+
+echo "╔════════════════════════════════════════════════════════════════╗"
+echo "║     GRANTING KEY VAULT PERMISSIONS                                ║"
+echo "╚════════════════════════════════════════════════════════════════╝"
+
+echo "User: $USER_EMAIL"
+echo "Object ID: $OBJECT_ID"
+echo "Subscription: $SUBSCRIPTION_ID"
+
+# Set subscription
+az account set --subscription "$SUBSCRIPTION_ID"
+
+echo "Processing subscription: $SUBSCRIPTION_ID"
+
+SUCCESS_COUNT=0
+FAILED_COUNT=0
+RBAC_COUNT=0
+POLICY_COUNT=0
+
+# Get all Key Vault names in this subscription
+for KV in $(az keyvault list --query "[].name" -o tsv 2>/dev/null); do
+  echo "  -> Updating Key Vault: $KV"
+  
+  # Get resource group
+  KV_RG=$(az keyvault show --name "$KV" --query "resourceGroup" -o tsv 2>/dev/null)
+  
+  if [ -z "$KV_RG" ]; then
+    echo "    ❌ Could not get resource group"
+    ((FAILED_COUNT++))
+    continue
+  fi
+  
+  # Check if RBAC-enabled
+  IS_RBAC=$(az keyvault show --name "$KV" --query "properties.enableRbacAuthorization" -o tsv 2>/dev/null)
+  
+  if [ "$IS_RBAC" = "true" ]; then
+    # Use RBAC role assignment
+    echo "    Using RBAC (Key Vault Secrets Officer)"
+    az role assignment create \
+      --role "Key Vault Secrets Officer" \
+      --assignee "$OBJECT_ID" \
+      --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$KV_RG/providers/Microsoft.KeyVault/vaults/$KV" \
+      > /dev/null 2>&1
+    
+    if [ $? -eq 0 ]; then
+      echo "    ✅ RBAC role assigned"
+      ((SUCCESS_COUNT++))
+      ((RBAC_COUNT++))
+    else
+      echo "    ❌ Failed to assign RBAC role"
+      ((FAILED_COUNT++))
+    fi
+  else
+    # Use access policy
+    echo "    Using Access Policy"
+    az keyvault set-policy \
+      --name "$KV" \
+      --object-id "$OBJECT_ID" \
+      --secret-permissions get list set delete backup restore recover purge \
+      > /dev/null 2>&1
+    
+    if [ $? -eq 0 ]; then
+      echo "    ✅ Access policy updated"
+      ((SUCCESS_COUNT++))
+      ((POLICY_COUNT++))
+    else
+      echo "    ❌ Failed to update access policy"
+      ((FAILED_COUNT++))
+    fi
+  fi
+  
+done
+
+echo "======================================================================"
+echo "📊 SUMMARY"
+echo "======================================================================"
+echo "Total Key Vaults processed: $((SUCCESS_COUNT + FAILED_COUNT))"
+echo "✅ Success: $SUCCESS_COUNT"
+echo "   - Access Policy: $POLICY_COUNT"
+echo "   - RBAC: $RBAC_COUNT"
+echo "❌ Failed: $FAILED_COUNT"
+
+if [ $FAILED_COUNT -eq 0 ]; then
+  echo "✅ All permissions granted successfully"
+  exit 0
+else
+  echo "⚠️  Some permissions failed - check errors above"
+  exit 1
+fi
+
--- a/scripts/key-management/manage-keyvaults.sh
+++ b/scripts/key-management/manage-keyvaults.sh
@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Comprehensive Key Vault management script
+# Handles deployment, verification, permission grants, and secret storage
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+load_env --file "$PROJECT_ROOT/.env" ${ENV_PROFILE:+--profile "$ENV_PROFILE"}
+SCRIPT_NAME="manage-keyvaults.sh"
+SCRIPT_DESC="Manage Key Vault lifecycle: deploy, status, permissions, store-keys, verify, list, complete"
+SCRIPT_USAGE="${SCRIPT_NAME} [deploy|status|permissions|store-keys|verify|list|complete] [--dry-run] [--region <name>] [--help]"
+SCRIPT_OPTIONS="--dry-run         Do not execute changes\n--region <name>   Limit to a specific region\n--help            Show usage"
+SCRIPT_REQUIREMENTS="Azure CLI (ensure_azure_cli), permissions to manage Key Vaults"
+handle_help "${1:-}"
+
+# Initialize
+SUBSCRIPTION_ID="$(get_subscription_id)"
+ensure_azure_cli || exit 1
+set_subscription "$SUBSCRIPTION_ID" || true
+
+# Functions
+show_help() {
+    cat << EOF
+Key Vault Management Script
+
+Usage: $0 [COMMAND] [OPTIONS]
+
+Commands:
+  deploy           - Deploy all Key Vaults (Phase 1)
+  status           - Check Key Vault deployment status
+  permissions      - Grant permissions to all Key Vaults
+  store-keys       - Store validator node keys in Key Vaults
+  verify           - Verify Key Vault configuration
+  list             - List all Key Vaults and their secrets count
+  complete         - Run all steps: deploy, permissions, store-keys
+
+Options:
+  --dry-run        - Show what would be done without executing
+  --region REGION  - Process specific region only
+  --help           - Show this help message
+
+Examples:
+  $0 deploy                      # Deploy all Key Vaults
+  $0 status                      # Check status
+  $0 complete                    # Run all steps
+  $0 store-keys --dry-run        # Preview secret storage
+  $0 permissions                 # Grant permissions
+
+EOF
+}
+
+deploy_keyvaults() {
+    log_info "Deploying Key Vaults..."
+    bash "$PROJECT_ROOT/scripts/deployment/deploy-keyvaults-only.sh"
+}
+
+check_status() {
+    log_info "Checking Key Vault status..."
+    bash "$SCRIPT_DIR/check-keyvault-status.sh"
+}
+
+grant_permissions() {
+    log_info "Granting Key Vault permissions..."
+    
+    # Try parallel script first, fall back to sequential
+    if [ -f "$SCRIPT_DIR/grant-keyvault-permissions-parallel.sh" ]; then
+        bash "$SCRIPT_DIR/grant-keyvault-permissions-parallel.sh"
+    else
+        bash "$SCRIPT_DIR/grant-keyvault-permissions.sh"
+    fi
+}
+
+store_keys() {
+    log_info "Storing validator keys in Key Vaults..."
+    
+    if [ "$DRY_RUN" = "1" ]; then
+        export DRY_RUN=1
+    fi
+    
+    bash "$SCRIPT_DIR/store-nodes-in-keyvault.sh"
+}
+
+verify_keyvaults() {
+    log_info "Verifying Key Vault configuration..."
+    
+    # Check Azure login
+    
+    log_success "Azure authenticated"
+    
+    # Get all Key Vaults
+    VAULTS=$(az keyvault list --query "[].name" -o tsv 2>/dev/null)
+    
+    if [ -z "$VAULTS" ]; then
+        log_error "No Key Vaults found"
+        exit 1
+    fi
+    
+    TOTAL=0
+    VERIFIED=0
+    
+    for KV in $VAULTS; do
+        TOTAL=$((TOTAL + 1))
+        
+        # Check Key Vault properties
+        KV_RG=$(az keyvault show --name "$KV" --query "resourceGroup" -o tsv 2>/dev/null)
+        KV_LOCATION=$(az keyvault show --name "$KV" --query "location" -o tsv 2>/dev/null)
+        IS_RBAC=$(az keyvault show --name "$KV" --query "properties.enableRbacAuthorization" -o tsv 2>/dev/null)
+        SOFT_DELETE=$(az keyvault show --name "$KV" --query "properties.enableSoftDelete" -o tsv 2>/dev/null)
+        PURGE_PROTECTION=$(az keyvault show --name "$KV" --query "properties.enablePurgeProtection" -o tsv 2>/dev/null)
+        
+        # Check secrets count
+        SECRETS_COUNT=$(az keyvault secret list --vault-name "$KV" --query "length(@)" -o tsv 2>/dev/null || echo "0")
+        
+        echo "Key Vault: $KV"
+        echo "  Resource Group: $KV_RG"
+        echo "  Location: $KV_LOCATION"
+        echo "  RBAC Enabled: $IS_RBAC"
+        echo "  Soft Delete: $SOFT_DELETE"
+        echo "  Purge Protection: $PURGE_PROTECTION"
+        echo "  Secrets: $SECRETS_COUNT"
+        
+        if [ "$SOFT_DELETE" = "true" ] && [ "$IS_RBAC" = "true" ]; then
+            log_success "  Configuration OK"
+            VERIFIED=$((VERIFIED + 1))
+        else
+            log_warn "  Consider enabling RBAC and Soft Delete"
+        fi
+        
+    done
+    
+    echo "=" | awk '{printf "%-64s\n", ""}'
+    echo "📊 SUMMARY"
+    echo "=" | awk '{printf "%-64s\n", ""}'
+    echo "Total Key Vaults: $TOTAL"
+    echo "Verified: $VERIFIED"
+    log_success "Verification complete"
+}
+
+list_keyvaults() {
+    log_info "Listing Key Vaults and secrets..."
+    
+    VAULTS=$(az keyvault list --query "[].name" -o tsv 2>/dev/null)
+    
+    if [ -z "$VAULTS" ]; then
+        log_error "❌ No Key Vaults found"
+        exit 1
+    fi
+    
+    echo "Key Vault Name | Secrets | Resource Group | Location"
+    echo "--------------------------------------------------------"
+    
+    for KV in $VAULTS; do
+        SECRETS_COUNT=$(az keyvault secret list --vault-name "$KV" --query "length(@)" -o tsv 2>/dev/null || echo "0")
+        KV_RG=$(az keyvault show --name "$KV" --query "resourceGroup" -o tsv 2>/dev/null)
+        KV_LOCATION=$(az keyvault show --name "$KV" --query "location" -o tsv 2>/dev/null)
+        
+        printf "%-40s | %7s | %-20s | %s\n" "$KV" "$SECRETS_COUNT" "$KV_RG" "$KV_LOCATION"
+    done
+    
+}
+
+run_complete() {
+    log_info "Running complete Key Vault setup..."
+    
+    # Step 1: Deploy
+    log_warn "Step 1/4: Deploying Key Vaults..."
+    deploy_keyvaults
+    
+    # Step 2: Check status
+    log_warn "Step 2/4: Checking deployment status..."
+    check_status || {
+        log_error "❌ Key Vault deployment incomplete"
+        exit 1
+    }
+    
+    # Step 3: Grant permissions
+    log_warn "Step 3/4: Granting permissions..."
+    grant_permissions
+    
+    # Step 4: Store keys
+    log_warn "Step 4/4: Storing validator keys..."
+    store_keys
+    
+    log_success "✅ Key Vault setup complete!"
+}
+
+# Main script
+COMMAND="${1:-help}"
+
+case "$COMMAND" in
+    deploy)
+        deploy_keyvaults
+        ;;
+    status)
+        check_status
+        ;;
+    permissions)
+        grant_permissions
+        ;;
+    store-keys)
+        # Check for --dry-run flag
+        if [ "$2" = "--dry-run" ]; then
+            export DRY_RUN=1
+        fi
+        store_keys
+        ;;
+    verify)
+        verify_keyvaults
+        ;;
+    list)
+        list_keyvaults
+        ;;
+    complete)
+        run_complete
+        ;;
+    help|--help|-h)
+        show_help
+        ;;
+    *)
+        log_error "Error: Unknown command: $COMMAND"
+        show_help
+        exit 1
+        ;;
+esac
+
--- a/scripts/key-management/rotate-keys.sh
+++ b/scripts/key-management/rotate-keys.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+# Key rotation script for validator and oracle keys
+# This script rotates keys in Azure Key Vault and updates Kubernetes secrets
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# Configuration
+KEY_VAULT_NAME="${KEY_VAULT_NAME:-defi-oracle-kv}"
+NAMESPACE="${NAMESPACE:-besu-network}"
+KEY_TYPE="${1:-validator}"  # validator or oracle
+NUM_KEYS="${2:-4}"
+
+
+log_success "Starting key rotation for $KEY_TYPE keys"
+
+# Check if Azure CLI is installed
+if ! command -v az &> /dev/null; then
+    log_error "Error: Azure CLI not found. Please install Azure CLI."
+    exit 1
+fi
+
+# Check if logged in to Azure
+if ! az account show &> /dev/null; then
+    log_error "Error: Not logged in to Azure. Please run 'az login'."
+    exit 1
+fi
+
+# Generate new keys
+log_warn "Generating new keys..."
+for i in $(seq 1 $NUM_KEYS); do
+    # Generate new private key
+    NEW_KEY=$(openssl rand -hex 32)
+    
+    # Store in Key Vault
+    KEY_NAME="${KEY_TYPE}-key-${i}"
+    az keyvault secret set \
+        --vault-name "$KEY_VAULT_NAME" \
+        --name "$KEY_NAME" \
+        --value "$NEW_KEY" \
+        --content-type "text/plain" \
+        --tags "type=$KEY_TYPE" "index=$i" "rotated=$(date +%Y-%m-%d)"
+    
+    if [ $? -eq 0 ]; then
+        log_success "✓ Key stored in Key Vault: $KEY_NAME"
+    else
+        log_error "✗ Failed to store key in Key Vault: $KEY_NAME"
+        exit 1
+    fi
+done
+
+# Update Kubernetes secrets
+log_warn "Updating Kubernetes secrets..."
+SECRET_NAME="besu-${KEY_TYPE}-keys"
+
+# Create secret from Key Vault
+kubectl create secret generic "$SECRET_NAME" \
+    --from-literal=key-1=$(az keyvault secret show --vault-name "$KEY_VAULT_NAME" --name "${KEY_TYPE}-key-1" --query value -o tsv) \
+    --from-literal=key-2=$(az keyvault secret show --vault-name "$KEY_VAULT_NAME" --name "${KEY_TYPE}-key-2" --query value -o tsv) \
+    --from-literal=key-3=$(az keyvault secret show --vault-name "$KEY_VAULT_NAME" --name "${KEY_TYPE}-key-3" --query value -o tsv) \
+    --from-literal=key-4=$(az keyvault secret show --vault-name "$KEY_VAULT_NAME" --name "${KEY_TYPE}-key-4" --query value -o tsv) \
+    -n "$NAMESPACE" \
+    --dry-run=client -o yaml | kubectl apply -f -
+
+if [ $? -eq 0 ]; then
+    log_success "✓ Kubernetes secret updated: $SECRET_NAME"
+else
+    log_error "✗ Failed to update Kubernetes secret"
+    exit 1
+fi
+
+# Restart pods to use new keys
+log_warn "Restarting pods to use new keys..."
+if [ "$KEY_TYPE" == "validator" ]; then
+    kubectl rollout restart statefulset/besu-validator -n "$NAMESPACE"
+    log_success "✓ Validator pods restarted"
+elif [ "$KEY_TYPE" == "oracle" ]; then
+    kubectl rollout restart deployment/oracle-publisher -n "$NAMESPACE"
+    log_success "✓ Oracle publisher pods restarted"
+fi
+
+# Wait for pods to be ready
+log_warn "Waiting for pods to be ready..."
+if [ "$KEY_TYPE" == "validator" ]; then
+    kubectl wait --for=condition=ready pod -l component=validator -n "$NAMESPACE" --timeout=300s
+elif [ "$KEY_TYPE" == "oracle" ]; then
+    kubectl wait --for=condition=ready pod -l app=oracle-publisher -n "$NAMESPACE" --timeout=300s
+fi
+
+# Verify keys are working
+log_warn "Verifying keys are working..."
+# Add verification logic here
+# For validators: Check if blocks are being produced
+# For oracle: Check if oracle updates are working
+
+log_success "Key rotation completed successfully"
+
+# Archive old keys (optional)
+log_warn "Archiving old keys..."
+# Move old keys to archive in Key Vault
+# az keyvault secret set --vault-name "$KEY_VAULT_NAME" --name "${KEY_TYPE}-key-1-archived-$(date +%Y%m%d)" --value "<old-key>"
+
+log_success "Key rotation process completed"
+
--- a/scripts/key-management/store-nodes-in-keyvault.sh
+++ b/scripts/key-management/store-nodes-in-keyvault.sh
@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Store validator node keys in Azure Key Vaults
+# Each region's Key Vault stores the validator keys for that region
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+source "$SCRIPT_DIR/../lib/init.sh"
+# Support both 48-node and 36-node configurations
+NODES_FILE="/tmp/node_addresses_36.json"
+if [ ! -f "$NODES_FILE" ]; then
+    NODES_FILE="/tmp/node_addresses.json"
+fi
+KV_MAPPING="/tmp/key_vault_secrets_36.json"
+if [ ! -f "$KV_MAPPING" ]; then
+    KV_MAPPING="/tmp/key_vault_secrets.json"
+fi
+
+log_section "STORING VALIDATOR NODE KEYS IN AZURE KEY VAULTS"
+
+# Dry-run mode (set DRY_RUN=1 to skip actual storage)
+DRY_RUN="${DRY_RUN:-0}"
+
+# Load node data using Python
+python3 << PYTHON_EOF
+import json
+import subprocess
+import sys
+import os
+
+# Support both 48-node and 36-node configurations
+nodes_file = '/tmp/node_addresses_36.json'
+if not os.path.exists(nodes_file):
+    nodes_file = '/tmp/node_addresses.json'
+
+kv_file = '/tmp/key_vault_secrets_36.json'
+if not os.path.exists(kv_file):
+    kv_file = '/tmp/key_vault_secrets.json'
+
+with open(nodes_file, 'r') as f:
+    nodes = json.load(f)
+
+with open(kv_file, 'r') as f:
+    kv_mapping = json.load(f)
+
+print("╔════════════════════════════════════════════════════════════════╗")
+print("║     STORING NODE KEYS IN AZURE KEY VAULTS                       ║")
+print("╚════════════════════════════════════════════════════════════════╝")
+print()
+
+dry_run = os.environ.get('DRY_RUN', '0') == '1'
+stored_count = 0
+success_count = 0
+failed_count = 0
+skipped_count = 0
+total_count = 0
+errors = []
+
+# Verify Key Vaults exist first
+print("Verifying Key Vaults exist...")
+kv_verified = {}
+for region, config in kv_mapping.items():
+    kv_name = config['key_vault_name']
+    
+    # Check if Key Vault exists
+    result = subprocess.run(
+        ['az', 'keyvault', 'show', '--name', kv_name, '--query', 'name', '-o', 'tsv'],
+        capture_output=True,
+        text=True
+    )
+    
+    if result.returncode == 0 and result.stdout.strip() == kv_name:
+        kv_verified[region] = True
+        print(f"  ✓ {kv_name}")
+    else:
+        kv_verified[region] = False
+        print(f"  ✗ {kv_name} (NOT FOUND)")
+        errors.append(f"Key Vault not found: {kv_name} for region {region}")
+
+print()
+
+if not dry_run and len([k for k, v in kv_verified.items() if not v]) > 0:
+    print("❌ ERROR: Some Key Vaults do not exist. Please deploy them first.")
+    print("   Run: bash scripts/deployment/deploy-keyvaults-only.sh")
+    sys.exit(1)
+
+# Store secrets
+for region, config in kv_mapping.items():
+    kv_name = config['key_vault_name']
+    region_nodes = config['nodes']
+    
+    if not kv_verified.get(region, False):
+        print(f"⚠️  Skipping region {region} - Key Vault not found: {kv_name}")
+        skipped_count += len(region_nodes)
+        continue
+    
+    print(f"Region: {region}")
+    print(f"  Key Vault: {kv_name}")
+    
+    for node in region_nodes:
+        node_id = node['id']
+        secret_name = f"validator-{node_id}-private-key"
+        private_key = node.get('private_key_hex', '')
+        
+        if not private_key:
+            print(f"    ⚠️  Skipping {secret_name} - no private key found")
+            skipped_count += 1
+            total_count += 1
+            continue
+        
+        if dry_run:
+            print(f"    [DRY RUN] Would store: {secret_name}")
+        else:
+            # Check if secret already exists
+            check_result = subprocess.run(
+                ['az', 'keyvault', 'secret', 'show', '--vault-name', kv_name, '--name', secret_name, '--query', 'name', '-o', 'tsv'],
+                capture_output=True,
+                text=True
+            )
+            
+            if check_result.returncode == 0:
+                print(f"    ⚠️  {secret_name} already exists - skipping")
+                skipped_count += 1
+            else:
+                # Store private key as secret
+                cmd = [
+                    'az', 'keyvault', 'secret', 'set',
+                    '--vault-name', kv_name,
+                    '--name', secret_name,
+                    '--value', private_key,
+                    '--output', 'none'
+                ]
+                
+                result = subprocess.run(cmd, capture_output=True, text=True)
+                
+                if result.returncode == 0:
+                    print(f"    ✓ Stored: {secret_name}")
+                    success_count += 1
+                    stored_count += 1
+                else:
+                    error_msg = f"Failed to store {secret_name}: {result.stderr.strip()}"
+                    print(f"    ✗ {error_msg}")
+                    errors.append(error_msg)
+                    failed_count += 1
+        
+        total_count += 1
+    
+    print()
+
+print("=" * 64)
+print("📊 SUMMARY")
+print("=" * 64)
+print()
+
+print(f"Total nodes: {len(nodes)}")
+print(f"Secrets to store: {total_count}")
+if not dry_run:
+    print(f"✓ Successfully stored: {success_count}")
+    print(f"✗ Failed: {failed_count}")
+    print(f"⚠️  Skipped: {skipped_count}")
+    print(f"Key Vaults: {len(kv_mapping)}")
+else:
+    print(f"[DRY RUN] Would store: {total_count}")
+print()
+
+if errors:
+    print("=" * 64)
+    print("❌ ERRORS")
+    print("=" * 64)
+    for error in errors:
+        print(f"  • {error}")
+    print()
+
+if failed_count > 0:
+    sys.exit(1)
+
+PYTHON_EOF
+
+EXIT_CODE=$?
+
+if [ $EXIT_CODE -eq 0 ]; then
+    log_success "Key Vault storage complete!"
+    echo ""
+else
+    log_error "Key Vault storage completed with errors"
+    echo ""
+    exit $EXIT_CODE
+fi
+