Add Oracle Aggregator and CCIP Integration
- Introduced Aggregator.sol for Chainlink-compatible oracle functionality, including round-based updates and access control. - Added OracleWithCCIP.sol to extend Aggregator with CCIP cross-chain messaging capabilities. - Created .gitmodules to include OpenZeppelin contracts as a submodule. - Developed a comprehensive deployment guide in NEXT_STEPS_COMPLETE_GUIDE.md for Phase 2 and smart contract deployment. - Implemented Vite configuration for the orchestration portal, supporting both Vue and React frameworks. - Added server-side logic for the Multi-Cloud Orchestration Portal, including API endpoints for environment management and monitoring. - Created scripts for resource import and usage validation across non-US regions. - Added tests for CCIP error handling and integration to ensure robust functionality. - Included various new files and directories for the orchestration portal and deployment scripts.
This commit is contained in:
381
terraform/multi-cloud/modules/aws/main.tf
Normal file
381
terraform/multi-cloud/modules/aws/main.tf
Normal file
@@ -0,0 +1,381 @@
|
||||
# AWS Infrastructure Module
|
||||
# Creates EKS cluster, networking, and supporting resources for AWS environments
|
||||
|
||||
locals {
|
||||
env = var.environment_config
|
||||
|
||||
# Extract AWS-specific config
|
||||
aws_config = try(local.env.aws, {})
|
||||
|
||||
# Extract infrastructure config
|
||||
infra = try(local.env.infrastructure, {})
|
||||
k8s_config = try(local.infra.kubernetes, {})
|
||||
net_config = try(local.infra.networking, {})
|
||||
|
||||
# Naming
|
||||
name_prefix = "${local.env.name}-${var.environment}"
|
||||
|
||||
# Node pools
|
||||
node_pools = try(local.k8s_config.node_pools, {})
|
||||
}
|
||||
|
||||
# VPC
|
||||
resource "aws_vpc" "main" {
|
||||
cidr_block = try(local.net_config.vpc_cidr, "10.0.0.0/16")
|
||||
enable_dns_hostnames = true
|
||||
enable_dns_support = true
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-vpc"
|
||||
Type = "Multi-Cloud"
|
||||
})
|
||||
}
|
||||
|
||||
# Internet Gateway
|
||||
resource "aws_internet_gateway" "main" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-igw"
|
||||
})
|
||||
}
|
||||
|
||||
# Subnets
|
||||
resource "aws_subnet" "eks" {
|
||||
for_each = {
|
||||
for idx, subnet in try(local.net_config.subnets, []) : subnet.name => subnet
|
||||
if subnet.name == "eks" || subnet.name == "gke" || subnet.name == "aks"
|
||||
}
|
||||
|
||||
vpc_id = aws_vpc.main.id
|
||||
cidr_block = each.value.cidr
|
||||
availability_zone = try(each.value.availability_zone, data.aws_availability_zones.available.names[0])
|
||||
|
||||
map_public_ip_on_launch = true
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-subnet-${each.key}"
|
||||
"kubernetes.io/role/elb" = "1"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_subnet" "validators" {
|
||||
for_each = {
|
||||
for idx, subnet in try(local.net_config.subnets, []) : subnet.name => subnet
|
||||
if subnet.name == "validators"
|
||||
}
|
||||
|
||||
vpc_id = aws_vpc.main.id
|
||||
cidr_block = each.value.cidr
|
||||
availability_zone = try(each.value.availability_zone, data.aws_availability_zones.available.names[1])
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-subnet-validators"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_subnet" "rpc" {
|
||||
for_each = {
|
||||
for idx, subnet in try(local.net_config.subnets, []) : subnet.name => subnet
|
||||
if subnet.name == "rpc"
|
||||
}
|
||||
|
||||
vpc_id = aws_vpc.main.id
|
||||
cidr_block = each.value.cidr
|
||||
availability_zone = try(each.value.availability_zone, data.aws_availability_zones.available.names[0])
|
||||
|
||||
map_public_ip_on_launch = true
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-subnet-rpc"
|
||||
"kubernetes.io/role/elb" = "1"
|
||||
})
|
||||
}
|
||||
|
||||
# Route Table
|
||||
resource "aws_route_table" "main" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
route {
|
||||
cidr_block = "0.0.0.0/0"
|
||||
gateway_id = aws_internet_gateway.main.id
|
||||
}
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-rt"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_route_table_association" "subnets" {
|
||||
for_each = merge(
|
||||
aws_subnet.eks,
|
||||
aws_subnet.validators,
|
||||
aws_subnet.rpc
|
||||
)
|
||||
|
||||
subnet_id = each.value.id
|
||||
route_table_id = aws_route_table.main.id
|
||||
}
|
||||
|
||||
# Security Groups
|
||||
resource "aws_security_group" "eks_cluster" {
|
||||
name = "${local.name_prefix}-eks-cluster-sg"
|
||||
description = "Security group for EKS cluster"
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
egress {
|
||||
from_port = 0
|
||||
to_port = 0
|
||||
protocol = "-1"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-eks-cluster-sg"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_security_group" "validators" {
|
||||
name = "${local.name_prefix}-validators-sg"
|
||||
description = "Security group for validator nodes (private)"
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
ingress {
|
||||
from_port = 30303
|
||||
to_port = 30303
|
||||
protocol = "tcp"
|
||||
cidr_blocks = [aws_vpc.main.cidr_block]
|
||||
description = "P2P port from internal"
|
||||
}
|
||||
|
||||
egress {
|
||||
from_port = 0
|
||||
to_port = 0
|
||||
protocol = "-1"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-validators-sg"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_security_group" "rpc" {
|
||||
name = "${local.name_prefix}-rpc-sg"
|
||||
description = "Security group for RPC nodes"
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
ingress {
|
||||
from_port = 8545
|
||||
to_port = 8545
|
||||
protocol = "tcp"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
description = "RPC HTTP"
|
||||
}
|
||||
|
||||
ingress {
|
||||
from_port = 8546
|
||||
to_port = 8546
|
||||
protocol = "tcp"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
description = "RPC WebSocket"
|
||||
}
|
||||
|
||||
egress {
|
||||
from_port = 0
|
||||
to_port = 0
|
||||
protocol = "-1"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-rpc-sg"
|
||||
})
|
||||
}
|
||||
|
||||
# EKS Cluster
|
||||
resource "aws_eks_cluster" "main" {
|
||||
name = "${local.name_prefix}-eks"
|
||||
role_arn = aws_iam_role.eks_cluster.arn
|
||||
version = try(local.k8s_config.version, "1.28")
|
||||
|
||||
vpc_config {
|
||||
subnet_ids = [for s in aws_subnet.eks : s.id]
|
||||
security_group_ids = [aws_security_group.eks_cluster.id]
|
||||
endpoint_private_access = true
|
||||
endpoint_public_access = true
|
||||
}
|
||||
|
||||
enabled_cluster_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
|
||||
|
||||
depends_on = [
|
||||
aws_cloudwatch_log_group.eks,
|
||||
aws_iam_role_policy_attachment.eks_cluster_policy,
|
||||
]
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-eks"
|
||||
})
|
||||
}
|
||||
|
||||
# EKS Node Groups
|
||||
resource "aws_eks_node_group" "system" {
|
||||
count = try(local.node_pools.system.count, 0) > 0 ? 1 : 0
|
||||
|
||||
cluster_name = aws_eks_cluster.main.name
|
||||
node_group_name = "system"
|
||||
node_role_arn = aws_iam_role.eks_node.arn
|
||||
subnet_ids = [for s in aws_subnet.eks : s.id]
|
||||
|
||||
instance_types = [try(local.node_pools.system.instance_type, "t3.medium")]
|
||||
capacity_type = "ON_DEMAND"
|
||||
|
||||
scaling_config {
|
||||
desired_size = try(local.node_pools.system.count, 1)
|
||||
max_size = try(local.node_pools.system.count, 1) * 2
|
||||
min_size = 1
|
||||
}
|
||||
|
||||
labels = {
|
||||
pool = "system"
|
||||
role = "system"
|
||||
}
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-node-system"
|
||||
Pool = "system"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_eks_node_group" "validators" {
|
||||
count = try(local.node_pools.validators.count, 0) > 0 ? 1 : 0
|
||||
|
||||
cluster_name = aws_eks_cluster.main.name
|
||||
node_group_name = "validators"
|
||||
node_role_arn = aws_iam_role.eks_node.arn
|
||||
subnet_ids = [for s in aws_subnet.validators : s.id]
|
||||
|
||||
instance_types = [try(local.node_pools.validators.instance_type, "t3.medium")]
|
||||
capacity_type = "ON_DEMAND"
|
||||
|
||||
scaling_config {
|
||||
desired_size = try(local.node_pools.validators.count, 1)
|
||||
max_size = try(local.node_pools.validators.count, 1) * 2
|
||||
min_size = 1
|
||||
}
|
||||
|
||||
labels = {
|
||||
pool = "validators"
|
||||
role = "validator"
|
||||
}
|
||||
|
||||
taint {
|
||||
key = "role"
|
||||
value = "validator"
|
||||
effect = "NO_SCHEDULE"
|
||||
}
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-node-validators"
|
||||
Pool = "validators"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_eks_node_group" "rpc" {
|
||||
count = try(local.node_pools.rpc.count, 0) > 0 ? 1 : 0
|
||||
|
||||
cluster_name = aws_eks_cluster.main.name
|
||||
node_group_name = "rpc"
|
||||
node_role_arn = aws_iam_role.eks_node.arn
|
||||
subnet_ids = [for s in aws_subnet.rpc : s.id]
|
||||
|
||||
instance_types = [try(local.node_pools.rpc.instance_type, "t3.medium")]
|
||||
capacity_type = "ON_DEMAND"
|
||||
|
||||
scaling_config {
|
||||
desired_size = try(local.node_pools.rpc.count, 1)
|
||||
max_size = try(local.node_pools.rpc.count, 1) * 2
|
||||
min_size = 1
|
||||
}
|
||||
|
||||
labels = {
|
||||
pool = "rpc"
|
||||
role = "rpc"
|
||||
}
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Name = "${local.name_prefix}-node-rpc"
|
||||
Pool = "rpc"
|
||||
})
|
||||
}
|
||||
|
||||
# Data sources
|
||||
data "aws_availability_zones" "available" {
|
||||
state = "available"
|
||||
}
|
||||
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
# IAM Roles and Policies
|
||||
resource "aws_iam_role" "eks_cluster" {
|
||||
name = "${local.name_prefix}-eks-cluster-role"
|
||||
|
||||
assume_role_policy = jsonencode({
|
||||
Version = "2012-10-17"
|
||||
Statement = [{
|
||||
Action = "sts:AssumeRole"
|
||||
Effect = "Allow"
|
||||
Principal = {
|
||||
Service = "eks.amazonaws.com"
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "eks_cluster_policy" {
|
||||
policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
|
||||
role = aws_iam_role.eks_cluster.name
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "eks_node" {
|
||||
name = "${local.name_prefix}-eks-node-role"
|
||||
|
||||
assume_role_policy = jsonencode({
|
||||
Version = "2012-10-17"
|
||||
Statement = [{
|
||||
Action = "sts:AssumeRole"
|
||||
Effect = "Allow"
|
||||
Principal = {
|
||||
Service = "ec2.amazonaws.com"
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "eks_worker_node_policy" {
|
||||
policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
|
||||
role = aws_iam_role.eks_node.name
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "eks_cni_policy" {
|
||||
policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
|
||||
role = aws_iam_role.eks_node.name
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "eks_container_registry_policy" {
|
||||
policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
|
||||
role = aws_iam_role.eks_node.name
|
||||
}
|
||||
|
||||
# CloudWatch Log Group
|
||||
resource "aws_cloudwatch_log_group" "eks" {
|
||||
name = "/aws/eks/${local.name_prefix}-eks/cluster"
|
||||
retention_in_days = 7
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
44
terraform/multi-cloud/modules/aws/outputs.tf
Normal file
44
terraform/multi-cloud/modules/aws/outputs.tf
Normal file
@@ -0,0 +1,44 @@
|
||||
# Outputs for AWS Module
|
||||
|
||||
output "vpc_id" {
|
||||
value = aws_vpc.main.id
|
||||
description = "VPC ID"
|
||||
}
|
||||
|
||||
output "cluster_name" {
|
||||
value = aws_eks_cluster.main.name
|
||||
description = "EKS cluster name"
|
||||
}
|
||||
|
||||
output "cluster_endpoint" {
|
||||
value = aws_eks_cluster.main.endpoint
|
||||
description = "EKS cluster endpoint"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
output "cluster_certificate_authority_data" {
|
||||
value = aws_eks_cluster.main.certificate_authority[0].data
|
||||
description = "EKS cluster certificate authority data"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
output "cluster_security_group_id" {
|
||||
value = aws_security_group.eks_cluster.id
|
||||
description = "EKS cluster security group ID"
|
||||
}
|
||||
|
||||
output "subnet_ids" {
|
||||
value = {
|
||||
eks = [for s in aws_subnet.eks : s.id]
|
||||
validators = [for s in aws_subnet.validators : s.id]
|
||||
rpc = [for s in aws_subnet.rpc : s.id]
|
||||
}
|
||||
description = "Subnet IDs by type"
|
||||
}
|
||||
|
||||
output "kubeconfig" {
|
||||
value = null # Will be generated by external data source or script
|
||||
description = "Kubeconfig for the cluster"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
50
terraform/multi-cloud/modules/aws/variables.tf
Normal file
50
terraform/multi-cloud/modules/aws/variables.tf
Normal file
@@ -0,0 +1,50 @@
|
||||
# Variables for AWS Module
|
||||
|
||||
variable "environment_config" {
|
||||
description = "Environment configuration from environments.yaml"
|
||||
type = object({
|
||||
name = string
|
||||
role = string
|
||||
provider = string
|
||||
type = string
|
||||
region = string
|
||||
location = string
|
||||
enabled = bool
|
||||
components = list(string)
|
||||
infrastructure = object({
|
||||
kubernetes = object({
|
||||
provider = string
|
||||
version = string
|
||||
node_pools = map(object({
|
||||
count = number
|
||||
instance_type = string
|
||||
}))
|
||||
})
|
||||
networking = object({
|
||||
vpc_cidr = string
|
||||
subnets = list(object({
|
||||
name = string
|
||||
cidr = string
|
||||
availability_zone = optional(string)
|
||||
}))
|
||||
})
|
||||
})
|
||||
aws = object({
|
||||
account_id = string
|
||||
region = string
|
||||
vpc_id = string
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
description = "Environment name"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Tags to apply to resources"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
114
terraform/multi-cloud/modules/azure-arc/main.tf
Normal file
114
terraform/multi-cloud/modules/azure-arc/main.tf
Normal file
@@ -0,0 +1,114 @@
|
||||
# Azure Arc Integration Module
|
||||
# Onboards Kubernetes clusters from any provider to Azure Arc for unified management
|
||||
|
||||
locals {
|
||||
# Resource group for Arc resources
|
||||
resource_group_name = var.resource_group_name
|
||||
location = var.location
|
||||
}
|
||||
|
||||
# Resource Group for Arc resources
|
||||
resource "azurerm_resource_group" "arc" {
|
||||
name = local.resource_group_name
|
||||
location = local.location
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# Azure Arc Connected Kubernetes Cluster resources
|
||||
# Note: Actual onboarding is done via az connectedk8s connect command
|
||||
# This resource represents the Arc resource in Azure
|
||||
|
||||
resource "azapi_resource" "arc_clusters" {
|
||||
for_each = var.clusters
|
||||
|
||||
type = "Microsoft.Kubernetes/connectedClusters@2023-11-01-preview"
|
||||
name = "${each.key}-arc"
|
||||
location = local.location
|
||||
|
||||
parent_id = azurerm_resource_group.arc.id
|
||||
|
||||
body = jsonencode({
|
||||
properties = {
|
||||
agentPublicKeyCertificate = "" # Populated during onboarding
|
||||
distribution = each.value.provider == "aws" ? "EKS" : (
|
||||
each.value.provider == "gcp" ? "GKE" : (
|
||||
each.value.provider == "onprem" ? "AKS" : "AKS"
|
||||
)
|
||||
)
|
||||
infrastructure = each.value.provider
|
||||
kubernetesVersion = "" # Will be populated
|
||||
totalNodeCount = 0 # Will be populated
|
||||
}
|
||||
})
|
||||
|
||||
tags = merge(var.tags, {
|
||||
Provider = each.value.provider
|
||||
Region = each.value.region
|
||||
Cluster = each.value.name
|
||||
})
|
||||
}
|
||||
|
||||
# Azure Arc extensions (optional - for GitOps, monitoring, etc.)
|
||||
resource "azapi_resource" "arc_gitops" {
|
||||
for_each = {
|
||||
for k, v in var.clusters : k => v
|
||||
if var.enable_gitops
|
||||
}
|
||||
|
||||
type = "Microsoft.KubernetesConfiguration/extensions@2022-11-01"
|
||||
name = "arc-gitops-${each.key}"
|
||||
parent_id = azapi_resource.arc_clusters[each.key].id
|
||||
|
||||
body = jsonencode({
|
||||
properties = {
|
||||
extensionType = "microsoft.flux"
|
||||
autoUpgradeMinorVersion = true
|
||||
releaseTrain = "Stable"
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [azapi_resource.arc_clusters]
|
||||
}
|
||||
|
||||
# Output script for onboarding clusters
|
||||
resource "local_file" "arc_onboarding_script" {
|
||||
for_each = var.clusters
|
||||
|
||||
filename = "${path.module}/../../../../scripts/arc-onboard-${each.key}.sh"
|
||||
content = <<-EOT
|
||||
#!/bin/bash
|
||||
# Azure Arc Onboarding Script for ${each.key}
|
||||
# Cluster: ${each.value.name}
|
||||
# Provider: ${each.value.provider}
|
||||
# Region: ${each.value.region}
|
||||
|
||||
set -e
|
||||
|
||||
# Install Azure CLI extension for Arc
|
||||
az extension add --name connectedk8s || az extension update --name connectedk8s
|
||||
|
||||
# Login to Azure (if not already)
|
||||
# az login
|
||||
|
||||
# Set subscription
|
||||
az account set --subscription "${var.azure_subscription_id}"
|
||||
|
||||
# Connect cluster to Azure Arc
|
||||
az connectedk8s connect \
|
||||
--name "${each.key}-arc" \
|
||||
--resource-group "${local.resource_group_name}" \
|
||||
--location "${local.location}" \
|
||||
--kube-config "${each.value.kubeconfig}" \
|
||||
--kube-context "" \
|
||||
--tags \
|
||||
Provider=${each.value.provider} \
|
||||
Region=${each.value.region} \
|
||||
Cluster=${each.value.name}
|
||||
|
||||
echo "Cluster ${each.key} onboarded to Azure Arc successfully!"
|
||||
EOT
|
||||
|
||||
file_permission = "0755"
|
||||
}
|
||||
|
||||
45
terraform/multi-cloud/modules/azure-arc/variables.tf
Normal file
45
terraform/multi-cloud/modules/azure-arc/variables.tf
Normal file
@@ -0,0 +1,45 @@
|
||||
# Variables for Azure Arc Module
|
||||
|
||||
variable "clusters" {
|
||||
description = "Map of clusters to onboard to Azure Arc"
|
||||
type = map(object({
|
||||
name = string
|
||||
provider = string
|
||||
region = string
|
||||
kubeconfig = string
|
||||
}))
|
||||
}
|
||||
|
||||
variable "azure_subscription_id" {
|
||||
description = "Azure subscription ID"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "azure_tenant_id" {
|
||||
description = "Azure tenant ID"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "resource_group_name" {
|
||||
description = "Resource group name for Arc resources"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "location" {
|
||||
description = "Azure region for Arc resources"
|
||||
type = string
|
||||
default = "westus"
|
||||
}
|
||||
|
||||
variable "enable_gitops" {
|
||||
description = "Enable GitOps extension for Arc clusters"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Tags to apply to resources"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
123
terraform/multi-cloud/modules/azure/main.tf
Normal file
123
terraform/multi-cloud/modules/azure/main.tf
Normal file
@@ -0,0 +1,123 @@
|
||||
# Azure Infrastructure Module
|
||||
# Adapts existing Azure modules for multi-cloud architecture
|
||||
|
||||
locals {
|
||||
env = var.environment_config
|
||||
|
||||
# Extract Azure-specific config
|
||||
azure_config = try(local.env.azure, {})
|
||||
|
||||
# Extract infrastructure config
|
||||
infra = try(local.env.infrastructure, {})
|
||||
k8s_config = try(local.infra.kubernetes, {})
|
||||
net_config = try(local.infra.networking, {})
|
||||
|
||||
# Naming
|
||||
name_prefix = "${local.env.name}-${var.environment}"
|
||||
|
||||
# Node pools
|
||||
node_pools = try(local.k8s_config.node_pools, {})
|
||||
|
||||
# Region
|
||||
location = try(local.env.region, "westeurope")
|
||||
}
|
||||
|
||||
# Resource Group
|
||||
resource "azurerm_resource_group" "main" {
|
||||
name = try(local.azure_config.resource_group_name, "${local.name_prefix}-rg")
|
||||
location = local.location
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# Network Module (reuse existing)
|
||||
module "networking" {
|
||||
source = "../../modules/networking"
|
||||
|
||||
resource_group_name = azurerm_resource_group.main.name
|
||||
location = local.location
|
||||
cluster_name = "${local.name_prefix}-aks"
|
||||
environment = var.environment
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# Key Vault Module (reuse existing)
|
||||
module "keyvault" {
|
||||
source = "../../modules/secrets"
|
||||
|
||||
resource_group_name = azurerm_resource_group.main.name
|
||||
location = local.location
|
||||
key_vault_name = try(local.env.secrets.key_vault_name, "${local.name_prefix}-kv")
|
||||
environment = var.environment
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# AKS Module (reuse existing, with modifications)
|
||||
module "aks" {
|
||||
source = "../../modules/kubernetes"
|
||||
|
||||
resource_group_name = azurerm_resource_group.main.name
|
||||
location = local.location
|
||||
cluster_name = "${local.name_prefix}-aks"
|
||||
kubernetes_version = try(local.k8s_config.version, "1.28")
|
||||
|
||||
# Convert node_pools config to node_count and vm_size format
|
||||
node_count = {
|
||||
system = try(local.node_pools.system.count, 1)
|
||||
validators = try(local.node_pools.validators.count, 0)
|
||||
sentries = try(local.node_pools.sentries.count, 0)
|
||||
rpc = try(local.node_pools.rpc.count, 0)
|
||||
}
|
||||
|
||||
vm_size = {
|
||||
system = try(local.node_pools.system.vm_size, "Standard_D2s_v3")
|
||||
validators = try(local.node_pools.validators.vm_size, "Standard_D4s_v3")
|
||||
sentries = try(local.node_pools.sentries.vm_size, "Standard_D4s_v3")
|
||||
rpc = try(local.node_pools.rpc.vm_size, "Standard_D8s_v3")
|
||||
}
|
||||
|
||||
environment = var.environment
|
||||
tags = var.tags
|
||||
|
||||
vnet_subnet_id = module.networking.aks_subnet_id
|
||||
node_subnet_id = module.networking.node_subnet_id
|
||||
key_vault_id = module.keyvault.key_vault_id
|
||||
|
||||
depends_on = [
|
||||
module.networking,
|
||||
module.keyvault
|
||||
]
|
||||
}
|
||||
|
||||
# Storage Module (reuse existing)
|
||||
module "storage" {
|
||||
source = "../../modules/storage"
|
||||
|
||||
resource_group_name = azurerm_resource_group.main.name
|
||||
location = local.location
|
||||
cluster_name = "${local.name_prefix}-aks"
|
||||
environment = var.environment
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# Azure Arc onboarding (if enabled)
|
||||
resource "azapi_resource" "arc_cluster" {
|
||||
count = try(local.env.azure.arc_enabled, false) ? 1 : 0
|
||||
|
||||
type = "Microsoft.Kubernetes/connectedClusters@2023-11-01-preview"
|
||||
name = "${local.name_prefix}-arc"
|
||||
location = local.location
|
||||
|
||||
parent_id = azurerm_resource_group.main.id
|
||||
|
||||
body = jsonencode({
|
||||
properties = {
|
||||
agentPublicKeyCertificate = "" # Will be populated by Arc agent
|
||||
distribution = "AKS"
|
||||
infrastructure = "azure"
|
||||
}
|
||||
})
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
33
terraform/multi-cloud/modules/azure/outputs.tf
Normal file
33
terraform/multi-cloud/modules/azure/outputs.tf
Normal file
@@ -0,0 +1,33 @@
|
||||
# Outputs for Azure Module
|
||||
|
||||
output "resource_group_name" {
|
||||
value = azurerm_resource_group.main.name
|
||||
description = "Resource group name"
|
||||
}
|
||||
|
||||
output "cluster_name" {
|
||||
value = module.aks.cluster_name
|
||||
description = "AKS cluster name"
|
||||
}
|
||||
|
||||
output "cluster_endpoint" {
|
||||
value = module.aks.cluster_fqdn
|
||||
description = "AKS cluster endpoint"
|
||||
}
|
||||
|
||||
output "region" {
|
||||
value = local.location
|
||||
description = "Azure region"
|
||||
}
|
||||
|
||||
output "kubeconfig" {
|
||||
value = module.aks.kubeconfig
|
||||
description = "Kubeconfig for the cluster"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
output "vnet_id" {
|
||||
value = module.networking.vnet_id
|
||||
description = "Virtual network ID"
|
||||
}
|
||||
|
||||
30
terraform/multi-cloud/modules/azure/variables.tf
Normal file
30
terraform/multi-cloud/modules/azure/variables.tf
Normal file
@@ -0,0 +1,30 @@
|
||||
# Variables for Azure Module
|
||||
|
||||
variable "environment_config" {
|
||||
description = "Environment configuration from environments.yaml"
|
||||
type = any
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
description = "Environment name"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "subscription_id" {
|
||||
description = "Azure subscription ID"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "tenant_id" {
|
||||
description = "Azure tenant ID"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Tags to apply to resources"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
367
terraform/multi-cloud/modules/gcp/main.tf
Normal file
367
terraform/multi-cloud/modules/gcp/main.tf
Normal file
@@ -0,0 +1,367 @@
|
||||
# Google Cloud Platform Infrastructure Module
|
||||
# Creates GKE cluster, networking, and supporting resources for GCP environments
|
||||
|
||||
locals {
|
||||
env = var.environment_config
|
||||
|
||||
# Extract GCP-specific config
|
||||
gcp_config = try(local.env.gcp, {})
|
||||
|
||||
# Extract infrastructure config
|
||||
infra = try(local.env.infrastructure, {})
|
||||
k8s_config = try(local.infra.kubernetes, {})
|
||||
net_config = try(local.infra.networking, {})
|
||||
|
||||
# Naming
|
||||
name_prefix = "${local.env.name}-${var.environment}"
|
||||
|
||||
# Node pools
|
||||
node_pools = try(local.k8s_config.node_pools, {})
|
||||
|
||||
# Project and region
|
||||
project_id = try(local.gcp_config.project_id, var.gcp_project_id)
|
||||
region = try(local.env.region, var.gcp_default_region)
|
||||
}
|
||||
|
||||
# VPC Network
|
||||
resource "google_compute_network" "main" {
|
||||
name = "${local.name_prefix}-vpc"
|
||||
auto_create_subnetworks = false
|
||||
|
||||
project = local.project_id
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# Subnets
|
||||
resource "google_compute_subnetwork" "gke" {
|
||||
for_each = {
|
||||
for idx, subnet in try(local.net_config.subnets, []) : subnet.name => subnet
|
||||
if subnet.name == "gke" || subnet.name == "eks" || subnet.name == "aks"
|
||||
}
|
||||
|
||||
name = "${local.name_prefix}-subnet-${each.key}"
|
||||
ip_cidr_range = each.value.cidr
|
||||
region = try(each.value.region, local.region)
|
||||
network = google_compute_network.main.id
|
||||
project = local.project_id
|
||||
|
||||
private_ip_google_access = true
|
||||
|
||||
secondary_ip_range {
|
||||
range_name = "${local.name_prefix}-pods"
|
||||
ip_cidr_range = cidrsubnet(each.value.cidr, 8, 1)
|
||||
}
|
||||
|
||||
secondary_ip_range {
|
||||
range_name = "${local.name_prefix}-services"
|
||||
ip_cidr_range = cidrsubnet(each.value.cidr, 8, 2)
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_compute_subnetwork" "validators" {
|
||||
for_each = {
|
||||
for idx, subnet in try(local.net_config.subnets, []) : subnet.name => subnet
|
||||
if subnet.name == "validators"
|
||||
}
|
||||
|
||||
name = "${local.name_prefix}-subnet-validators"
|
||||
ip_cidr_range = each.value.cidr
|
||||
region = try(each.value.region, local.region)
|
||||
network = google_compute_network.main.id
|
||||
project = local.project_id
|
||||
|
||||
private_ip_google_access = true
|
||||
}
|
||||
|
||||
resource "google_compute_subnetwork" "rpc" {
|
||||
for_each = {
|
||||
for idx, subnet in try(local.net_config.subnets, []) : subnet.name => subnet
|
||||
if subnet.name == "rpc"
|
||||
}
|
||||
|
||||
name = "${local.name_prefix}-subnet-rpc"
|
||||
ip_cidr_range = each.value.cidr
|
||||
region = try(each.value.region, local.region)
|
||||
network = google_compute_network.main.id
|
||||
project = local.project_id
|
||||
|
||||
private_ip_google_access = true
|
||||
}
|
||||
|
||||
# Firewall Rules
|
||||
resource "google_compute_firewall" "allow_internal" {
|
||||
name = "${local.name_prefix}-allow-internal"
|
||||
network = google_compute_network.main.name
|
||||
project = local.project_id
|
||||
|
||||
allow {
|
||||
protocol = "icmp"
|
||||
}
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = ["0-65535"]
|
||||
}
|
||||
|
||||
allow {
|
||||
protocol = "udp"
|
||||
ports = ["0-65535"]
|
||||
}
|
||||
|
||||
source_ranges = [google_compute_network.main.ipv4_range]
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "allow_validators_p2p" {
|
||||
name = "${local.name_prefix}-allow-validators-p2p"
|
||||
network = google_compute_network.main.name
|
||||
project = local.project_id
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = ["30303"]
|
||||
}
|
||||
|
||||
source_ranges = [google_compute_network.main.ipv4_range]
|
||||
target_tags = ["validators"]
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "allow_rpc" {
|
||||
name = "${local.name_prefix}-allow-rpc"
|
||||
network = google_compute_network.main.name
|
||||
project = local.project_id
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = ["8545", "8546"]
|
||||
}
|
||||
|
||||
source_ranges = ["0.0.0.0/0"]
|
||||
target_tags = ["rpc"]
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# GKE Cluster
|
||||
resource "google_container_cluster" "main" {
|
||||
name = "${local.name_prefix}-gke"
|
||||
location = local.region
|
||||
project = local.project_id
|
||||
|
||||
# We can't create a cluster with no node pool defined, but we want to only use
|
||||
# separately managed node pools. So we create the smallest possible default
|
||||
# node pool and immediately delete it.
|
||||
remove_default_node_pool = true
|
||||
initial_node_count = 1
|
||||
|
||||
network = google_compute_network.main.name
|
||||
subnetwork = [for s in google_compute_subnetwork.gke : s.name][0]
|
||||
|
||||
# Enable Workload Identity
|
||||
workload_identity_config {
|
||||
workload_pool = "${local.project_id}.svc.id.goog"
|
||||
}
|
||||
|
||||
# Enable private cluster
|
||||
private_cluster_config {
|
||||
enable_private_nodes = true
|
||||
enable_private_endpoint = false
|
||||
master_ipv4_cidr_block = "172.16.0.0/28"
|
||||
}
|
||||
|
||||
# Enable logging and monitoring
|
||||
logging_service = "logging.googleapis.com/kubernetes"
|
||||
monitoring_service = "monitoring.googleapis.com/kubernetes"
|
||||
|
||||
# Network policy
|
||||
network_policy {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
# Release channel
|
||||
release_channel {
|
||||
channel = "REGULAR"
|
||||
}
|
||||
|
||||
# Maintenance window
|
||||
maintenance_policy {
|
||||
daily_maintenance_window {
|
||||
start_time = "03:00"
|
||||
}
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
google_project_service.container,
|
||||
google_project_service.compute,
|
||||
]
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# Delete default node pool
|
||||
resource "google_container_node_pool" "default" {
|
||||
name = "${local.name_prefix}-default-pool"
|
||||
location = local.region
|
||||
cluster = google_container_cluster.main.name
|
||||
project = local.project_id
|
||||
node_count = 0 # Immediately scale to 0 to effectively delete it
|
||||
|
||||
management {
|
||||
auto_repair = true
|
||||
auto_upgrade = true
|
||||
}
|
||||
}
|
||||
|
||||
# Node Pools
|
||||
resource "google_container_node_pool" "system" {
|
||||
count = try(local.node_pools.system.count, 0) > 0 ? 1 : 0
|
||||
|
||||
name = "system"
|
||||
location = local.region
|
||||
cluster = google_container_cluster.main.name
|
||||
project = local.project_id
|
||||
node_count = try(local.node_pools.system.count, 1)
|
||||
|
||||
node_config {
|
||||
machine_type = try(local.node_pools.system.machine_type, "e2-medium")
|
||||
disk_size_gb = 100
|
||||
disk_type = "pd-standard"
|
||||
service_account = google_service_account.gke_nodes.email
|
||||
|
||||
oauth_scopes = [
|
||||
"https://www.googleapis.com/auth/cloud-platform",
|
||||
]
|
||||
|
||||
labels = {
|
||||
pool = "system"
|
||||
role = "system"
|
||||
}
|
||||
|
||||
tags = ["system"]
|
||||
}
|
||||
|
||||
management {
|
||||
auto_repair = true
|
||||
auto_upgrade = true
|
||||
}
|
||||
|
||||
autoscaling {
|
||||
min_node_count = 1
|
||||
max_node_count = try(local.node_pools.system.count, 1) * 2
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_container_node_pool" "validators" {
|
||||
count = try(local.node_pools.validators.count, 0) > 0 ? 1 : 0
|
||||
|
||||
name = "validators"
|
||||
location = local.region
|
||||
cluster = google_container_cluster.main.name
|
||||
project = local.project_id
|
||||
node_count = try(local.node_pools.validators.count, 1)
|
||||
|
||||
node_config {
|
||||
machine_type = try(local.node_pools.validators.machine_type, "e2-medium")
|
||||
disk_size_gb = 512
|
||||
disk_type = "pd-ssd"
|
||||
service_account = google_service_account.gke_nodes.email
|
||||
|
||||
oauth_scopes = [
|
||||
"https://www.googleapis.com/auth/cloud-platform",
|
||||
]
|
||||
|
||||
labels = {
|
||||
pool = "validators"
|
||||
role = "validator"
|
||||
}
|
||||
|
||||
taint {
|
||||
key = "role"
|
||||
value = "validator"
|
||||
effect = "NO_SCHEDULE"
|
||||
}
|
||||
|
||||
tags = ["validators"]
|
||||
}
|
||||
|
||||
management {
|
||||
auto_repair = true
|
||||
auto_upgrade = true
|
||||
}
|
||||
|
||||
autoscaling {
|
||||
min_node_count = 1
|
||||
max_node_count = try(local.node_pools.validators.count, 1) * 2
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_container_node_pool" "rpc" {
|
||||
count = try(local.node_pools.rpc.count, 0) > 0 ? 1 : 0
|
||||
|
||||
name = "rpc"
|
||||
location = local.region
|
||||
cluster = google_container_cluster.main.name
|
||||
project = local.project_id
|
||||
node_count = try(local.node_pools.rpc.count, 1)
|
||||
|
||||
node_config {
|
||||
machine_type = try(local.node_pools.rpc.machine_type, "e2-medium")
|
||||
disk_size_gb = 256
|
||||
disk_type = "pd-ssd"
|
||||
service_account = google_service_account.gke_nodes.email
|
||||
|
||||
oauth_scopes = [
|
||||
"https://www.googleapis.com/auth/cloud-platform",
|
||||
]
|
||||
|
||||
labels = {
|
||||
pool = "rpc"
|
||||
role = "rpc"
|
||||
}
|
||||
|
||||
tags = ["rpc"]
|
||||
}
|
||||
|
||||
management {
|
||||
auto_repair = true
|
||||
auto_upgrade = true
|
||||
}
|
||||
|
||||
autoscaling {
|
||||
min_node_count = 1
|
||||
max_node_count = try(local.node_pools.rpc.count, 1) * 2
|
||||
}
|
||||
}
|
||||
|
||||
# Service Account for GKE nodes
|
||||
resource "google_service_account" "gke_nodes" {
|
||||
account_id = "${local.name_prefix}-gke-nodes"
|
||||
display_name = "GKE Nodes Service Account"
|
||||
project = local.project_id
|
||||
}
|
||||
|
||||
resource "google_project_iam_member" "gke_nodes" {
|
||||
project = local.project_id
|
||||
role = "roles/container.nodeServiceAccount"
|
||||
member = "serviceAccount:${google_service_account.gke_nodes.email}"
|
||||
}
|
||||
|
||||
# Enable required APIs
|
||||
resource "google_project_service" "container" {
|
||||
project = local.project_id
|
||||
service = "container.googleapis.com"
|
||||
|
||||
disable_on_destroy = false
|
||||
}
|
||||
|
||||
resource "google_project_service" "compute" {
|
||||
project = local.project_id
|
||||
service = "compute.googleapis.com"
|
||||
|
||||
disable_on_destroy = false
|
||||
}
|
||||
|
||||
39
terraform/multi-cloud/modules/gcp/outputs.tf
Normal file
39
terraform/multi-cloud/modules/gcp/outputs.tf
Normal file
@@ -0,0 +1,39 @@
|
||||
# Outputs for GCP Module
|
||||
|
||||
output "vpc_id" {
|
||||
value = google_compute_network.main.id
|
||||
description = "VPC network ID"
|
||||
}
|
||||
|
||||
output "cluster_name" {
|
||||
value = google_container_cluster.main.name
|
||||
description = "GKE cluster name"
|
||||
}
|
||||
|
||||
output "cluster_endpoint" {
|
||||
value = google_container_cluster.main.endpoint
|
||||
description = "GKE cluster endpoint"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
output "cluster_ca_certificate" {
|
||||
value = google_container_cluster.main.master_auth[0].cluster_ca_certificate
|
||||
description = "GKE cluster CA certificate"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
output "subnet_ids" {
|
||||
value = {
|
||||
gke = [for s in google_compute_subnetwork.gke : s.id]
|
||||
validators = [for s in google_compute_subnetwork.validators : s.id]
|
||||
rpc = [for s in google_compute_subnetwork.rpc : s.id]
|
||||
}
|
||||
description = "Subnet IDs by type"
|
||||
}
|
||||
|
||||
output "kubeconfig" {
|
||||
value = null # Will be generated by external data source or script
|
||||
description = "Kubeconfig for the cluster"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
62
terraform/multi-cloud/modules/gcp/variables.tf
Normal file
62
terraform/multi-cloud/modules/gcp/variables.tf
Normal file
@@ -0,0 +1,62 @@
|
||||
# Variables for GCP Module
|
||||
|
||||
variable "environment_config" {
|
||||
description = "Environment configuration from environments.yaml"
|
||||
type = object({
|
||||
name = string
|
||||
role = string
|
||||
provider = string
|
||||
type = string
|
||||
region = string
|
||||
location = string
|
||||
enabled = bool
|
||||
components = list(string)
|
||||
infrastructure = object({
|
||||
kubernetes = object({
|
||||
provider = string
|
||||
version = string
|
||||
node_pools = map(object({
|
||||
count = number
|
||||
machine_type = string
|
||||
}))
|
||||
})
|
||||
networking = object({
|
||||
vpc_cidr = string
|
||||
subnets = list(object({
|
||||
name = string
|
||||
cidr = string
|
||||
region = optional(string)
|
||||
}))
|
||||
})
|
||||
})
|
||||
gcp = object({
|
||||
project_id = string
|
||||
region = string
|
||||
zone = string
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
description = "Environment name"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "gcp_project_id" {
|
||||
description = "GCP project ID (fallback if not in config)"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "gcp_default_region" {
|
||||
description = "GCP default region (fallback if not in config)"
|
||||
type = string
|
||||
default = "europe-west1"
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Tags to apply to resources"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
150
terraform/multi-cloud/modules/observability/main.tf
Normal file
150
terraform/multi-cloud/modules/observability/main.tf
Normal file
@@ -0,0 +1,150 @@
|
||||
# Observability Module
|
||||
# Provides unified logging, metrics, and tracing across all environments
|
||||
|
||||
locals {
|
||||
environments = var.environments
|
||||
global_config = var.global_config
|
||||
clusters = var.clusters
|
||||
|
||||
# Extract observability config
|
||||
obs_config = try(local.global_config, {})
|
||||
logging_config = try(local.obs_config.logging, {})
|
||||
metrics_config = try(local.obs_config.metrics, {})
|
||||
tracing_config = try(local.obs_config.tracing, {})
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# LOGGING CONFIGURATION
|
||||
# ============================================
|
||||
# Generate logging configuration per environment
|
||||
resource "local_file" "logging_config" {
|
||||
for_each = local.clusters
|
||||
|
||||
filename = "${path.module}/../../../../config/observability/logging-${each.key}.yaml"
|
||||
content = yamlencode({
|
||||
cluster = each.key
|
||||
provider = try(local.logging_config.provider, "loki")
|
||||
endpoint = try(local.logging_config.central_endpoint, "")
|
||||
config = {
|
||||
loki = {
|
||||
url = try(local.logging_config.central_endpoint, "http://loki:3100")
|
||||
labels = {
|
||||
cluster = each.key
|
||||
provider = try(local.environments[each.key].provider, "unknown")
|
||||
}
|
||||
}
|
||||
elasticsearch = {
|
||||
url = try(local.logging_config.central_endpoint, "")
|
||||
index = "besu-network-${each.key}"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# METRICS CONFIGURATION
|
||||
# ============================================
|
||||
# Generate Prometheus/metrics configuration
|
||||
resource "local_file" "metrics_config" {
|
||||
for_each = local.clusters
|
||||
|
||||
filename = "${path.module}/../../../../config/observability/metrics-${each.key}.yaml"
|
||||
content = yamlencode({
|
||||
cluster = each.key
|
||||
provider = try(local.metrics_config.provider, "prometheus")
|
||||
endpoint = try(local.metrics_config.central_endpoint, "")
|
||||
scrape_configs = [
|
||||
{
|
||||
job_name = "besu-validators"
|
||||
kubernetes_sd_configs = [{
|
||||
role = "pod"
|
||||
}]
|
||||
relabel_configs = [{
|
||||
source_labels = ["__meta_kubernetes_pod_label_role"]
|
||||
action = "keep"
|
||||
regex = "validator"
|
||||
}]
|
||||
},
|
||||
{
|
||||
job_name = "besu-rpc"
|
||||
kubernetes_sd_configs = [{
|
||||
role = "pod"
|
||||
}]
|
||||
relabel_configs = [{
|
||||
source_labels = ["__meta_kubernetes_pod_label_role"]
|
||||
action = "keep"
|
||||
regex = "rpc"
|
||||
}]
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# TRACING CONFIGURATION
|
||||
# ============================================
|
||||
# Generate tracing configuration
|
||||
resource "local_file" "tracing_config" {
|
||||
for_each = local.clusters
|
||||
|
||||
filename = "${path.module}/../../../../config/observability/tracing-${each.key}.yaml"
|
||||
content = yamlencode({
|
||||
cluster = each.key
|
||||
provider = try(local.tracing_config.provider, "jaeger")
|
||||
endpoint = try(local.tracing_config.central_endpoint, "")
|
||||
config = {
|
||||
jaeger = {
|
||||
agent_host = try(split(":", local.tracing_config.central_endpoint)[0], "jaeger")
|
||||
agent_port = try(split(":", local.tracing_config.central_endpoint)[1], "6831")
|
||||
}
|
||||
zipkin = {
|
||||
url = try(local.tracing_config.central_endpoint, "http://zipkin:9411")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# GRAFANA DASHBOARDS
|
||||
# ============================================
|
||||
# Generate Grafana dashboard configuration
|
||||
resource "local_file" "grafana_dashboards" {
|
||||
filename = "${path.module}/../../../../config/observability/grafana-dashboards.yaml"
|
||||
content = yamlencode({
|
||||
dashboards = {
|
||||
for name, cluster in local.clusters : name => {
|
||||
title = "Besu Network - ${name}"
|
||||
panels = [
|
||||
{
|
||||
title = "Block Production Rate"
|
||||
targets = [
|
||||
{
|
||||
expr = "rate(besu_blockchain_blocks_added_total[5m])"
|
||||
legendFormat = "{{cluster}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
title = "Validator Health"
|
||||
targets = [
|
||||
{
|
||||
expr = "besu_validator_health"
|
||||
legendFormat = "{{validator}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
title = "RPC Requests"
|
||||
targets = [
|
||||
{
|
||||
expr = "rate(besu_rpc_requests_total[5m])"
|
||||
legendFormat = "{{method}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
27
terraform/multi-cloud/modules/observability/variables.tf
Normal file
27
terraform/multi-cloud/modules/observability/variables.tf
Normal file
@@ -0,0 +1,27 @@
|
||||
# Variables for Observability Module
|
||||
|
||||
variable "environments" {
|
||||
description = "Map of all environments"
|
||||
type = map(any)
|
||||
}
|
||||
|
||||
variable "global_config" {
|
||||
description = "Global observability configuration"
|
||||
type = map(any)
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "clusters" {
|
||||
description = "Map of clusters for observability"
|
||||
type = map(object({
|
||||
endpoint = string
|
||||
kubeconfig = string
|
||||
}))
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Tags to apply to resources"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
231
terraform/multi-cloud/modules/onprem-hci/main.tf
Normal file
231
terraform/multi-cloud/modules/onprem-hci/main.tf
Normal file
@@ -0,0 +1,231 @@
|
||||
# On-Premises HCI Infrastructure Module
|
||||
# Supports Azure Stack HCI and vSphere-based HCI clusters
|
||||
|
||||
locals {
|
||||
env = var.environment_config
|
||||
|
||||
# Extract on-prem config
|
||||
onprem_config = try(local.env.onprem, {})
|
||||
hci_config = try(local.env.azure_stack_hci, {})
|
||||
|
||||
# Extract infrastructure config
|
||||
infra = try(local.env.infrastructure, {})
|
||||
k8s_config = try(local.infra.kubernetes, {})
|
||||
hci_k8s = try(local.k8s_config.hci, {})
|
||||
|
||||
# Naming
|
||||
name_prefix = "${local.env.name}-${var.environment}"
|
||||
|
||||
# HCI platform
|
||||
hci_platform = try(local.onprem_config.hci_platform, "azure-stack-hci")
|
||||
|
||||
# Node pools
|
||||
node_pools = try(local.hci_k8s.node_pools, {})
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# AZURE STACK HCI
|
||||
# ============================================
|
||||
# Azure Stack HCI uses Azure Arc to manage Kubernetes clusters
|
||||
# The cluster is pre-provisioned on-prem, we just onboard it to Azure Arc
|
||||
|
||||
resource "azapi_resource" "azure_stack_hci_cluster" {
|
||||
count = local.hci_platform == "azure-stack-hci" && try(local.hci_config.enabled, false) ? 1 : 0
|
||||
|
||||
type = "Microsoft.Kubernetes/connectedClusters@2023-11-01-preview"
|
||||
name = try(local.hci_config.cluster_name, "${local.name_prefix}-hci")
|
||||
location = try(local.hci_config.location, "westus") # Azure region for Arc resource
|
||||
|
||||
parent_id = "/subscriptions/${var.azure_subscription_id}/resourceGroups/${try(local.hci_config.resource_group, "rg-hci-${var.environment}")}"
|
||||
|
||||
body = jsonencode({
|
||||
properties = {
|
||||
agentPublicKeyCertificate = "" # Will be populated by Arc agent installed on-prem
|
||||
distribution = "AKS"
|
||||
infrastructure = "azure_stack_hci"
|
||||
}
|
||||
})
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# VSPHERE-BASED HCI
|
||||
# ============================================
|
||||
# For vSphere, we provision VMs and install Kubernetes
|
||||
|
||||
# Data sources for vSphere
|
||||
data "vsphere_datacenter" "dc" {
|
||||
count = local.hci_platform == "vsphere" ? 1 : 0
|
||||
name = try(local.onprem_config.datacenter, "datacenter1")
|
||||
}
|
||||
|
||||
data "vsphere_datastore" "datastore" {
|
||||
count = local.hci_platform == "vsphere" ? 1 : 0
|
||||
name = try(local.onprem_config.datastore, "datastore1")
|
||||
datacenter_id = data.vsphere_datacenter.dc[0].id
|
||||
}
|
||||
|
||||
data "vsphere_compute_cluster" "cluster" {
|
||||
count = local.hci_platform == "vsphere" ? 1 : 0
|
||||
name = try(local.onprem_config.cluster, "cluster1")
|
||||
datacenter_id = data.vsphere_datacenter.dc[0].id
|
||||
}
|
||||
|
||||
data "vsphere_network" "network" {
|
||||
count = local.hci_platform == "vsphere" ? 1 : 0
|
||||
name = try(local.onprem_config.network, "VM Network")
|
||||
datacenter_id = data.vsphere_datacenter.dc[0].id
|
||||
}
|
||||
|
||||
data "vsphere_virtual_machine" "template" {
|
||||
count = local.hci_platform == "vsphere" ? 1 : 0
|
||||
name = try(local.onprem_config.vm_template, "ubuntu-22.04-template")
|
||||
datacenter_id = data.vsphere_datacenter.dc[0].id
|
||||
}
|
||||
|
||||
# System nodes
|
||||
resource "vsphere_virtual_machine" "system" {
|
||||
count = local.hci_platform == "vsphere" && try(local.node_pools.system.count, 0) > 0 ? local.node_pools.system.count : 0
|
||||
name = "${local.name_prefix}-system-${count.index + 1}"
|
||||
resource_pool_id = data.vsphere_compute_cluster.cluster[0].resource_pool_id
|
||||
datastore_id = data.vsphere_datastore.datastore[0].id
|
||||
|
||||
num_cpus = 2
|
||||
memory = 4096
|
||||
guest_id = data.vsphere_virtual_machine.template[0].guest_id
|
||||
scsi_type = data.vsphere_virtual_machine.template[0].scsi_type
|
||||
firmware = data.vsphere_virtual_machine.template[0].firmware
|
||||
|
||||
network_interface {
|
||||
network_id = data.vsphere_network.network[0].id
|
||||
}
|
||||
|
||||
disk {
|
||||
label = "disk0"
|
||||
size = 100
|
||||
eagerly_scrub = false
|
||||
thin_provisioned = true
|
||||
}
|
||||
|
||||
clone {
|
||||
template_uuid = data.vsphere_virtual_machine.template[0].id
|
||||
|
||||
customize {
|
||||
linux_options {
|
||||
host_name = "${local.name_prefix}-system-${count.index + 1}"
|
||||
domain = try(local.env.identity.domain, "local")
|
||||
}
|
||||
|
||||
network_interface {
|
||||
ipv4_address = cidrhost(try(local.env.infrastructure.networking.subnet_cidr, "192.168.1.0/24"), count.index + 10)
|
||||
ipv4_netmask = 24
|
||||
}
|
||||
|
||||
ipv4_gateway = try(local.env.infrastructure.networking.gateway, "192.168.1.1")
|
||||
dns_server_list = ["8.8.8.8", "8.8.4.4"]
|
||||
}
|
||||
}
|
||||
|
||||
tags = [for k, v in var.tags : "${k}=${v}"]
|
||||
}
|
||||
|
||||
# Validator nodes
|
||||
resource "vsphere_virtual_machine" "validators" {
|
||||
count = local.hci_platform == "vsphere" && try(local.node_pools.validators.count, 0) > 0 ? local.node_pools.validators.count : 0
|
||||
name = "${local.name_prefix}-validator-${count.index + 1}"
|
||||
resource_pool_id = data.vsphere_compute_cluster.cluster[0].resource_pool_id
|
||||
datastore_id = data.vsphere_datastore.datastore[0].id
|
||||
|
||||
num_cpus = 4
|
||||
memory = 8192
|
||||
guest_id = data.vsphere_virtual_machine.template[0].guest_id
|
||||
scsi_type = data.vsphere_virtual_machine.template[0].scsi_type
|
||||
firmware = data.vsphere_virtual_machine.template[0].firmware
|
||||
|
||||
network_interface {
|
||||
network_id = data.vsphere_network.network[0].id
|
||||
}
|
||||
|
||||
disk {
|
||||
label = "disk0"
|
||||
size = 512
|
||||
eagerly_scrub = false
|
||||
thin_provisioned = true
|
||||
}
|
||||
|
||||
clone {
|
||||
template_uuid = data.vsphere_virtual_machine.template[0].id
|
||||
|
||||
customize {
|
||||
linux_options {
|
||||
host_name = "${local.name_prefix}-validator-${count.index + 1}"
|
||||
domain = try(local.env.identity.domain, "local")
|
||||
}
|
||||
|
||||
network_interface {
|
||||
ipv4_address = cidrhost(try(local.env.infrastructure.networking.subnet_cidr, "192.168.1.0/24"), count.index + 20)
|
||||
ipv4_netmask = 24
|
||||
}
|
||||
|
||||
ipv4_gateway = try(local.env.infrastructure.networking.gateway, "192.168.1.1")
|
||||
dns_server_list = ["8.8.8.8", "8.8.4.4"]
|
||||
}
|
||||
}
|
||||
|
||||
tags = [for k, v in var.tags : "${k}=${v}"]
|
||||
}
|
||||
|
||||
# RPC nodes
|
||||
resource "vsphere_virtual_machine" "rpc" {
|
||||
count = local.hci_platform == "vsphere" && try(local.node_pools.rpc.count, 0) > 0 ? local.node_pools.rpc.count : 0
|
||||
name = "${local.name_prefix}-rpc-${count.index + 1}"
|
||||
resource_pool_id = data.vsphere_compute_cluster.cluster[0].resource_pool_id
|
||||
datastore_id = data.vsphere_datastore.datastore[0].id
|
||||
|
||||
num_cpus = 4
|
||||
memory = 8192
|
||||
guest_id = data.vsphere_virtual_machine.template[0].guest_id
|
||||
scsi_type = data.vsphere_virtual_machine.template[0].scsi_type
|
||||
firmware = data.vsphere_virtual_machine.template[0].firmware
|
||||
|
||||
network_interface {
|
||||
network_id = data.vsphere_network.network[0].id
|
||||
}
|
||||
|
||||
disk {
|
||||
label = "disk0"
|
||||
size = 256
|
||||
eagerly_scrub = false
|
||||
thin_provisioned = true
|
||||
}
|
||||
|
||||
clone {
|
||||
template_uuid = data.vsphere_virtual_machine.template[0].id
|
||||
|
||||
customize {
|
||||
linux_options {
|
||||
host_name = "${local.name_prefix}-rpc-${count.index + 1}"
|
||||
domain = try(local.env.identity.domain, "local")
|
||||
}
|
||||
|
||||
network_interface {
|
||||
ipv4_address = cidrhost(try(local.env.infrastructure.networking.subnet_cidr, "192.168.1.0/24"), count.index + 30)
|
||||
ipv4_netmask = 24
|
||||
}
|
||||
|
||||
ipv4_gateway = try(local.env.infrastructure.networking.gateway, "192.168.1.1")
|
||||
dns_server_list = ["8.8.8.8", "8.8.4.4"]
|
||||
}
|
||||
}
|
||||
|
||||
tags = [for k, v in var.tags : "${k}=${v}"]
|
||||
}
|
||||
|
||||
# Note: Kubernetes cluster installation on these VMs would be done via:
|
||||
# - Cloud-init scripts
|
||||
# - Ansible playbooks
|
||||
# - kubeadm
|
||||
# - Rancher/K3s
|
||||
# This is outside Terraform's scope but can be orchestrated via provisioners or external tools
|
||||
|
||||
31
terraform/multi-cloud/modules/onprem-hci/outputs.tf
Normal file
31
terraform/multi-cloud/modules/onprem-hci/outputs.tf
Normal file
@@ -0,0 +1,31 @@
|
||||
# Outputs for On-Prem HCI Module
|
||||
|
||||
output "cluster_name" {
|
||||
value = local.hci_platform == "azure-stack-hci" ? (
|
||||
try(azapi_resource.azure_stack_hci_cluster[0].name, "${local.name_prefix}-hci")
|
||||
) : (
|
||||
"${local.name_prefix}-hci"
|
||||
)
|
||||
description = "HCI cluster name"
|
||||
}
|
||||
|
||||
output "region" {
|
||||
value = try(var.environment_config.region, "onprem")
|
||||
description = "Region/location identifier"
|
||||
}
|
||||
|
||||
output "kubeconfig" {
|
||||
value = null # Will be generated by external script or Arc agent
|
||||
description = "Kubeconfig for the cluster"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
output "vm_ips" {
|
||||
value = local.hci_platform == "vsphere" ? {
|
||||
system = [for vm in vsphere_virtual_machine.system : vm.default_ip_address]
|
||||
validators = [for vm in vsphere_virtual_machine.validators : vm.default_ip_address]
|
||||
rpc = [for vm in vsphere_virtual_machine.rpc : vm.default_ip_address]
|
||||
} : {}
|
||||
description = "VM IP addresses (vSphere only)"
|
||||
}
|
||||
|
||||
43
terraform/multi-cloud/modules/onprem-hci/variables.tf
Normal file
43
terraform/multi-cloud/modules/onprem-hci/variables.tf
Normal file
@@ -0,0 +1,43 @@
|
||||
# Variables for On-Prem HCI Module
|
||||
|
||||
variable "environment_config" {
|
||||
description = "Environment configuration from environments.yaml"
|
||||
type = any
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
description = "Environment name"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "vsphere_user" {
|
||||
description = "vSphere username"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "vsphere_password" {
|
||||
description = "vSphere password"
|
||||
type = string
|
||||
default = ""
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "vsphere_server" {
|
||||
description = "vSphere server address"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "azure_subscription_id" {
|
||||
description = "Azure subscription ID (for Azure Stack HCI)"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Tags to apply to resources"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
159
terraform/multi-cloud/modules/service-mesh/main.tf
Normal file
159
terraform/multi-cloud/modules/service-mesh/main.tf
Normal file
@@ -0,0 +1,159 @@
|
||||
# Service Mesh Module
|
||||
# Deploys Istio, Linkerd, or Kuma across all clusters for cross-cloud communication
|
||||
|
||||
locals {
|
||||
# Service mesh configuration
|
||||
mesh_provider = var.provider
|
||||
|
||||
# Cluster configurations
|
||||
clusters = var.clusters
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# ISTIO DEPLOYMENT
|
||||
# ============================================
|
||||
resource "helm_release" "istio_base" {
|
||||
count = local.mesh_provider == "istio" ? length(local.clusters) : 0
|
||||
|
||||
name = "istio-base"
|
||||
repository = "https://istio-release.storage.googleapis.com/charts"
|
||||
chart = "base"
|
||||
version = "1.19.0"
|
||||
namespace = "istio-system"
|
||||
create_namespace = true
|
||||
|
||||
# Dynamic provider configuration would be needed here
|
||||
# For now, this is a template that would be applied per cluster
|
||||
}
|
||||
|
||||
resource "helm_release" "istio_istiod" {
|
||||
count = local.mesh_provider == "istio" ? length(local.clusters) : 0
|
||||
|
||||
name = "istiod"
|
||||
repository = "https://istio-release.storage.googleapis.com/charts"
|
||||
chart = "istiod"
|
||||
version = "1.19.0"
|
||||
namespace = "istio-system"
|
||||
|
||||
values = [yamlencode({
|
||||
meshConfig = {
|
||||
defaultConfig = {
|
||||
proxyStatsMatcher = {
|
||||
inclusionRegexps = [".*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
pilot = {
|
||||
env = {
|
||||
PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY = true
|
||||
}
|
||||
}
|
||||
})]
|
||||
|
||||
depends_on = [helm_release.istio_base]
|
||||
}
|
||||
|
||||
resource "helm_release" "istio_gateway" {
|
||||
count = local.mesh_provider == "istio" ? length(local.clusters) : 0
|
||||
|
||||
name = "istio-gateway"
|
||||
repository = "https://istio-release.storage.googleapis.com/charts"
|
||||
chart = "gateway"
|
||||
version = "1.19.0"
|
||||
namespace = "istio-system"
|
||||
|
||||
values = [yamlencode({
|
||||
service = {
|
||||
type = "LoadBalancer"
|
||||
}
|
||||
})]
|
||||
|
||||
depends_on = [helm_release.istio_istiod]
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# LINKERD DEPLOYMENT
|
||||
# ============================================
|
||||
resource "helm_release" "linkerd_crds" {
|
||||
count = local.mesh_provider == "linkerd" ? length(local.clusters) : 0
|
||||
|
||||
name = "linkerd-crds"
|
||||
repository = "https://helm.linkerd.io/stable"
|
||||
chart = "linkerd-crds"
|
||||
version = "1.15.0"
|
||||
namespace = "linkerd"
|
||||
create_namespace = true
|
||||
}
|
||||
|
||||
resource "helm_release" "linkerd_control_plane" {
|
||||
count = local.mesh_provider == "linkerd" ? length(local.clusters) : 0
|
||||
|
||||
name = "linkerd-control-plane"
|
||||
repository = "https://helm.linkerd.io/stable"
|
||||
chart = "linkerd-control-plane"
|
||||
version = "1.15.0"
|
||||
namespace = "linkerd"
|
||||
|
||||
values = [yamlencode({
|
||||
identity = {
|
||||
issuer = {
|
||||
scheme = "kubernetes.io/tls"
|
||||
}
|
||||
}
|
||||
proxy = {
|
||||
resources = {
|
||||
cpu = {
|
||||
request = "100m"
|
||||
}
|
||||
memory = {
|
||||
request = "128Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
})]
|
||||
|
||||
depends_on = [helm_release.linkerd_crds]
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# KUMA DEPLOYMENT
|
||||
# ============================================
|
||||
resource "helm_release" "kuma_control_plane" {
|
||||
count = local.mesh_provider == "kuma" ? length(local.clusters) : 0
|
||||
|
||||
name = "kuma"
|
||||
repository = "https://kumahq.github.io/charts"
|
||||
chart = "kuma"
|
||||
version = "2.5.0"
|
||||
namespace = "kuma-system"
|
||||
create_namespace = true
|
||||
|
||||
values = [yamlencode({
|
||||
controlPlane = {
|
||||
mode = "zone"
|
||||
zones = {
|
||||
enabled = true
|
||||
}
|
||||
}
|
||||
})]
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# CROSS-CLUSTER CONFIGURATION
|
||||
# ============================================
|
||||
# Generate configuration files for cross-cluster mesh setup
|
||||
resource "local_file" "mesh_config" {
|
||||
for_each = local.clusters
|
||||
|
||||
filename = "${path.module}/../../../../config/mesh/${each.key}-mesh-config.yaml"
|
||||
content = yamlencode({
|
||||
cluster = each.key
|
||||
provider = local.mesh_provider
|
||||
mTLS = var.mTLS_enabled
|
||||
endpoints = {
|
||||
for k, v in local.clusters : k => v.endpoint
|
||||
if k != each.key
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
31
terraform/multi-cloud/modules/service-mesh/variables.tf
Normal file
31
terraform/multi-cloud/modules/service-mesh/variables.tf
Normal file
@@ -0,0 +1,31 @@
|
||||
# Variables for Service Mesh Module
|
||||
|
||||
variable "provider" {
|
||||
description = "Service mesh provider (istio, linkerd, kuma)"
|
||||
type = string
|
||||
validation {
|
||||
condition = contains(["istio", "linkerd", "kuma"], var.provider)
|
||||
error_message = "Service mesh provider must be one of: istio, linkerd, kuma"
|
||||
}
|
||||
}
|
||||
|
||||
variable "clusters" {
|
||||
description = "Map of clusters to deploy service mesh to"
|
||||
type = map(object({
|
||||
endpoint = string
|
||||
kubeconfig = string
|
||||
}))
|
||||
}
|
||||
|
||||
variable "mTLS_enabled" {
|
||||
description = "Enable mutual TLS for service mesh"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Tags to apply to resources"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user