Architecting Global Resilience: Multi-Region Cloud Spanner Deployments for Uninterrupted Operations

  1# main.tf
  2
  3# Define project and region variables
  4variable "project_id" {
  5  description = "The GCP project ID."
  6  type        = string
  7}
  8
  9variable "regions" {
 10  description = "List of regions for application deployment."
 11  type        = list(string)
 12  default     = ["us-east4", "europe-west1"]
 13}
 14
 15# --- Cloud Run Services (Placeholder for actual deployment) ---
 16# In a real setup, these would be defined by google_cloud_run_service resources.
 17# For this example, we'll assume they exist and get their service URLs.
 18resource "google_cloud_run_service" "app_us" {
 19  project  = var.project_id
 20  location = "us-east4"
 21  name     = "global-spanner-app-us"
 22  template {
 23    spec {
 24      containers {
 25        image = "gcr.io/cloudrun/hello" # Placeholder image
 26      }
 27    }
 28  }
 29  autogenerate_revision_name = true
 30}
 31
 32resource "google_cloud_run_service" "app_eu" {
 33  project  = var.project_id
 34  location = "europe-west1"
 35  name     = "global-spanner-app-eu"
 36  template {
 37    spec {
 38      containers {
 39        image = "gcr.io/cloudrun/hello" # Placeholder image
 40      }
 41    }
 42  }
 43  autogenerate_revision_name = true
 44}
 45
 46# --- Health Check for Backend Services ---
 47resource "google_compute_health_check" "http_health_check" {
 48  project            = var.project_id
 49  name               = "http-health-check-80"
 50  timeout_sec        = 5
 51  check_interval_sec = 5
 52  request_path       = "/healthz" # Application health endpoint
 53  port               = 80
 54  protocol           = "HTTP"
 55}
 56
 57# --- Serverless NEGs for Cloud Run services ---
 58resource "google_compute_region_network_endpoint_group" "serverless_neg_us" {
 59  project              = var.project_id
 60  region               = "us-east4"
 61  name                 = "serverless-neg-us-east4"
 62  network_endpoint_type = "SERVERLESS"
 63  cloud_run {
 64    service = google_cloud_run_service.app_us.name
 65  }
 66}
 67
 68resource "google_compute_region_network_endpoint_group" "serverless_neg_eu" {
 69  project              = var.project_id
 70  region               = "europe-west1"
 71  name                 = "serverless-neg-europe-west1"
 72  network_endpoint_type = "SERVERLESS"
 73  cloud_run {
 74    service = google_cloud_run_service.app_eu.name
 75  }
 76}
 77
 78# --- Global Backend Services ---
 79resource "google_compute_backend_service" "backend_service_us" {
 80  project              = var.project_id
 81  name                 = "backend-service-us-east4"
 82  port_name            = "http"
 83  protocol             = "HTTP"
 84  timeout_sec          = 30
 85  enable_cdn           = false
 86  load_balancing_scheme = "EXTERNAL"
 87  health_checks        = [google_compute_health_check.http_health_check.id]
 88  network_tier         = "PREMIUM" # Critical for global Anycast IP and low latency
 89
 90  backend {
 91    group = google_compute_region_network_endpoint_group.serverless_neg_us.id
 92    # You might specify balancing modes like UTILIZATION or RATE depending on your needs.
 93  }
 94}
 95
 96resource "google_compute_backend_service" "backend_service_eu" {
 97  project              = var.project_id
 98  name                 = "backend-service-europe-west1"
 99  port_name            = "http"
100  protocol             = "HTTP"
101  timeout_sec          = 30
102  enable_cdn           = false
103  load_balancing_scheme = "EXTERNAL"
104  health_checks        = [google_compute_health_check.http_health_check.id]
105  network_tier         = "PREMIUM" # Critical for global Anycast IP and low latency
106
107  backend {
108    group = google_compute_region_network_endpoint_group.serverless_neg_eu.id
109  }
110}
111
112# --- URL Map to direct traffic ---
113resource "google_compute_url_map" "url_map" {
114  project         = var.project_id
115  name            = "global-spanner-url-map"
116  default_service = google_compute_backend_service.backend_service_us.id # Default to one region
117
118  # Host rule example (can be expanded for more complex routing)
119  host_rule {
120    hosts        = ["www.example.com"]
121    path_matcher = "allpaths"
122  }
123
124  path_matcher {
125    name            = "allpaths"
126    default_service = google_compute_backend_service.backend_service_us.id
127    # For global, usually you want backend services to distribute based on proximity
128    # For advanced path-based routing, add more path_rules here
129  }
130}
131
132# --- SSL Certificate (Managed for simplicity) ---
133resource "google_compute_managed_ssl_certificate" "managed_cert" {
134  project = var.project_id
135  name    = "global-spanner-managed-cert"
136  managed {
137    domains = ["www.example.com"]
138  }
139}
140
141# --- Target HTTP(S) Proxy ---
142resource "google_compute_target_https_proxy" "https_proxy" {
143  project            = var.project_id
144  name               = "global-spanner-https-proxy"
145  url_map            = google_compute_url_map.url_map.id
146  ssl_certificates   = [google_compute_managed_ssl_certificate.managed_cert.id]
147  # For production, enable SSL policy: ssl_policy = "projects/PROJECT_ID/global/sslPolicies/SSL_POLICY_NAME"
148}
149
150# --- Global Forwarding Rule (The Anycast IP) ---
151resource "google_compute_global_forwarding_rule" "https_forwarding_rule" {
152  project               = var.project_id
153  name                  = "global-spanner-https-forwarding-rule"
154  ip_protocol           = "TCP"
155  port_range            = "443"
156  target                = google_compute_target_https_proxy.https_proxy.id
157  load_balancing_scheme = "EXTERNAL"
158  ip_address            = google_compute_global_address.static_ip.id
159}
160
161# --- Static IP Address for the Load Balancer ---
162resource "google_compute_global_address" "static_ip" {
163  project       = var.project_id
164  name          = "global-spanner-lb-ip"
165  ip_version    = "IPV4"
166  network_tier  = "PREMIUM"
167}
168
169# Output the IP address of the load balancer
170output "lb_ip_address" {
171  description = "The IP address of the Global External HTTP(S) Load Balancer."
172  value       = google_compute_global_address.static_ip.address
173}

  1package main
  2
  3import (
  4	"context"
  5	"fmt"
  6	"log"
  7	"math/rand"
  8	"time"
  9
 10	"cloud.google.com/go/spanner"
 11	"google.golang.org/grpc/codes"
 12	"google.golang.org/grpc/status"
 13
 14	// Prometheus client for metrics. In a real application, ensure this is properly initialized and exposed.
 15	"github.com/prometheus/client_golang/prometheus"
 16	"github.com/prometheus/client_golang/prometheus/promauto"
 17)
 18
 19// Define Prometheus metrics for Spanner operations
 20var (
 21	spannerRetriesTotal = promauto.NewCounterVec(prometheus.CounterOpts{
 22		Name: "spanner_retries_total",
 23		Help: "Total number of Cloud Spanner operation retries.",
 24	}, []string{"operation_type", "status_code"})
 25
 26	spannerOperationLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
 27		Name:    "spanner_operation_latency_milliseconds",
 28		Help:    "Latency of Cloud Spanner operations in milliseconds.",
 29		Buckets: []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000}, // Milliseconds
 30	}, []string{"operation_type", "status_code"})
 31)
 32
 33// SpannerOperationFunc defines the type for a function that performs a Spanner operation.
 34// It receives a transaction object, allowing it to execute within a Spanner transaction.
 35type SpannerOperationFunc func(ctx context.Context, txn *spanner.Transaction) error
 36
 37// executeWithRetries wraps a Spanner operation with robust retry logic and metrics.
 38// It handles retryable gRPC errors with exponential backoff and jitter.
 39func executeWithRetries(ctx context.Context, client *spanner.Client, operationType string, op SpannerOperationFunc) error {
 40	const (
 41		maxRetries    = 10
 42		initialBackoff = 100 * time.Millisecond
 43		maxBackoff    = 5 * time.Second
 44		jitterFactor  = 0.5 // Jitter up to 50% of the backoff duration
 45	)
 46
 47	backoff := initialBackoff
 48	rand.Seed(time.Now().UnixNano()) // Seed for jitter
 49
 50	for i := 0; i < maxRetries; i++ {
 51		startTime := time.Now()
 52		// In a real application, the client should not be nil. This is for demonstration
 53		// to simulate errors without a live Spanner connection.
 54		var err error
 55		if client != nil {
 56			err = client.RunInTransaction(ctx, func(txn *spanner.Transaction) error {
 57				return op(ctx, txn)
 58			})
 59		} else {
 60			// Simulate an error for testing the retry logic without a real client
 61			err = op(ctx, nil)
 62		}
 63		latency := time.Since(startTime)
 64
 65		statusCode := codes.OK
 66		if err != nil {
 67			if s, ok := status.FromError(err); ok {
 68				statusCode = s.Code()
 69			}
 70		}
 71
 72		spannerOperationLatency.WithLabelValues(operationType, statusCode.String()).Observe(float64(latency.Milliseconds()))
 73
 74		if err == nil {
 75			log.Printf("Spanner operation '%s' successful after %d retries. Latency: %s", operationType, i, latency)
 76			return nil
 77		}
 78
 79		// Check for retryable errors. Cloud Spanner operations often return ABORTED for contention.
 80		// UNAVAILABLE and DEADLINE_EXCEEDED (for short operations) are also commonly retryable.
 81		if s, ok := status.FromError(err); ok {
 82			switch s.Code() {
 83			case codes.Unavailable, codes.Aborted, codes.DeadlineExceeded, codes.Internal: // Add other retryable codes if necessary
 84				log.Printf("Spanner operation '%s' failed with retryable error (%s). Retrying (attempt %d/%d)... Error: %v", operationType, s.Code(), i+1, maxRetries, err)
 85				spannerRetriesTotal.WithLabelValues(operationType, s.Code().String()).Inc()
 86
 87				// Apply exponential backoff with jitter
 88				jitter := time.Duration(rand.Int63n(int64(float64(backoff) * jitterFactor)))
 89				sleepDuration := backoff + jitter
 90				if sleepDuration > maxBackoff {
 91					sleepDuration = maxBackoff
 92				}
 93				time.Sleep(sleepDuration)
 94				backoff *= 2
 95				continue // Retry the operation
 96			default:
 97				log.Printf("Spanner operation '%s' failed with non-retryable error (%s): %v", operationType, s.Code(), err)
 98				return err // Non-retryable error, exit
 99			}
100		} else {
101			log.Printf("Spanner operation '%s' failed with unknown error: %v", operationType, err)
102			return err // Unknown error, exit
103		}
104	}
105
106	return fmt.Errorf("spanner operation '%s' failed after %d retries", operationType, maxRetries)
107}
108
109// Example usage within a main function (requires actual Spanner client initialization)
110func main() {
111	ctx := context.Background()
112
113	// IMPORTANT: Replace with your actual Spanner client initialization.
114	// dbPath := "projects/YOUR_PROJECT_ID/instances/YOUR_INSTANCE_ID/databases/YOUR_DATABASE_ID"
115	// client, err := spanner.NewClient(ctx, dbPath)
116	// if err != nil {
117	//	log.Fatalf("Failed to create Spanner client: %v", err)
118	// }
119	// defer client.Close()
120
121	// For demonstration, we'll use a nil client. In a real scenario, this must be a valid Spanner client.
122	// The `op` function logic below will simulate errors for testing the retry wrapper.
123	var client *spanner.Client // Placeholder for actual client
124
125	// --- Example Write Operation ---
126	writeSimulatedOp := func(ctx context.Context, txn *spanner.Transaction) error {
127		// Simulate a retryable error (e.g., transaction aborted due to contention) 2 out of 3 times initially.
128		if rand.Intn(3) != 0 {
129			return status.Error(codes.Aborted, "simulated transaction aborted due to contention")
130		}
131		// In a real application, perform Spanner mutations here, e.g.:
132		// m := []*spanner.Mutation{
133		//	spanner.Insert("Singers", []string{"SingerId", "FirstName"}, []interface{}{1, "John"}),
134		// }
135		// txn.BufferWrite(m)
136		log.Println("Simulating successful Spanner write...")
137		return nil
138	}
139
140	fmt.Println("\n--- Attempting write operation with retries ---")
141	err := executeWithRetries(ctx, client, "insert_singer", writeSimulatedOp)
142	if err != nil {
143		fmt.Printf("Final write operation result: %v\n", err)
144	} else {
145		fmt.Println("Write operation completed successfully.")
146	}
147
148	// --- Example Read Operation ---
149	readSimulatedOp := func(ctx context.Context, txn *spanner.Transaction) error {
150		// Simulate a retryable error (e.g., database unavailable) 1 out of 5 times initially.
151		if rand.Intn(5) == 0 {
152			return status.Error(codes.Unavailable, "simulated database temporarily unavailable")
153		}
154		// In a real application, perform Spanner reads here, e.g.:
155		// iter := txn.Read(ctx, "Albums", spanner.AllKeys(), []string{"AlbumId", "AlbumTitle"})\
156		// defer iter.Stop()
157		// _, err := iter.Next()
158		// if err != nil && err != iterator.Done { return err }
159		log.Println("Simulating successful Spanner read...")
160		return nil
161	}
162
163	fmt.Println("\n--- Attempting read operation with retries ---")
164	err = executeWithRetries(ctx, client, "read_album", readSimulatedOp)
165	if err != nil {
166		fmt.Printf("Final read operation result: %v\n", err)
167	} else {
168		fmt.Println("Read operation completed successfully.")
169	}
170
171	// In a production environment, expose Prometheus metrics via an HTTP endpoint:
172	// http.Handle("/metrics", promhttp.Handler())
173	// log.Fatal(http.ListenAndServe(":8080", nil))
174}