AWS実践 #5 モニタリング — CloudWatch アラームと X-Ray

┌──────────────┬──────────────┬──────────────┬──────────────┐
│   Metrics    │     Logs     │    Traces    │    Events    │
├──────────────┼──────────────┼──────────────┼──────────────┤
│ "どれだけ"    │ "何が"        │ "どこで"      │ "いつ"        │
│  リクエスト数 │ stacktrace   │ DB が 5 秒   │ デプロイ、    │
│  CPU、メモリ │ access log   │ 外部 API 1 秒│ スケール、    │
│              │              │              │ フェイルオーバー│
├──────────────┼──────────────┼──────────────┼──────────────┤
│ CloudWatch   │ CloudWatch   │ X-Ray        │ EventBridge  │
│ Metrics      │ Logs         │              │              │
└──────────────┴──────────────┴──────────────┴──────────────┘

Log Group: /ecs/blog-api
   │
   ├── Log Stream: api/<task-id-1>     ← Task ごとに stream 1 つ
   ├── Log Stream: api/<task-id-2>
   └── Log Stream: api/<task-id-3>

aws logs put-retention-policy \
  --log-group-name /ecs/blog-api \
  --retention-in-days 30

import logging, json, sys

class JsonFormatter(logging.Formatter):
    def format(self, record):
        return json.dumps({
            "level":   record.levelname,
            "logger":  record.name,
            "message": record.getMessage(),
            "ts":      self.formatTime(record),
            **getattr(record, "extra", {}),
        })

handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.INFO, handlers=[handler])

@app.middleware("http")
async def access_log(request, call_next):
    start = time.time()
    response = await call_next(request)
    logging.info("access", extra={"extra": {
        "method": request.method,
        "path": str(request.url.path),
        "status": response.status_code,
        "duration_ms": int((time.time() - start) * 1000),
        "request_id": request.state.request_id,
    }})
    return response

aws ecs update-cluster-settings \
  --cluster blog-cluster \
  --settings name=containerInsights,value=enabled

import json, time, logging

def emit_metric(metric_name, value, unit="Count", **dims):
    payload = {
        "_aws": {
            "Timestamp": int(time.time() * 1000),
            "CloudWatchMetrics": [{
                "Namespace": "BlogApp",
                "Dimensions": [list(dims.keys())],
                "Metrics": [{"Name": metric_name, "Unit": unit}],
            }],
        },
        metric_name: value,
        **dims,
    }
    logging.info(json.dumps(payload))

emit_metric("PostCreated", 1, env="prod")
emit_metric("CommentCreated", 1, env="prod")
emit_metric("LoginFailed", 1, source_ip="...")

aws cloudwatch put-metric-alarm \
  --alarm-name "blog-alb-5xx-burst" \
  --metric-name HTTPCode_Target_5XX_Count \
  --namespace AWS/ApplicationELB \
  --statistic Sum \
  --period 60 \
  --evaluation-periods 5 \
  --datapoints-to-alarm 3 \
  --threshold 5 \
  --comparison-operator GreaterThanOrEqualToThreshold \
  --treat-missing-data notBreaching \
  --dimensions Name=LoadBalancer,Value=app/blog-alb/abc123 \
  --alarm-actions arn:aws:sns:ap-northeast-2:123456789012:ops-alerts

aws cloudwatch put-composite-alarm \
  --alarm-name "blog-real-incident" \
  --alarm-rule "ALARM('blog-alb-5xx-burst') AND OK('blog-running-tasks-low')"

CloudWatch Alarm
   │
   ▼
SNS Topic (ops-alerts)
   │
   ├── Email subscription   (運用陣)
   ├── SMS subscription      (oncall)
   ├── Lambda subscription   ← Slack webhook に変換
   └── PagerDuty / OpsGenie

import json, os, urllib.request

WEBHOOK = os.environ["SLACK_WEBHOOK"]

def handler(event, context):
    for record in event["Records"]:
        msg = json.loads(record["Sns"]["Message"])
        text = (
            f":rotating_light: *{msg['AlarmName']}*\n"
            f"Region: {msg['Region']}\n"
            f"State: {msg['NewStateValue']} (was {msg['OldStateValue']})\n"
            f"Reason: {msg['NewStateReason']}\n"
        )
        req = urllib.request.Request(
            WEBHOOK,
            data=json.dumps({"text": text}).encode(),
            headers={"Content-Type": "application/json"},
        )
        urllib.request.urlopen(req)

Request: POST /posts                       4.2s
   │
   ├── ALB                                 0.01s
   │
   └── ECS api                             4.15s
         │
         ├── auth.verify_token             0.05s
         │
         ├── db.posts.insert               3.80s   ← 犯人
         │     └── RDS PostgreSQL          3.78s
         │           └── (slow query)
         │
         └── notify.publish (SNS)          0.30s
               └── SNS:Publish             0.28s

pip install aws-xray-sdk

from aws_xray_sdk.core import xray_recorder
from aws_xray_sdk.ext.fastapi.middleware import XRayMiddleware
from aws_xray_sdk.ext.sqlalchemy.query import XRayQuery

xray_recorder.configure(service="blog-api")

app = FastAPI()
app.add_middleware(XRayMiddleware, recorder=xray_recorder)

# SQLAlchemy のトレース
from aws_xray_sdk.ext.sqlalchemy_core import unpatch
# (engine 生成時に自動 patch)

{
  "containerDefinitions": [
    { "name": "api", ... },
    {
      "name": "xray-daemon",
      "image": "public.ecr.aws/xray/aws-xray-daemon:latest",
      "portMappings": [{ "containerPort": 2000, "protocol": "udp" }],
      "essential": false
    }
  ]
}

{
  "version": 2,
  "rules": [{
    "description": "Default",
    "service_name": "*",
    "http_method": "*",
    "url_path": "*",
    "fixed_target": 1,
    "rate": 0.05
  }],
  "default": { "fixed_target": 1, "rate": 0.05 }
}

[1] Requests/s (ALB)         [2] 5xx rate (ALB)         [3] p99 latency
[4] ECS CPU (Service)         [5] ECS Memory             [6] Running tasks
[7] RDS CPU                   [8] RDS Connections        [9] RDS FreeStorage

resource "aws_cloudwatch_dashboard" "blog" {
  dashboard_name = "blog-overview"
  dashboard_body = jsonencode({
    widgets = [
      {
        type   = "metric",
        x      = 0, y = 0, width = 8, height = 6,
        properties = {
          metrics = [["AWS/ApplicationELB", "RequestCount", "LoadBalancer", "app/blog-alb/abc123"]],
          period  = 60, stat = "Sum", region = "ap-northeast-2",
          title   = "Requests/min"
        }
      }
      # ... あと 8 個
    ]
  })
}