AWSS3管理Skill aws-s3-management

AWS S3 Management

管理S3桶，包括版本控制、加密、访问控制、生命周期策略和复制。用于对象存储、静态网站和数据湖。

何时使用

• 静态网站托管
• 数据备份和归档
• 媒体库和CDN源
• 数据湖和分析
• 日志存储和分析
• 应用程序资产存储
• 灾难恢复
• 数据共享和协作

实施示例

1. 使用AWS CLI创建和配置S3桶

# 创建桶
aws s3api create-bucket \
  --bucket my-app-bucket-$(date +%s) \
  --region us-east-1

# 启用版本控制
aws s3api put-bucket-versioning \
  --bucket my-app-bucket \
  --versioning-configuration Status=Enabled

# 阻止公共访问
aws s3api put-public-access-block \
  --bucket my-app-bucket \
  --public-access-block-configuration \
    BlockPublicAcls=true,IgnorePublicAcls=true,\
    BlockPublicPolicy=true,RestrictPublicBuckets=true

# 启用加密
aws s3api put-bucket-encryption \
  --bucket my-app-bucket \
  --server-side-encryption-configuration '{
    "Rules": [{
      "ApplyServerSideEncryptionByDefault": {
        "SSEAlgorithm": "AES256"
      }
    }]
  }'

# 上传文件和元数据
aws s3 cp index.html s3://my-app-bucket/ \
  --cache-control "max-age=3600" \
  --metadata "author=john,version=1"

# 同步目录到S3
aws s3 sync ./dist s3://my-app-bucket/ \
  --delete \
  --exclude "*.map"

# 列出带有元数据的对象
aws s3api list-objects-v2 \
  --bucket my-app-bucket \
  --query 'Contents[].{Key:Key,Size:Size,Modified:LastModified}'

2. S3生命周期策略配置

# 创建生命周期策略
aws s3api put-bucket-lifecycle-configuration \
  --bucket my-app-bucket \
  --lifecycle-configuration '{
    "Rules": [
      {
        "Id": "archive-old-logs",
        "Status": "Enabled",
        "Prefix": "logs/",
        "Transitions": [
          {
            "Days": 30,
            "StorageClass": "STANDARD_IA"
          },
          {
            "Days": 90,
            "StorageClass": "GLACIER"
          }
        ],
        "Expiration": {
          "Days": 365
        }
      },
      {
        "Id": "cleanup-incomplete-uploads",
        "Status": "Enabled",
        "AbortIncompleteMultipartUpload": {
          "DaysAfterInitiation": 7
        }
      }
    ]
  }'

# 获取桶生命周期
aws s3api get-bucket-lifecycle-configuration \
  --bucket my-app-bucket

3. Terraform S3配置

# s3.tf
terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
  }
}

provider "aws" {
  region = "us-east-1"
}

# S3桶
resource "aws_s3_bucket" "app_data" {
  bucket = "my-app-data-${data.aws_caller_identity.current.account_id}"
}

# 阻止公共访问
resource "aws_s3_bucket_public_access_block" "app_data" {
  bucket = aws_s3_bucket.app_data.id

  block_public_acls       = true
  block_public_policy     = true
  ignore_public_acls      = true
  restrict_public_buckets = true
}

# 启用版本控制
resource "aws_s3_bucket_versioning" "app_data" {
  bucket = aws_s3_bucket.app_data.id

  versioning_configuration {
    status = "Enabled"
  }
}

# 服务器端加密
resource "aws_s3_bucket_server_side_encryption_configuration" "app_data" {
  bucket = aws_s3_bucket.app_data.id

  rule {
    apply_server_side_encryption_by_default {
      sse_algorithm = "AES256"
    }
  }
}

# 生命周期策略
resource "aws_s3_bucket_lifecycle_configuration" "app_data" {
  bucket = aws_s3_bucket.app_data.id

  rule {
    id     = "archive-logs"
    status = "Enabled"

    filter {
      prefix = "logs/"
    }

    transition {
      days          = 30
      storage_class = "STANDARD_IA"
    }

    transition {
      days          = 90
      storage_class = "GLACIER"
    }

    expiration {
      days = 365
    }
  }

  rule {
    id     = "cleanup-incomplete-uploads"
    status = "Enabled"

    abort_incomplete_multipart_upload {
      days_after_initiation = 7
    }
  }
}

# CORS配置
resource "aws_s3_bucket_cors_configuration" "app_data" {
  bucket = aws_s3_bucket.app_data.id

  cors_rule {
    allowed_headers = ["*"]
    allowed_methods = ["GET", "PUT", "POST"]
    allowed_origins = ["https://example.com"]
    expose_headers  = ["ETag"]
    max_age_seconds = 3000
  }
}

# 桶策略用于CloudFront
resource "aws_s3_bucket_policy" "app_data" {
  bucket = aws_s3_bucket.app_data.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Sid    = "AllowCloudFront"
        Effect = "Allow"
        Principal = {
          Service = "cloudfront.amazonaws.com"
        }
        Action   = "s3:GetObject"
        Resource = "${aws_s3_bucket.app_data.arn}/*"
        Condition = {
          StringEquals = {
            "AWS:SourceArn" = "arn:aws:cloudfront::${data.aws_caller_identity.current.account_id}:distribution/${aws_cloudfront_distribution.app.id}"
          }
        }
      }
    ]
  })
}

# 启用日志记录
resource "aws_s3_bucket_logging" "app_data" {
  bucket = aws_s3_bucket.app_data.id

  target_bucket = aws_s3_bucket.logs.id
  target_prefix = "s3-logs/"
}

# 复制配置
resource "aws_s3_bucket_replication_configuration" "app_data" {
  depends_on = [aws_s3_bucket_versioning.app_data]
  role       = aws_iam_role.s3_replication.arn
  bucket     = aws_s3_bucket.app_data.id

  rule {
    status = "Enabled"

    filter {}

    destination {
      bucket       = aws_s3_bucket.replica.arn
      storage_class = "STANDARD_IA"

      replication_time {
        status = "Enabled"
        time {
          minutes = 15
        }
      }

      metrics {
        status = "Enabled"
        event_threshold {
          minutes = 15
        }
      }
    }
  }
}

data "aws_caller_identity" "current" {}

# 复制桶
resource "aws_s3_bucket" "replica" {
  bucket = "my-app-data-replica-${data.aws_caller_identity.current.account_id}"
}

resource "aws_s3_bucket_versioning" "replica" {
  bucket = aws_s3_bucket.replica.id

  versioning_configuration {
    status = "Enabled"
  }
}

# 日志桶
resource "aws_s3_bucket" "logs" {
  bucket = "my-app-logs-${data.aws_caller_identity.current.account_id}"
}

# IAM角色复制
resource "aws_iam_role" "s3_replication" {
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action = "sts:AssumeRole"
      Effect = "Allow"
      Principal = {
        Service = "s3.amazonaws.com"
      }
    }]
  })
}

resource "aws_iam_role_policy" "s3_replication" {
  role = aws_iam_role.s3_replication.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "s3:GetReplicationConfiguration",
          "s3:ListBucket"
        ]
        Resource = aws_s3_bucket.app_data.arn
      },
      {
        Effect = "Allow"
        Action = [
          "s3:GetObjectVersionForReplication",
          "s3:GetObjectVersionAcl"
        ]
        Resource = "${aws_s3_bucket.app_data.arn}/*"
      },
      {
        Effect = "Allow"
        Action = [
          "s3:ReplicateObject",
          "s3:ReplicateDelete"
        ]
        Resource = "${aws_s3_bucket.replica.arn}/*"
      }
    ]
  })
}

4. 使用预签名URL访问S3

# 生成预签名URL（1小时过期）
aws s3 presign s3://my-app-bucket/private/document.pdf \
  --expires-in 3600

# 生成PUT（上传）预签名URL
aws s3 presign s3://my-app-bucket/uploads/file.jpg \
  --expires-in 3600 \
  --region us-east-1 \
  --request-method PUT

最佳实践

✅ DO

• 为重要数据启用版本控制
• 使用服务器端加密
• 默认阻止公共访问
• 实施生命周期策略
• 启用日志记录和监控
• 使用桶策略进行访问控制
• 对关键桶启用MFA删除
• 使用IAM角色而不是访问密钥
• 实施跨区域复制

❌ DON’T

• 使桶公开可访问
• 存储敏感凭据
• 忽略CloudTrail日志记录
• 使用过于宽松的策略
• 忘记设置生命周期规则
• 忽略加密要求

监控

• S3 CloudWatch指标
• CloudTrail用于API日志记录
• CloudWatch警报用于阈值
• S3清单用于对象跟踪
• S3访问分析器用于权限

资源

• AWS S3文档
• S3安全最佳实践