diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml index fd1c160..447b0fa 100644 --- a/.github/workflows/load-test-start.yml +++ b/.github/workflows/load-test-start.yml @@ -3,6 +3,10 @@ name: Load Test Start on: workflow_dispatch: inputs: + load_test_db_instance_type: + description: "Load-test MySQL EC2 instance type (required; no default)" + required: true + type: string switch_stage_to_loadtest: description: "Restart stage app with dev,loadtest profiles" required: true @@ -46,7 +50,14 @@ jobs: sudo apt-get install -y jq - name: Start load test environment + env: + DB_INSTANCE_TYPE: ${{ inputs.load_test_db_instance_type }} run: | + if [ -z "$DB_INSTANCE_TYPE" ]; then + echo "load_test_db_instance_type is required" >&2 + exit 1 + fi + export TF_VAR_load_generator_instance_type="c7i.large" export TF_VAR_create_load_generator="false" @@ -54,7 +65,7 @@ jobs: export TF_VAR_load_generator_instance_profile_name="${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" fi - args=() + args=(--db-instance-type "$DB_INSTANCE_TYPE") if [ "${{ inputs.switch_stage_to_loadtest }}" = "true" ]; then args+=(--switch-stage-to-loadtest) diff --git a/.github/workflows/load-test-stop.yml b/.github/workflows/load-test-stop.yml index 10f2e82..a1755a5 100644 --- a/.github/workflows/load-test-stop.yml +++ b/.github/workflows/load-test-stop.yml @@ -13,6 +13,10 @@ on: required: true default: true type: boolean + load_test_db_instance_type: + description: "DB EC2 type selected at Start; needed only if DB EC2 was never created" + required: false + type: string permissions: id-token: write @@ -51,6 +55,8 @@ jobs: sudo apt-get install -y jq - name: Stop load test environment + env: + DB_INSTANCE_TYPE: ${{ inputs.load_test_db_instance_type }} run: | args=() @@ -62,4 +68,8 @@ jobs: args+=(--skip-terraform-destroy) fi + if [ -n "$DB_INSTANCE_TYPE" ]; then + args+=(--db-instance-type "$DB_INSTANCE_TYPE") + fi + bash scripts/load_test/stop.sh "${args[@]}" diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index 16fb888..c8ac5cb 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -674,12 +674,42 @@ jobs: - name: Terraform Init working-directory: environment/load_test run: terraform init + - name: Select DB Type for Plan + id: db_type + working-directory: environment/load_test + run: | + set -euo pipefail + state_json="$(terraform show -json)" + if ! jq -e '.format_version | type == "string"' <<< "$state_json" >/dev/null; then + echo "Terraform state JSON could not be parsed" >&2 + exit 1 + fi + db_count="$(jq -r '[.values.root_module.resources[]? + | select(.address == "aws_instance.load_test_db")] | length' <<< "$state_json")" + if [[ "$db_count" == "1" ]]; then + db_type="$(jq -r '[.values.root_module.resources[]? + | select(.address == "aws_instance.load_test_db") | .values.instance_type][0] // empty' <<< "$state_json")" + if [[ -z "$db_type" ]]; then + echo "DB EC2 exists in state without an instance type" >&2 + exit 1 + fi + elif [[ "$db_count" == "0" ]]; then + # Plan-only fixture when Start has not created the DB EC2. + db_type="t4g.medium" + else + echo "Unexpected DB EC2 resources in Terraform state" >&2 + exit 1 + fi + echo "value=$db_type" >> "$GITHUB_OUTPUT" - name: Terraform Plan id: plan working-directory: environment/load_test + env: + LOAD_TEST_DB_INSTANCE_TYPE: ${{ steps.db_type.outputs.value }} run: | terraform plan -no-color -out=tfplan \ -var-file="../../config/secrets/load_test.tfvars" \ + -var="load_test_db_instance_type=$LOAD_TEST_DB_INSTANCE_TYPE" \ 2>&1 | tee plan_output.txt echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - name: Check Plan (L2) diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index 2f352e5..86f0c3e 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -13,6 +13,11 @@ output "load_test_db_instance_id" { value = aws_instance.load_test_db.id } +output "load_test_db_instance_type" { + description = "Load-test MySQL EC2 instance type selected by Start" + value = aws_instance.load_test_db.instance_type +} + output "load_test_db_private_ip" { description = "Load-test MySQL EC2 private IP" value = aws_instance.load_test_db.private_ip diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 62cbe2f..44cc4d4 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -104,7 +104,11 @@ variable "load_test_db_instance_name" { variable "load_test_db_instance_type" { description = "load-test MySQL EC2 인스턴스 타입입니다." type = string - default = "t4g.medium" + + validation { + condition = length(trimspace(var.load_test_db_instance_type)) > 0 + error_message = "load_test_db_instance_type must be supplied by Load Test Start." + } } variable "load_test_db_ami_id" { diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 66dc0a5..ef8187d 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -45,6 +45,7 @@ GitHub에서 **Actions > Load Test Start**를 수동 실행합니다. 입력값: +- `load_test_db_instance_type`: 필수 EC2 인스턴스 타입입니다. 기본값은 없으며 Start 실행 시마다 입력합니다. 선택한 값은 Terraform state에 기록되어 Run/Stop이 재사용합니다. - `switch_stage_to_loadtest`: `true` 또는 `false` - `true`이면 데이터 준비 후 stage 앱을 `dev,loadtest` profile로 재기동합니다. @@ -59,7 +60,7 @@ Start workflow 동작: 5. load-test MySQL EC2 user data가 prod MySQL 백업 S3 bucket의 최신 dump를 내려받아 복원합니다. 6. `scripts/load_test/start.sh`가 Terraform output에서 필요한 값을 읽습니다. 7. `scripts/load_test/start.sh`가 SSM으로 load-test MySQL EC2의 `cloud-init` 완료와 `/opt/solid-connection/load-test-db-ready` marker 생성을 확인합니다. 기본 대기 시간은 3600초입니다. -8. `switch_stage_to_loadtest=true`이면 stage 앱을 `dev,loadtest` profile로 재기동합니다. username/password는 앱의 Parameter Store 연동이 loadtest 경로에서 읽습니다. +8. `switch_stage_to_loadtest=true`이면 Nginx upstream으로 활성 blue/green 슬롯을 찾고 해당 슬롯의 앱만 같은 포트에서 `dev,loadtest` profile로 재기동합니다. 관리 포트의 health check 성공 후 stage MySQL 컨테이너를 중지합니다. 앱 전환에 실패하면 stage MySQL을 기동한 상태로 `dev` profile을 복구하고 Start를 실패 처리합니다. username/password는 앱의 Parameter Store 연동이 loadtest 경로에서 읽습니다. Start workflow는 load-generator EC2를 만들지 않습니다. 부하 생성용 EC2는 비용 누수를 막기 위해 Run workflow에서만 생성합니다. @@ -157,12 +158,15 @@ GitHub에서 **Actions > Load Test Stop**을 수동 실행합니다. - `true`이면 stage 앱을 기존 dev compose 구성으로 되돌립니다. - `destroy_infra`: `true` 또는 `false` - `true`이면 load-test Terraform stack을 destroy합니다. +- `load_test_db_instance_type`: 선택값입니다. Start의 Terraform apply가 DB EC2 생성 전에 실패해 state에 인스턴스 타입이 없을 때, 부분 생성된 리소스를 정리하려면 Start에서 선택했던 타입을 입력합니다. DB EC2가 state에 있으면 그 타입을 사용하며, 다른 값을 입력하면 Stop이 실패합니다. Stop workflow 동작: 1. `scripts/load_test/stop.sh`가 `environment/load_test`에서 Terraform init을 실행합니다. -2. `restore_stage_dev=true`이면 stage를 dev datasource 구성으로 복구합니다. -3. `destroy_infra=true`이면 Terraform destroy로 load-test MySQL EC2, 데이터 EBS 볼륨, 보안 그룹과 남아 있는 load-generator EC2를 제거합니다. +2. `restore_stage_dev=true`이면 stage MySQL 컨테이너를 다시 기동하고 준비 상태를 확인한 뒤 활성 슬롯의 앱을 같은 포트에서 `dev` profile로 재기동합니다. 활성 앱 컨테이너가 종료된 상태여도 프로젝트와 이미지가 확인되면 재생성합니다. MySQL 준비 전에 실패하면 정상 앱은 그대로 두고, dev 앱 재생성을 시도한 뒤 실패하면 진입 당시 `dev,loadtest`였던 경우에만 load-test profile 복구를 시도합니다. 롤백 후 MySQL을 중지한 경우 앱 health를 다시 확인합니다. +3. `destroy_infra=true`이면 Terraform state를 읽고 DB 타입 충돌을 stage 전환 전에 검사합니다. dev 앱 health check 성공 후 Terraform destroy로 load-test MySQL EC2, 데이터 EBS 볼륨, 보안 그룹과 남아 있는 load-generator EC2를 제거합니다. DB EC2가 state에 없다는 것이 확인되면 입력한 인스턴스 타입으로 부분 생성 리소스를 정리합니다. state 조회에 실패하면 중단합니다. `restore_stage_dev=false`와 `destroy_infra=true`의 조합은 거부합니다. + +Start부터 Stop까지 stage 배포를 겹치지 않게 운영합니다. stage MySQL은 컨테이너를 중지했다가 재기동하며 데이터 볼륨과 컨테이너를 삭제하지 않습니다. ## 참고 @@ -173,7 +177,7 @@ Stop workflow 동작: - load-test MySQL EC2 보안 그룹은 prod/stage API EC2 보안 그룹에서 들어오는 MySQL 접근만 허용합니다. - Load-test DB defaults updated after PR review: - `load_test_db_ami_id`: `ami-0501a03cd31b53e82` (`solid-connection-db-mysql-8.4.8-arm64-ubuntu24.04-awscli-recovery-tools`). - - `load_test_db_instance_type`: `t4g.medium`, matching the arm64 DB AMI family. + - `load_test_db_instance_type`: Start workflow의 필수 입력값입니다. 선택 시 DB AMI의 CPU 아키텍처와 호환되는 타입을 사용합니다. - `load_test_db_subnet_id`: when omitted, the prod DB subnet is used so the existing S3 Gateway Endpoint route is available. - `load_test_db_associate_public_ip`: remains `false`; SSM access is provided through Terraform-managed SSM interface endpoints. - `load_test_db_instance_profile_name`: defaults to the pre-created dedicated `solid-connection-load-test-db` instance profile. Terraform does not create IAM resources. diff --git a/scripts/load_test/run_k6.sh b/scripts/load_test/run_k6.sh index 6e86e5b..412d76b 100644 --- a/scripts/load_test/run_k6.sh +++ b/scripts/load_test/run_k6.sh @@ -103,6 +103,8 @@ tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" } +source scripts/load_test/terraform_db_type.sh + runner_targets=( -target=aws_security_group.load_generator -target=aws_instance.load_generator @@ -113,7 +115,8 @@ destroy_runner() { local cleanup_code=0 if [[ "$DESTROY_RUNNER" == "true" ]]; then - terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" "${runner_targets[@]}" || cleanup_code="$?" + terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" \ + -var="load_test_db_instance_type=$db_instance_type" "${runner_targets[@]}" || cleanup_code="$?" fi if [[ "$exit_code" -ne 0 ]]; then @@ -298,7 +301,9 @@ sync_file() { } terraform -chdir="$TERRAFORM_DIR" init -terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" "${runner_targets[@]}" +db_instance_type="$(load_test_db_instance_type_from_state)" +terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" \ + -var="load_test_db_instance_type=$db_instance_type" "${runner_targets[@]}" trap destroy_runner EXIT diff --git a/scripts/load_test/stage_app.sh b/scripts/load_test/stage_app.sh new file mode 100644 index 0000000..a35923a --- /dev/null +++ b/scripts/load_test/stage_app.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Sent to the stage instance through SSM by start.sh and stop.sh. +set -euo pipefail + +: "${STAGE_MODE:?}" +: "${STAGE_APP_DIR:?}" +: "${STAGE_COMPOSE_FILE:?}" + +if [[ "$STAGE_MODE" != "loadtest" && "$STAGE_MODE" != "dev" ]]; then + echo "Unknown stage mode: $STAGE_MODE" >&2 + exit 1 +fi + +cd "$STAGE_APP_DIR" +upstream_file="${STAGE_UPSTREAM_FILE:-/etc/nginx/conf.d/upstream.conf}" +if [[ ! -r "$upstream_file" || ! -f "$STAGE_COMPOSE_FILE" ]]; then + echo "Stage upstream or Compose file is unavailable" >&2 + exit 1 +fi + +servers=() +while IFS= read -r server; do + servers+=("$server") +done < <(awk ' + /^[[:space:]]*upstream[[:space:]]+app_backend[[:space:]]*\{/ { in_backend = 1; next } + in_backend && /^[[:space:]]*\}/ { exit } + in_backend && /^[[:space:]]*server[[:space:]]/ { print } +' "$upstream_file") +server_pattern='^[[:space:]]*server[[:space:]]+127\.0\.0\.1:(8080|9080);[[:space:]]*$' +if (( ${#servers[@]} != 1 )) || [[ ! "${servers[0]}" =~ $server_pattern ]]; then + echo "Expected exactly one active 127.0.0.1:8080 or :9080 upstream" >&2 + exit 1 +fi + +app_port="${BASH_REMATCH[1]}" +if [[ "$app_port" == "8080" ]]; then + slot="blue" + management_port="8081" +else + slot="green" + management_port="9081" +fi + +container="solid-connection-dev-$slot" +project="$container" +if ! app_running="$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)"; then + echo "Active slot container is unavailable: $container" >&2 + exit 1 +fi +if [[ "$app_running" != "true" && ( "$STAGE_MODE" != "dev" || "$app_running" != "false" ) ]]; then + echo "Active slot container is unavailable: $container" >&2 + exit 1 +fi +if [[ "$(docker inspect -f '{{ index .Config.Labels "com.docker.compose.project" }}' "$container")" != "$project" ]]; then + echo "Active slot container has the wrong Compose project: $container" >&2 + exit 1 +fi + +current_image="$(docker inspect -f '{{.Config.Image}}' "$container")" +container_env="$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$container")" +initial_profile="" +while IFS= read -r env_entry; do + if [[ "$env_entry" == SPRING_PROFILES_ACTIVE=* ]]; then + initial_profile="${env_entry#SPRING_PROFILES_ACTIVE=}" + fi +done <<< "$container_env" +image_pattern='^ghcr\.io/([^/]+)/solid-connection-dev:([^:]+)$' +if [[ ! "$current_image" =~ $image_pattern ]]; then + echo "Cannot infer the active app image from $current_image" >&2 + exit 1 +fi +owner="${BASH_REMATCH[1]}" +image_tag="${BASH_REMATCH[2]}" + +mysql_container="solid-connection-dev-mysql" +mysql_project="$(docker inspect -f '{{ index .Config.Labels "com.docker.compose.project" }}' "$mysql_container")" +if [[ -z "$mysql_project" || "$mysql_project" == "" ]]; then + echo "Cannot identify the stage MySQL Compose project" >&2 + exit 1 +fi +if [[ "$STAGE_MODE" == "loadtest" && "$(docker inspect -f '{{.State.Running}}' "$mysql_container")" != "true" ]]; then + echo "Stage MySQL must be running before the load-test switch" >&2 + exit 1 +fi + +override_file="$(mktemp "$STAGE_APP_DIR/.docker-compose.loadtest.XXXXXX")" +trap 'rm -f "$override_file"' EXIT +cat > "$override_file" <<'YAML' +services: + solid-connection-dev: + environment: + SPRING_PROFILES_ACTIVE: dev,loadtest +YAML + +recreate_app() { + local profile="$1" + local files=(-f "$STAGE_COMPOSE_FILE") + if [[ "$profile" == "loadtest" ]]; then + files+=(-f "$override_file") + fi + SLOT="$slot" APP_PORT="$app_port" MANAGEMENT_PORT="$management_port" \ + OWNER_LOWERCASE="$owner" IMAGE_TAG="$image_tag" \ + docker compose -p "$project" "${files[@]}" up -d --no-deps --force-recreate solid-connection-dev +} + +wait_for_app() { + local attempt + for (( attempt = 1; attempt <= ${STAGE_HEALTH_ATTEMPTS:-30}; attempt++ )); do + if curl -fsS --max-time 3 "http://127.0.0.1:${management_port}/actuator/health" >/dev/null; then + return 0 + fi + sleep "${STAGE_HEALTH_INTERVAL_SECONDS:-5}" + done + return 1 +} + +wait_for_mysql() { + local attempt + for (( attempt = 1; attempt <= ${STAGE_MYSQL_ATTEMPTS:-30}; attempt++ )); do + if docker exec "$mysql_container" mysqladmin ping -h 127.0.0.1 --silent >/dev/null 2>&1; then + return 0 + fi + sleep "${STAGE_HEALTH_INTERVAL_SECONDS:-5}" + done + return 1 +} + +dev_recreate_started=false +restore_dev() { + docker compose -p "$mysql_project" -f "$STAGE_COMPOSE_FILE" start mysql || return 1 + wait_for_mysql || return 1 + dev_recreate_started=true + recreate_app dev && wait_for_app +} + +if [[ "$STAGE_MODE" == "loadtest" ]]; then + if ! recreate_app loadtest || ! wait_for_app || + ! docker compose -p "$mysql_project" -f "$STAGE_COMPOSE_FILE" stop mysql || + ! wait_for_app; then + echo "Load-test switch failed; restoring dev in the same slot" >&2 + restore_dev || echo "Stage dev rollback also failed" >&2 + exit 1 + fi +else + if ! restore_dev; then + echo "Stage dev restore failed; retaining the load-test DB EC2" >&2 + if [[ "$initial_profile" == "dev,loadtest" && + ( "$dev_recreate_started" == "true" || "$app_running" != "true" ) ]]; then + recreate_app loadtest && wait_for_app && + docker compose -p "$mysql_project" -f "$STAGE_COMPOSE_FILE" stop mysql && + wait_for_app || + echo "Stage load-test rollback also failed" >&2 + fi + exit 1 + fi +fi diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index 3ecacf8..cb35b6a 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -4,6 +4,7 @@ set -euo pipefail TERRAFORM_DIR="environment/load_test" VAR_FILE="../../config/secrets/load_test.tfvars" DATABASE_NAME="" +DB_INSTANCE_TYPE="" SWITCH_STAGE_TO_LOADTEST="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" @@ -18,6 +19,7 @@ Options: --terraform-dir PATH Default: environment/load_test --var-file PATH Default: ../../config/secrets/load_test.tfvars --database-name VALUE Default: Terraform output load_test_db_name + --db-instance-type VALUE Required when applying Terraform --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml @@ -32,6 +34,7 @@ while [[ $# -gt 0 ]]; do --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; --var-file) VAR_FILE="$2"; shift 2 ;; --database-name) DATABASE_NAME="$2"; shift 2 ;; + --db-instance-type) DB_INSTANCE_TYPE="$2"; shift 2 ;; --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; @@ -160,8 +163,10 @@ wait_for_load_test_db_restore() { } if [[ "$SKIP_TERRAFORM_APPLY" != "true" ]]; then + require_value --db-instance-type "$DB_INSTANCE_TYPE" terraform -chdir="$TERRAFORM_DIR" init - terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" + terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" \ + -var="load_test_db_instance_type=$DB_INSTANCE_TYPE" fi stage_instance_id="$(tf_output stage_api_instance_id)" @@ -179,17 +184,10 @@ if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then stage_commands_json="$(jq -cn \ --arg app_dir "$STAGE_APP_DIR" \ --arg compose_file "$STAGE_COMPOSE_FILE" \ + --rawfile script scripts/load_test/stage_app.sh \ '{ commands: [ - "set -euo pipefail", - "cd \($app_dir)", - "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", - "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", - "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", - "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", - "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", - "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", - "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" + "STAGE_MODE=loadtest STAGE_APP_DIR=\($app_dir | @sh) STAGE_COMPOSE_FILE=\($compose_file | @sh) bash -s <<\\STAGE_APP_SCRIPT\n\($script)\nSTAGE_APP_SCRIPT" ] }')" diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh index 005a61c..82f1fb2 100644 --- a/scripts/load_test/stop.sh +++ b/scripts/load_test/stop.sh @@ -8,6 +8,7 @@ STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-900}" SKIP_TERRAFORM_DESTROY="false" +DB_INSTANCE_TYPE="" usage() { cat <<'EOF' @@ -19,6 +20,7 @@ Options: --restore-stage-dev Restart stage app through SSM with dev profile --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml + --db-instance-type VALUE Required only if Start failed before the DB EC2 entered state --ssm-command-timeout-seconds Default: 900 --skip-terraform-destroy -h, --help @@ -32,6 +34,7 @@ while [[ $# -gt 0 ]]; do --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --db-instance-type) DB_INSTANCE_TYPE="$2"; shift 2 ;; --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; -h|--help) usage; exit 0 ;; @@ -50,9 +53,10 @@ require_command terraform require_command aws require_command jq -tf_output() { - terraform -chdir="$TERRAFORM_DIR" output -raw "$1" -} +if [[ "$RESTORE_STAGE_DEV" != "true" && "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then + echo "Destroy requires --restore-stage-dev so stage cannot lose its DB" >&2 + exit 1 +fi send_ssm_command() { local instance_id="$1" @@ -105,23 +109,57 @@ send_ssm_command() { terraform -chdir="$TERRAFORM_DIR" init +if [[ "$RESTORE_STAGE_DEV" == "true" || "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then + if ! state_json="$(terraform -chdir="$TERRAFORM_DIR" show -json)" || + ! jq -e '.values | type == "object"' <<< "$state_json" >/dev/null; then + echo "Cannot read load-test Terraform state; refusing to change stage or destroy" >&2 + exit 1 + fi +fi + +if [[ "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then + db_resource_count="$(jq -r '[.values.root_module.resources[]? + | select(.address == "aws_instance.load_test_db")] | length' <<< "$state_json")" + if [[ "$db_resource_count" == "1" ]]; then + db_instance_type="$(jq -r '[.values.root_module.resources[]? + | select(.address == "aws_instance.load_test_db") | .values.instance_type][0] // empty' <<< "$state_json")" + if [[ -z "$db_instance_type" ]]; then + echo "DB EC2 exists in state without an instance type; refusing to destroy" >&2 + exit 1 + fi + elif [[ "$db_resource_count" == "0" && -n "$DB_INSTANCE_TYPE" ]]; then + db_instance_type="$DB_INSTANCE_TYPE" + elif [[ "$db_resource_count" == "0" ]]; then + echo "DB EC2 is absent from state; pass --db-instance-type to destroy the partial stack" >&2 + exit 1 + else + echo "Unexpected DB EC2 resources in Terraform state; refusing to destroy" >&2 + exit 1 + fi + if [[ -n "$DB_INSTANCE_TYPE" && "$DB_INSTANCE_TYPE" != "$db_instance_type" ]]; then + echo "Provided DB instance type differs from Terraform state: $db_instance_type" >&2 + exit 1 + fi +fi + if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then - stage_instance_id="$(tf_output stage_api_instance_id)" + stage_instance_id="$(jq -r ' + [.values.root_module.resources[]? + | select(.address == "data.aws_instance.stage_api") | .values.id][0] + // .values.outputs.stage_api_instance_id.value // empty + ' <<< "$state_json")" + if [[ -z "$stage_instance_id" ]]; then + echo "Cannot identify the stage API instance from Terraform state; refusing to destroy" >&2 + exit 1 + fi stage_commands_json="$(jq -cn \ --arg app_dir "$STAGE_APP_DIR" \ --arg compose_file "$STAGE_COMPOSE_FILE" \ + --rawfile script scripts/load_test/stage_app.sh \ '{ commands: [ - "set -euo pipefail", - "cd \($app_dir)", - "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", - "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", - "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", - "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", - "rm -f docker-compose.loadtest.override.yml", - "docker compose -f \($compose_file) down || true", - "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) up -d" + "STAGE_MODE=dev STAGE_APP_DIR=\($app_dir | @sh) STAGE_COMPOSE_FILE=\($compose_file | @sh) bash -s <<\\STAGE_APP_SCRIPT\n\($script)\nSTAGE_APP_SCRIPT" ] }')" @@ -129,7 +167,8 @@ if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then fi if [[ "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then - terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" + terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" \ + -var="load_test_db_instance_type=$db_instance_type" fi echo "Load test environment has been stopped." diff --git a/scripts/load_test/terraform_db_type.sh b/scripts/load_test/terraform_db_type.sh new file mode 100644 index 0000000..c28ca9f --- /dev/null +++ b/scripts/load_test/terraform_db_type.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Read the type selected by Start from the DB resource in state. +load_test_db_instance_type_from_state() { + local state_json value + if ! state_json="$(terraform -chdir="$TERRAFORM_DIR" show -json)"; then + echo "Cannot read Terraform state for the load-test DB instance type" >&2 + return 1 + fi + if ! value="$(jq -r ' + first(.values.root_module.resources[]? + | select(.address == "aws_instance.load_test_db") + | .values.instance_type) // empty + ' <<< "$state_json")"; then + echo "Cannot parse Terraform state for the load-test DB instance type" >&2 + return 1 + fi + if [[ -z "$value" ]]; then + echo "Cannot read the load-test DB instance type from Terraform state; run Start first" >&2 + return 1 + fi + printf '%s\n' "$value" +} diff --git a/scripts/load_test/tests/test_plan_db_type.py b/scripts/load_test/tests/test_plan_db_type.py new file mode 100644 index 0000000..f9657f1 --- /dev/null +++ b/scripts/load_test/tests/test_plan_db_type.py @@ -0,0 +1,95 @@ +"""Run the DB-type selection step from the Terraform Plan workflow.""" + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +WORKFLOW = ROOT / ".github/workflows/terraform-plan.yml" + + +def selection_script(): + lines = WORKFLOW.read_text().splitlines() + start = lines.index(" - name: Select DB Type for Plan") + run = lines.index(" run: |", start) + end = next( + i for i in range(run + 1, len(lines)) + if lines[i].startswith(" - name: ") + ) + return "\n".join(line[10:] for line in lines[run + 1:end]) + "\n" + + +class PlanDbTypeTests(unittest.TestCase): + def run_selection(self, state, *, fail_show=False): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + bin_dir = root / "bin" + bin_dir.mkdir() + terraform = bin_dir / "terraform" + terraform.write_text( + "#!/usr/bin/env bash\n" + '[[ "$*" == "show -json" ]] || exit 2\n' + '[[ "$MOCK_FAIL_SHOW" != true ]] || exit 1\n' + 'cat "$MOCK_STATE_FILE"\n' + ) + terraform.chmod(0o755) + state_file = root / "state.json" + state_file.write_text(state) + output = root / "output" + env = os.environ.copy() + env.update( + PATH=f"{bin_dir}:{env['PATH']}", + MOCK_STATE_FILE=str(state_file), + MOCK_FAIL_SHOW="true" if fail_show else "false", + GITHUB_OUTPUT=str(output), + ) + result = subprocess.run( + ["bash", "-c", selection_script()], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + check=False, + ) + return result, output.read_text() if output.exists() else "" + + def test_uses_fixture_only_when_db_is_absent(self): + result, output = self.run_selection('{"format_version":"1.0"}') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output, "value=t4g.medium\n") + + def test_uses_existing_db_instance_type(self): + state = { + "format_version": "1.0", + "values": {"root_module": {"resources": [ + {"address": "aws_instance.load_test_db", + "values": {"instance_type": "t4g.large"}} + ]}}, + } + result, output = self.run_selection(json.dumps(state)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output, "value=t4g.large\n") + + def test_refuses_unreadable_state(self): + result, output = self.run_selection("not-json", fail_show=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(output, "") + + def test_refuses_db_resource_without_type(self): + state = { + "format_version": "1.0", + "values": {"root_module": {"resources": [ + {"address": "aws_instance.load_test_db", "values": {}} + ]}}, + } + result, output = self.run_selection(json.dumps(state)) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(output, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/load_test/tests/test_stage_app.py b/scripts/load_test/tests/test_stage_app.py new file mode 100644 index 0000000..485949d --- /dev/null +++ b/scripts/load_test/tests/test_stage_app.py @@ -0,0 +1,203 @@ +"""Behavior tests for the shell script sent to the stage EC2 through SSM.""" + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts/load_test/stage_app.sh" + + +class StageAppTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.bin = self.root / "bin" + self.bin.mkdir() + (self.root / "docker-compose.dev.yml").write_text("services: {}\n") + self.upstream = self.root / "upstream.conf" + self.log = self.root / "calls.log" + self.mode = self.root / "app-mode" + self._command( + "docker", + r'''#!/usr/bin/env bash +set -e +printf 'docker %s\n' "$*" >> "$MOCK_LOG" +if [[ "$1" == "inspect" ]]; then + case "$4" in + solid-connection-dev-mysql) + case "$3" in + *State.Running*) echo true ;; + *compose.project*) echo solid-connection-dev ;; + esac ;; + solid-connection-dev-blue|solid-connection-dev-green) + case "$3" in + *State.Running*) echo "$MOCK_APP_RUNNING" ;; + *compose.project*) echo "$4" ;; + *Config.Image*) echo ghcr.io/solid-connection/solid-connection-dev:tag-123 ;; + *Config.Env*) echo "SPRING_PROFILES_ACTIVE=$MOCK_INITIAL_PROFILE" ;; + esac ;; + esac +elif [[ "$1" == "compose" ]]; then + action="" + service="${!#}" + for arg in "$@"; do + [[ "$arg" == "up" || "$arg" == "stop" ]] && action="$arg" + done + if [[ "$action" == "up" && "$service" == "solid-connection-dev" ]]; then + if [[ "$*" == *".docker-compose.loadtest."* ]]; then + echo loadtest > "$MOCK_MODE" + else + echo dev > "$MOCK_MODE" + fi + elif [[ "$action" == "stop" && "$service" == "mysql" ]]; then + touch "$MOCK_MYSQL_STOPPED" + fi +elif [[ "$1" == "exec" ]]; then + [[ "${MOCK_MYSQL_FAIL:-false}" != true ]] +fi +''', + ) + self._command( + "curl", + r'''#!/usr/bin/env bash +printf 'curl %s\n' "$*" >> "$MOCK_LOG" +mode="$(cat "$MOCK_MODE")" +if [[ "${MOCK_FAIL_AFTER_MYSQL_STOP:-false}" == true && "$mode" == loadtest && -f "$MOCK_MYSQL_STOPPED" ]]; then + exit 22 +fi +[[ "$mode" != "${MOCK_FAIL_MODE:-none}" ]] +''', + ) + + def _command(self, name, content): + path = self.bin / name + path.write_text(content) + path.chmod(0o755) + + def run_stage( + self, mode, port, *, fail_mode="none", mysql_fail=False, + fail_after_mysql_stop=False, initial_profile=None, app_running=True + ): + self.upstream.write_text( + f"upstream app_backend {{\n server 127.0.0.1:{port};\n}}\n" + ) + initial_profile = initial_profile or ("dev" if mode == "loadtest" else "dev,loadtest") + self.mode.write_text("loadtest" if initial_profile == "dev,loadtest" else "dev") + env = os.environ.copy() + env.update( + PATH=f"{self.bin}:{env['PATH']}", + STAGE_MODE=mode, + STAGE_APP_DIR=str(self.root), + STAGE_COMPOSE_FILE="docker-compose.dev.yml", + STAGE_UPSTREAM_FILE=str(self.upstream), + STAGE_HEALTH_ATTEMPTS="2", + STAGE_MYSQL_ATTEMPTS="2", + STAGE_HEALTH_INTERVAL_SECONDS="0", + MOCK_LOG=str(self.log), + MOCK_MODE=str(self.mode), + MOCK_MYSQL_STOPPED=str(self.root / "mysql-stopped"), + MOCK_FAIL_MODE=fail_mode, + MOCK_MYSQL_FAIL="true" if mysql_fail else "false", + MOCK_FAIL_AFTER_MYSQL_STOP="true" if fail_after_mysql_stop else "false", + MOCK_INITIAL_PROFILE=initial_profile, + MOCK_APP_RUNNING="true" if app_running else "false", + ) + result = subprocess.run( + ["bash", str(SCRIPT)], env=env, text=True, capture_output=True, check=False + ) + calls = self.log.read_text().splitlines() if self.log.exists() else [] + return result, calls + + def test_loadtest_recreates_only_active_slot_then_stops_mysql(self): + for port, slot, management in [(8080, "blue", 8081), (9080, "green", 9081)]: + with self.subTest(slot=slot): + self.log.unlink(missing_ok=True) + result, calls = self.run_stage("loadtest", port) + self.assertEqual(result.returncode, 0, result.stderr) + app_up = next(i for i, call in enumerate(calls) if "up -d --no-deps --force-recreate solid-connection-dev" in call) + health = next(i for i, call in enumerate(calls) if f":{management}/actuator/health" in call) + mysql_stop = next(i for i, call in enumerate(calls) if "stop mysql" in call) + self.assertIn(f"-p solid-connection-dev-{slot}", calls[app_up]) + self.assertLess(app_up, health) + self.assertLess(health, mysql_stop) + self.assertFalse(any(" down" in call for call in calls)) + + def test_invalid_upstream_fails_before_compose(self): + result, calls = self.run_stage("loadtest", 7777) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(any("docker compose" in call for call in calls)) + + def test_failed_loadtest_health_restores_dev_and_mysql(self): + result, calls = self.run_stage("loadtest", 8080, fail_mode="loadtest") + self.assertNotEqual(result.returncode, 0) + mysql_start = next(i for i, call in enumerate(calls) if "start mysql" in call) + dev_up = max(i for i, call in enumerate(calls) if "up -d --no-deps --force-recreate solid-connection-dev" in call) + self.assertLess(mysql_start, dev_up) + self.assertEqual(self.mode.read_text().strip(), "dev") + + def test_loadtest_unhealthy_after_mysql_stop_restores_dev(self): + result, calls = self.run_stage("loadtest", 8080, fail_after_mysql_stop=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.mode.read_text().strip(), "dev") + self.assertTrue(any("stop mysql" in call for call in calls)) + self.assertTrue(any("start mysql" in call for call in calls)) + + def test_stop_starts_mysql_before_dev_app(self): + result, calls = self.run_stage("dev", 9080) + self.assertEqual(result.returncode, 0, result.stderr) + mysql_start = next(i for i, call in enumerate(calls) if "start mysql" in call) + mysql_ping = next(i for i, call in enumerate(calls) if "mysqladmin ping" in call) + dev_up = next(i for i, call in enumerate(calls) if "up -d --no-deps --force-recreate solid-connection-dev" in call) + self.assertLess(mysql_start, mysql_ping) + self.assertLess(mysql_ping, dev_up) + self.assertEqual(self.mode.read_text().strip(), "dev") + + def test_stop_recreates_exited_active_app(self): + result, calls = self.run_stage("dev", 8080, app_running=False) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any("up -d --no-deps --force-recreate solid-connection-dev" in call for call in calls)) + self.assertEqual(self.mode.read_text().strip(), "dev") + + def test_start_rejects_exited_active_app(self): + result, calls = self.run_stage("loadtest", 8080, app_running=False) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(any("docker compose" in call for call in calls)) + + def test_stop_failure_preserves_initial_dev_profile(self): + result, calls = self.run_stage("dev", 8080, mysql_fail=True, initial_profile="dev") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.mode.read_text().strip(), "dev") + self.assertFalse(any(".docker-compose.loadtest." in call for call in calls)) + self.assertFalse(any("stop mysql" in call for call in calls)) + + def test_failed_dev_health_returns_failure(self): + result, calls = self.run_stage("dev", 8080, fail_mode="dev") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.mode.read_text().strip(), "loadtest") + self.assertTrue(any(".docker-compose.loadtest." in call for call in calls)) + self.assertTrue(any("stop mysql" in call for call in calls)) + + def test_stop_rollback_checks_health_after_stopping_mysql(self): + result, calls = self.run_stage( + "dev", 8080, fail_mode="dev", fail_after_mysql_stop=True + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Stage load-test rollback also failed", result.stderr) + mysql_stop = max(i for i, call in enumerate(calls) if "stop mysql" in call) + self.assertTrue(any("curl " in call for call in calls[mysql_stop + 1:])) + + def test_mysql_not_ready_prevents_dev_switch(self): + result, calls = self.run_stage("dev", 8080, mysql_fail=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.mode.read_text().strip(), "loadtest") + self.assertFalse(any("up -d --no-deps --force-recreate solid-connection-dev" in call for call in calls)) + self.assertFalse(any("stop mysql" in call for call in calls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/load_test/tests/test_workflow_guards.py b/scripts/load_test/tests/test_workflow_guards.py new file mode 100644 index 0000000..e475d85 --- /dev/null +++ b/scripts/load_test/tests/test_workflow_guards.py @@ -0,0 +1,211 @@ +"""Guard tests for the load-test entry points without AWS or Terraform changes.""" + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] + + +class WorkflowGuardTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.bin = self.root / "bin" + self.bin.mkdir() + self.log = self.root / "calls.log" + self.ssm_parameters = self.root / "ssm-parameters.json" + self._command( + "terraform", + r'''#!/usr/bin/env bash +printf 'terraform %s\n' "$*" >> "$MOCK_LOG" +case "$*" in + *"output -raw stage_api_instance_id"*) + if [[ "${MOCK_PARTIAL_STATE:-false}" == true ]]; then echo 'No outputs found'; exit 0; fi + echo i-stage ;; + *"output -raw load_test_db_instance_type"*) + if [[ "${MOCK_OLD_STATE:-false}" == true || "${MOCK_PARTIAL_STATE:-false}" == true ]]; then echo 'No outputs found'; exit 0; fi + echo t4g.large ;; + *"show -json"*) + if [[ "${MOCK_STATE_ERROR:-false}" == true ]]; then echo 'state backend unavailable' >&2; exit 1; fi + if [[ "${MOCK_PARTIAL_STATE:-false}" == true ]]; then + echo '{"values":{"root_module":{"resources":[{"address":"data.aws_instance.stage_api","values":{"id":"i-stage"}}]}}}' + else + echo '{"values":{"root_module":{"resources":[{"address":"aws_instance.load_test_db","values":{"instance_type":"t4g.large"}},{"address":"data.aws_instance.stage_api","values":{"id":"i-stage"}}]}}}' + fi ;; +esac +''', + ) + self._command( + "aws", + r'''#!/usr/bin/env bash +printf 'aws %s\n' "$*" >> "$MOCK_LOG" +case "$*" in + *"ssm send-command"*) + while [[ $# -gt 0 ]]; do + if [[ "$1" == "--parameters" ]]; then + printf '%s' "$2" > "$MOCK_SSM_PARAMETERS" + break + fi + shift + done + echo command-123 ;; + *"ssm describe-instance-information"*) echo Online ;; + *"ssm get-command-invocation"*"--query Status"*) echo "$MOCK_SSM_STATUS" ;; +esac +''', + ) + self._command("sleep", "#!/usr/bin/env bash\nexit 0\n") + + def _command(self, name, content): + path = self.bin / name + path.write_text(content) + path.chmod(0o755) + + def run_script( + self, name, *args, status="Success", old_state=False, + partial_state=False, state_error=False + ): + env = os.environ.copy() + env.update( + PATH=f"{self.bin}:{env['PATH']}", + MOCK_LOG=str(self.log), + MOCK_SSM_PARAMETERS=str(self.ssm_parameters), + MOCK_SSM_STATUS=status, + MOCK_OLD_STATE="true" if old_state else "false", + MOCK_PARTIAL_STATE="true" if partial_state else "false", + MOCK_STATE_ERROR="true" if state_error else "false", + ) + result = subprocess.run( + ["bash", f"scripts/load_test/{name}.sh", *args], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + check=False, + ) + calls = self.log.read_text().splitlines() if self.log.exists() else [] + return result, calls + + def assert_stage_ssm_payload_runs(self, expected_mode): + command = json.loads(self.ssm_parameters.read_text())["commands"][0] + self.assertIn(f"STAGE_MODE={expected_mode} ", command) + lines = command.splitlines() + self.assertEqual(lines[-1], "STAGE_APP_SCRIPT") + self.assertEqual("\n".join(lines[1:-1]), (ROOT / "scripts/load_test/stage_app.sh").read_text()) + probe = "\n".join((lines[0], "printf 'payload-ok\\n'", lines[-1])) + result = subprocess.run(["bash", "-c", probe], text=True, capture_output=True, check=False) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "payload-ok\n") + + def run_db_type_reader(self, *, state_error=False): + env = os.environ.copy() + env.update( + PATH=f"{self.bin}:{env['PATH']}", + MOCK_LOG=str(self.log), + MOCK_STATE_ERROR="true" if state_error else "false", + ) + return subprocess.run( + ["bash", "-c", "TERRAFORM_DIR=environment/load_test; " + "source scripts/load_test/terraform_db_type.sh; " + "load_test_db_instance_type_from_state"], + cwd=ROOT, env=env, text=True, capture_output=True, check=False, + ) + + def test_start_requires_db_type_before_terraform(self): + result, calls = self.run_script("start") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--db-instance-type", result.stderr) + self.assertFalse(any(call.startswith("terraform ") for call in calls)) + + def test_start_stage_ssm_payload_has_working_heredoc(self): + result, _ = self.run_script("start", "--skip-terraform-apply", "--switch-stage-to-loadtest") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_stage_ssm_payload_runs("loadtest") + + def test_stop_rejects_destroy_without_restore(self): + result, calls = self.run_script("stop") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Destroy requires", result.stderr) + self.assertFalse(any(call.startswith("terraform ") for call in calls)) + + def test_stop_stage_ssm_payload_has_working_heredoc(self): + result, _ = self.run_script("stop", "--restore-stage-dev", "--skip-terraform-destroy") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_stage_ssm_payload_runs("dev") + + def test_stop_keeps_db_when_stage_restore_fails(self): + result, calls = self.run_script("stop", "--restore-stage-dev", status="Failed") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(any(" destroy " in call for call in calls)) + + def test_stop_uses_selected_type_after_successful_restore(self): + result, calls = self.run_script("stop", "--restore-stage-dev") + self.assertEqual(result.returncode, 0, result.stderr) + stage_status = next(i for i, call in enumerate(calls) if "--query Status" in call) + destroy = next(i for i, call in enumerate(calls) if " destroy " in call) + self.assertLess(stage_status, destroy) + self.assertIn("-var=load_test_db_instance_type=t4g.large", calls[destroy]) + + def test_stop_can_read_type_from_state_before_output_was_added(self): + result, calls = self.run_script("stop", "--restore-stage-dev", old_state=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any("show -json" in call for call in calls)) + self.assertFalse(any("output -raw load_test_db_instance_type" in call for call in calls)) + self.assertTrue(any("-var=load_test_db_instance_type=t4g.large" in call for call in calls)) + + def test_stop_cleans_partial_state_with_explicit_db_type(self): + result, calls = self.run_script( + "stop", "--restore-stage-dev", "--db-instance-type", "t4g.medium", + partial_state=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any("ssm send-command --instance-ids i-stage " in call for call in calls)) + self.assertFalse(any("output -raw " in call for call in calls)) + stage_status = next(i for i, call in enumerate(calls) if "--query Status" in call) + destroy = next(i for i, call in enumerate(calls) if " destroy " in call) + self.assertLess(stage_status, destroy) + self.assertIn("-var=load_test_db_instance_type=t4g.medium", calls[destroy]) + + def test_stop_keeps_partial_state_without_explicit_db_type(self): + result, calls = self.run_script("stop", "--restore-stage-dev", partial_state=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("--db-instance-type", result.stderr) + self.assertFalse(any(" destroy " in call for call in calls)) + + def test_stop_rejects_db_type_conflicting_with_state(self): + result, calls = self.run_script("stop", "--restore-stage-dev", "--db-instance-type", "t4g.medium") + self.assertNotEqual(result.returncode, 0) + self.assertIn("differs from Terraform state", result.stderr) + self.assertFalse(any("ssm send-command" in call for call in calls)) + self.assertFalse(any(" destroy " in call for call in calls)) + + def test_stop_rejects_state_read_failure_before_stage_restore(self): + result, calls = self.run_script( + "stop", "--restore-stage-dev", "--db-instance-type", "t4g.medium", + state_error=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Cannot read load-test Terraform state", result.stderr) + self.assertFalse(any("ssm send-command" in call for call in calls)) + self.assertFalse(any(" destroy " in call for call in calls)) + + def test_db_type_reader_uses_resource_value(self): + result = self.run_db_type_reader() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "t4g.large") + self.assertFalse(any("output -raw" in call for call in self.log.read_text().splitlines())) + + def test_db_type_reader_rejects_state_read_error(self): + result = self.run_db_type_reader(state_error=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Cannot read Terraform state", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify/README.md b/scripts/verify/README.md index 44f3124..2316f8f 100644 --- a/scripts/verify/README.md +++ b/scripts/verify/README.md @@ -42,6 +42,14 @@ scripts/verify/verify.sh all --expect .verify/expect.txt | `git submodule update --init --recursive` (tfvars) | `SKIPPED` | | prod: DB EC2 로의 SSM 포트 포워딩 권한 | `SKIPPED` | +`load_test`는 DB EC2 인스턴스 타입의 Terraform 기본값이 없습니다. 로컬 L2를 실행할 때 DB EC2가 이미 state에 있으면 그 인스턴스 타입을 지정하세요. 아래 `t4g.medium`은 DB EC2가 없는 경우의 plan 검증 예시값이며 Start 실행의 기본값이 아닙니다. PR CI도 DB EC2가 있으면 state의 타입을 사용합니다. + +```bash +AWS_PROFILE=solid-connection-readonly \ +TF_VAR_load_test_db_instance_type=t4g.medium \ +scripts/verify/verify.sh l2 --expect .verify/expect.txt +``` + plan 전문은 `.verify/l2/.plan.txt` 에 저장됩니다. (gitignore 대상) ### 의도 파일 (`--expect`) diff --git a/scripts/verify/verify.sh b/scripts/verify/verify.sh index 658a348..c312153 100755 --- a/scripts/verify/verify.sh +++ b/scripts/verify/verify.sh @@ -192,7 +192,7 @@ run_on_host() { docker_args+=(-v "$aws_dir:/root/.aws:ro") fi local var - for var in AWS_PROFILE AWS_REGION AWS_DEFAULT_REGION AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN; do + for var in AWS_PROFILE AWS_REGION AWS_DEFAULT_REGION AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN TF_VAR_load_test_db_instance_type; do [[ -n "${!var:-}" ]] && docker_args+=(-e "$var") done fi