Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion .github/workflows/load-test-start.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ name: Load Test Start
on:
workflow_dispatch:
inputs:
load_test_db_instance_type:
description: "Load-test MySQL EC2 instance type (required; no default)"
required: true
type: string
switch_stage_to_loadtest:
description: "Restart stage app with dev,loadtest profiles"
required: true
Expand Down Expand Up @@ -46,15 +50,22 @@ jobs:
sudo apt-get install -y jq

- name: Start load test environment
env:
DB_INSTANCE_TYPE: ${{ inputs.load_test_db_instance_type }}
run: |
if [ -z "$DB_INSTANCE_TYPE" ]; then
echo "load_test_db_instance_type is required" >&2
exit 1
fi

export TF_VAR_load_generator_instance_type="c7i.large"
export TF_VAR_create_load_generator="false"

if [ -n "${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" ]; then
export TF_VAR_load_generator_instance_profile_name="${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}"
fi

args=()
args=(--db-instance-type "$DB_INSTANCE_TYPE")

if [ "${{ inputs.switch_stage_to_loadtest }}" = "true" ]; then
args+=(--switch-stage-to-loadtest)
Expand Down
10 changes: 10 additions & 0 deletions .github/workflows/load-test-stop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ on:
required: true
default: true
type: boolean
load_test_db_instance_type:
description: "DB EC2 type selected at Start; needed only if DB EC2 was never created"
required: false
type: string

permissions:
id-token: write
Expand Down Expand Up @@ -51,6 +55,8 @@ jobs:
sudo apt-get install -y jq

- name: Stop load test environment
env:
DB_INSTANCE_TYPE: ${{ inputs.load_test_db_instance_type }}
run: |
args=()

Expand All @@ -62,4 +68,8 @@ jobs:
args+=(--skip-terraform-destroy)
fi

if [ -n "$DB_INSTANCE_TYPE" ]; then
args+=(--db-instance-type "$DB_INSTANCE_TYPE")
fi

bash scripts/load_test/stop.sh "${args[@]}"
30 changes: 30 additions & 0 deletions .github/workflows/terraform-plan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -674,12 +674,42 @@ jobs:
- name: Terraform Init
working-directory: environment/load_test
run: terraform init
- name: Select DB Type for Plan
id: db_type
working-directory: environment/load_test
run: |
set -euo pipefail
state_json="$(terraform show -json)"
if ! jq -e '.format_version | type == "string"' <<< "$state_json" >/dev/null; then
echo "Terraform state JSON could not be parsed" >&2
exit 1
fi
db_count="$(jq -r '[.values.root_module.resources[]?
| select(.address == "aws_instance.load_test_db")] | length' <<< "$state_json")"
if [[ "$db_count" == "1" ]]; then
db_type="$(jq -r '[.values.root_module.resources[]?
| select(.address == "aws_instance.load_test_db") | .values.instance_type][0] // empty' <<< "$state_json")"
if [[ -z "$db_type" ]]; then
echo "DB EC2 exists in state without an instance type" >&2
exit 1
fi
elif [[ "$db_count" == "0" ]]; then
# Plan-only fixture when Start has not created the DB EC2.
db_type="t4g.medium"
else
echo "Unexpected DB EC2 resources in Terraform state" >&2
exit 1
fi
echo "value=$db_type" >> "$GITHUB_OUTPUT"
- name: Terraform Plan
id: plan
working-directory: environment/load_test
env:
LOAD_TEST_DB_INSTANCE_TYPE: ${{ steps.db_type.outputs.value }}
run: |
terraform plan -no-color -out=tfplan \
-var-file="../../config/secrets/load_test.tfvars" \
-var="load_test_db_instance_type=$LOAD_TEST_DB_INSTANCE_TYPE" \
2>&1 | tee plan_output.txt
echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT
- name: Check Plan (L2)
Expand Down
5 changes: 5 additions & 0 deletions environment/load_test/output.tf
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ output "load_test_db_instance_id" {
value = aws_instance.load_test_db.id
}

output "load_test_db_instance_type" {
description = "Load-test MySQL EC2 instance type selected by Start"
value = aws_instance.load_test_db.instance_type
}

output "load_test_db_private_ip" {
description = "Load-test MySQL EC2 private IP"
value = aws_instance.load_test_db.private_ip
Expand Down
6 changes: 5 additions & 1 deletion environment/load_test/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,11 @@ variable "load_test_db_instance_name" {
variable "load_test_db_instance_type" {
description = "load-test MySQL EC2 인스턴스 타입입니다."
type = string
default = "t4g.medium"

Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
validation {
condition = length(trimspace(var.load_test_db_instance_type)) > 0
error_message = "load_test_db_instance_type must be supplied by Load Test Start."
}
}

variable "load_test_db_ami_id" {
Expand Down
12 changes: 8 additions & 4 deletions scripts/load_test/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ GitHub에서 **Actions > Load Test Start**를 수동 실행합니다.

입력값:

- `load_test_db_instance_type`: 필수 EC2 인스턴스 타입입니다. 기본값은 없으며 Start 실행 시마다 입력합니다. 선택한 값은 Terraform state에 기록되어 Run/Stop이 재사용합니다.
- `switch_stage_to_loadtest`: `true` 또는 `false`
- `true`이면 데이터 준비 후 stage 앱을 `dev,loadtest` profile로 재기동합니다.

Expand All @@ -59,7 +60,7 @@ Start workflow 동작:
5. load-test MySQL EC2 user data가 prod MySQL 백업 S3 bucket의 최신 dump를 내려받아 복원합니다.
6. `scripts/load_test/start.sh`가 Terraform output에서 필요한 값을 읽습니다.
7. `scripts/load_test/start.sh`가 SSM으로 load-test MySQL EC2의 `cloud-init` 완료와 `/opt/solid-connection/load-test-db-ready` marker 생성을 확인합니다. 기본 대기 시간은 3600초입니다.
8. `switch_stage_to_loadtest=true`이면 stage 앱을 `dev,loadtest` profile로 재기동합니다. username/password는 앱의 Parameter Store 연동이 loadtest 경로에서 읽습니다.
8. `switch_stage_to_loadtest=true`이면 Nginx upstream으로 활성 blue/green 슬롯을 찾고 해당 슬롯의 앱만 같은 포트에서 `dev,loadtest` profile로 재기동합니다. 관리 포트의 health check 성공 후 stage MySQL 컨테이너를 중지합니다. 앱 전환에 실패하면 stage MySQL을 기동한 상태로 `dev` profile을 복구하고 Start를 실패 처리합니다. username/password는 앱의 Parameter Store 연동이 loadtest 경로에서 읽습니다.

Start workflow는 load-generator EC2를 만들지 않습니다. 부하 생성용 EC2는 비용 누수를 막기 위해 Run workflow에서만 생성합니다.

Expand Down Expand Up @@ -157,12 +158,15 @@ GitHub에서 **Actions > Load Test Stop**을 수동 실행합니다.
- `true`이면 stage 앱을 기존 dev compose 구성으로 되돌립니다.
- `destroy_infra`: `true` 또는 `false`
- `true`이면 load-test Terraform stack을 destroy합니다.
- `load_test_db_instance_type`: 선택값입니다. Start의 Terraform apply가 DB EC2 생성 전에 실패해 state에 인스턴스 타입이 없을 때, 부분 생성된 리소스를 정리하려면 Start에서 선택했던 타입을 입력합니다. DB EC2가 state에 있으면 그 타입을 사용하며, 다른 값을 입력하면 Stop이 실패합니다.

Stop workflow 동작:

1. `scripts/load_test/stop.sh`가 `environment/load_test`에서 Terraform init을 실행합니다.
2. `restore_stage_dev=true`이면 stage를 dev datasource 구성으로 복구합니다.
3. `destroy_infra=true`이면 Terraform destroy로 load-test MySQL EC2, 데이터 EBS 볼륨, 보안 그룹과 남아 있는 load-generator EC2를 제거합니다.
2. `restore_stage_dev=true`이면 stage MySQL 컨테이너를 다시 기동하고 준비 상태를 확인한 뒤 활성 슬롯의 앱을 같은 포트에서 `dev` profile로 재기동합니다. 활성 앱 컨테이너가 종료된 상태여도 프로젝트와 이미지가 확인되면 재생성합니다. MySQL 준비 전에 실패하면 정상 앱은 그대로 두고, dev 앱 재생성을 시도한 뒤 실패하면 진입 당시 `dev,loadtest`였던 경우에만 load-test profile 복구를 시도합니다. 롤백 후 MySQL을 중지한 경우 앱 health를 다시 확인합니다.
3. `destroy_infra=true`이면 Terraform state를 읽고 DB 타입 충돌을 stage 전환 전에 검사합니다. dev 앱 health check 성공 후 Terraform destroy로 load-test MySQL EC2, 데이터 EBS 볼륨, 보안 그룹과 남아 있는 load-generator EC2를 제거합니다. DB EC2가 state에 없다는 것이 확인되면 입력한 인스턴스 타입으로 부분 생성 리소스를 정리합니다. state 조회에 실패하면 중단합니다. `restore_stage_dev=false`와 `destroy_infra=true`의 조합은 거부합니다.

Start부터 Stop까지 stage 배포를 겹치지 않게 운영합니다. stage MySQL은 컨테이너를 중지했다가 재기동하며 데이터 볼륨과 컨테이너를 삭제하지 않습니다.

## 참고

Expand All @@ -173,7 +177,7 @@ Stop workflow 동작:
- load-test MySQL EC2 보안 그룹은 prod/stage API EC2 보안 그룹에서 들어오는 MySQL 접근만 허용합니다.
- Load-test DB defaults updated after PR review:
- `load_test_db_ami_id`: `ami-0501a03cd31b53e82` (`solid-connection-db-mysql-8.4.8-arm64-ubuntu24.04-awscli-recovery-tools`).
- `load_test_db_instance_type`: `t4g.medium`, matching the arm64 DB AMI family.
- `load_test_db_instance_type`: Start workflow의 필수 입력값입니다. 선택 시 DB AMI의 CPU 아키텍처와 호환되는 타입을 사용합니다.
- `load_test_db_subnet_id`: when omitted, the prod DB subnet is used so the existing S3 Gateway Endpoint route is available.
- `load_test_db_associate_public_ip`: remains `false`; SSM access is provided through Terraform-managed SSM interface endpoints.
- `load_test_db_instance_profile_name`: defaults to the pre-created dedicated `solid-connection-load-test-db` instance profile. Terraform does not create IAM resources.
Expand Down
9 changes: 7 additions & 2 deletions scripts/load_test/run_k6.sh
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ tf_output() {
terraform -chdir="$TERRAFORM_DIR" output -raw "$1"
}

source scripts/load_test/terraform_db_type.sh

runner_targets=(
-target=aws_security_group.load_generator
-target=aws_instance.load_generator
Expand All @@ -113,7 +115,8 @@ destroy_runner() {
local cleanup_code=0

if [[ "$DESTROY_RUNNER" == "true" ]]; then
terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" "${runner_targets[@]}" || cleanup_code="$?"
terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" \
-var="load_test_db_instance_type=$db_instance_type" "${runner_targets[@]}" || cleanup_code="$?"
fi

if [[ "$exit_code" -ne 0 ]]; then
Expand Down Expand Up @@ -298,7 +301,9 @@ sync_file() {
}

terraform -chdir="$TERRAFORM_DIR" init
terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" "${runner_targets[@]}"
db_instance_type="$(load_test_db_instance_type_from_state)"
terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" \
-var="load_test_db_instance_type=$db_instance_type" "${runner_targets[@]}"

trap destroy_runner EXIT

Expand Down
156 changes: 156 additions & 0 deletions scripts/load_test/stage_app.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
#!/usr/bin/env bash
# Sent to the stage instance through SSM by start.sh and stop.sh.
set -euo pipefail

: "${STAGE_MODE:?}"
: "${STAGE_APP_DIR:?}"
: "${STAGE_COMPOSE_FILE:?}"

if [[ "$STAGE_MODE" != "loadtest" && "$STAGE_MODE" != "dev" ]]; then
echo "Unknown stage mode: $STAGE_MODE" >&2
exit 1
fi

cd "$STAGE_APP_DIR"
upstream_file="${STAGE_UPSTREAM_FILE:-/etc/nginx/conf.d/upstream.conf}"
if [[ ! -r "$upstream_file" || ! -f "$STAGE_COMPOSE_FILE" ]]; then
echo "Stage upstream or Compose file is unavailable" >&2
exit 1
fi

servers=()
while IFS= read -r server; do
servers+=("$server")
done < <(awk '
/^[[:space:]]*upstream[[:space:]]+app_backend[[:space:]]*\{/ { in_backend = 1; next }
in_backend && /^[[:space:]]*\}/ { exit }
in_backend && /^[[:space:]]*server[[:space:]]/ { print }
' "$upstream_file")
server_pattern='^[[:space:]]*server[[:space:]]+127\.0\.0\.1:(8080|9080);[[:space:]]*$'
if (( ${#servers[@]} != 1 )) || [[ ! "${servers[0]}" =~ $server_pattern ]]; then
echo "Expected exactly one active 127.0.0.1:8080 or :9080 upstream" >&2
exit 1
fi

app_port="${BASH_REMATCH[1]}"
if [[ "$app_port" == "8080" ]]; then
slot="blue"
management_port="8081"
else
slot="green"
management_port="9081"
fi

container="solid-connection-dev-$slot"
project="$container"
if ! app_running="$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)"; then
echo "Active slot container is unavailable: $container" >&2
exit 1
fi
if [[ "$app_running" != "true" && ( "$STAGE_MODE" != "dev" || "$app_running" != "false" ) ]]; then
echo "Active slot container is unavailable: $container" >&2
exit 1
fi
if [[ "$(docker inspect -f '{{ index .Config.Labels "com.docker.compose.project" }}' "$container")" != "$project" ]]; then
echo "Active slot container has the wrong Compose project: $container" >&2
exit 1
fi

current_image="$(docker inspect -f '{{.Config.Image}}' "$container")"
container_env="$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$container")"
initial_profile=""
while IFS= read -r env_entry; do
if [[ "$env_entry" == SPRING_PROFILES_ACTIVE=* ]]; then
initial_profile="${env_entry#SPRING_PROFILES_ACTIVE=}"
fi
done <<< "$container_env"
image_pattern='^ghcr\.io/([^/]+)/solid-connection-dev:([^:]+)$'
if [[ ! "$current_image" =~ $image_pattern ]]; then
echo "Cannot infer the active app image from $current_image" >&2
exit 1
fi
owner="${BASH_REMATCH[1]}"
image_tag="${BASH_REMATCH[2]}"

mysql_container="solid-connection-dev-mysql"
mysql_project="$(docker inspect -f '{{ index .Config.Labels "com.docker.compose.project" }}' "$mysql_container")"
if [[ -z "$mysql_project" || "$mysql_project" == "<no value>" ]]; then
echo "Cannot identify the stage MySQL Compose project" >&2
exit 1
fi
if [[ "$STAGE_MODE" == "loadtest" && "$(docker inspect -f '{{.State.Running}}' "$mysql_container")" != "true" ]]; then
echo "Stage MySQL must be running before the load-test switch" >&2
exit 1
fi

override_file="$(mktemp "$STAGE_APP_DIR/.docker-compose.loadtest.XXXXXX")"
trap 'rm -f "$override_file"' EXIT
cat > "$override_file" <<'YAML'
services:
solid-connection-dev:
environment:
SPRING_PROFILES_ACTIVE: dev,loadtest
YAML

recreate_app() {
local profile="$1"
local files=(-f "$STAGE_COMPOSE_FILE")
if [[ "$profile" == "loadtest" ]]; then
files+=(-f "$override_file")
fi
SLOT="$slot" APP_PORT="$app_port" MANAGEMENT_PORT="$management_port" \
OWNER_LOWERCASE="$owner" IMAGE_TAG="$image_tag" \
docker compose -p "$project" "${files[@]}" up -d --no-deps --force-recreate solid-connection-dev
}

wait_for_app() {
local attempt
for (( attempt = 1; attempt <= ${STAGE_HEALTH_ATTEMPTS:-30}; attempt++ )); do
if curl -fsS --max-time 3 "http://127.0.0.1:${management_port}/actuator/health" >/dev/null; then
return 0
fi
sleep "${STAGE_HEALTH_INTERVAL_SECONDS:-5}"
done
return 1
}

wait_for_mysql() {
local attempt
for (( attempt = 1; attempt <= ${STAGE_MYSQL_ATTEMPTS:-30}; attempt++ )); do
if docker exec "$mysql_container" mysqladmin ping -h 127.0.0.1 --silent >/dev/null 2>&1; then
return 0
fi
sleep "${STAGE_HEALTH_INTERVAL_SECONDS:-5}"
done
return 1
}

dev_recreate_started=false
restore_dev() {
docker compose -p "$mysql_project" -f "$STAGE_COMPOSE_FILE" start mysql || return 1
wait_for_mysql || return 1
dev_recreate_started=true
recreate_app dev && wait_for_app
}

if [[ "$STAGE_MODE" == "loadtest" ]]; then
if ! recreate_app loadtest || ! wait_for_app ||
! docker compose -p "$mysql_project" -f "$STAGE_COMPOSE_FILE" stop mysql ||
! wait_for_app; then
echo "Load-test switch failed; restoring dev in the same slot" >&2
restore_dev || echo "Stage dev rollback also failed" >&2
exit 1
fi
else
if ! restore_dev; then
echo "Stage dev restore failed; retaining the load-test DB EC2" >&2
if [[ "$initial_profile" == "dev,loadtest" &&
( "$dev_recreate_started" == "true" || "$app_running" != "true" ) ]]; then
recreate_app loadtest && wait_for_app &&
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
docker compose -p "$mysql_project" -f "$STAGE_COMPOSE_FILE" stop mysql &&
wait_for_app ||
echo "Stage load-test rollback also failed" >&2
fi
exit 1
fi
fi
Loading
Loading