Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 14 additions & 16 deletions .buildkite/pipeline.yaml
Original file line number Diff line number Diff line change
@@ -1,37 +1,37 @@
$schema: "http://json.schemastore.org/buildkite"
$schema: 'http://json.schemastore.org/buildkite'
env:
# GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z"
VAGRANT_RUN_ENV: "CI"
IMAGE: 'e2e-db8a40720-2024-08-26t08-44-10z'
VAGRANT_RUN_ENV: 'CI'
steps:
- label: ":docker:"
- label: ':docker:'
command: .buildkite/validate-docker-compose.sh
agents: { queue: "vagrant" }
agents: { queue: 'vagrant' }

- label: ":lipstick:"
- label: ':lipstick:'
command: .buildkite/prettier-check.sh
agents: { queue: "vagrant" }
agents: { queue: 'vagrant' }

- label: ":git: :sleuth_or_spy:"
- label: ':git: :sleuth_or_spy:'
command: .buildkite/verify-release/verify-release.sh
agents: { queue: standard }

- label: ":rice: docker-compose-test"
- label: ':rice: docker-compose-test'
command: .buildkite/vagrant-run.sh docker-test
artifact_paths: ./*.log
agents: { queue: "vagrant" }
agents: { queue: 'vagrant' }

# Please keep in mind that the release manifest uses specific branch names when creating releases.
# Therefore, if you update them, or if you decide to change how we detect what kind of build we're dealing
# with, please update this file as well.
- label: "(internal) Release: test"
- label: '(internal) Release: test'
if: build.branch =~ /^internal\/release-.*/
plugins:
- ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
command: |
sg release run test --workdir=. --config-from-commit

- label: "(promote) Release: test"
- label: '(promote) Release: test'
if: build.branch =~ /^promote\/release-.*/
plugins:
- ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
Expand All @@ -40,18 +40,16 @@ steps:

- wait

- label: "(internal) Release: finalize"
- label: '(internal) Release: finalize'
if: build.branch =~ /^internal\/release-.*/
plugins:
- ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
command: |
sg release run internal finalize --workdir=. --config-from-commit


- label: "(promote) Release: finalize"
- label: '(promote) Release: finalize'
if: build.branch =~ /^promote\/release-.*/
plugins:
- ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
command: |
sg release run promote-to-public finalize --workdir=. --config-from-commit

2 changes: 0 additions & 2 deletions .prettierignore

This file was deleted.

4 changes: 2 additions & 2 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode",
"[shellscript]": {
"editor.defaultFormatter": "foxundermoon.shell-format",
},
"editor.defaultFormatter": "foxundermoon.shell-format"
}
}
44 changes: 22 additions & 22 deletions batch-change/release.yaml
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
meta:
productName: "deploy-sourcegraph-docker"
productName: 'deploy-sourcegraph-docker'
owners:
- "@sourcegraph/release"
repository: "github.com/sourcegraph/deploy-sourcegraph-docker"
- '@sourcegraph/release'
repository: 'github.com/sourcegraph/deploy-sourcegraph-docker'

inputs:
- releaseId: server

requirements:
# We use wget here, because curl --fail-with-body was introduced in a version ulterior to what we can have on the CI agents.
- name: "wget"
cmd: "wget --help"
- name: "GitHub cli exists"
cmd: "which gh"
fixInstructions: "install GitHub cli"
- name: "Docker username"
env: "DOCKER_USERNAME"
- name: "Docker password"
env: "DOCKER_PASSWORD"
- name: "GH auth status"
cmd: "gh auth status"
fixInstructions: "gh auth login"
- name: 'wget'
cmd: 'wget --help'
- name: 'GitHub cli exists'
cmd: 'which gh'
fixInstructions: 'install GitHub cli'
- name: 'Docker username'
env: 'DOCKER_USERNAME'
- name: 'Docker password'
env: 'DOCKER_PASSWORD'
- name: 'GH auth status'
cmd: 'gh auth status'
fixInstructions: 'gh auth login'

internal:
# Please keep in mind that the CI pipeline uses the branch names defined below when creating releases.
Expand Down Expand Up @@ -48,7 +48,7 @@ internal:
sg ops update-images --registry ${registry} --kind compose --pin-tag {{inputs.server.tag}} --docker-username $DOCKER_USERNAME --docker-password $DOCKER_PASSWORD ../docker-compose/
finalize:
steps:
- name: "git:finalize"
- name: 'git:finalize'
cmd: |
set -eu

Expand All @@ -64,7 +64,7 @@ internal:
cat << EOF | buildkite-agent annotate --style info
Internal release is ready for promotion under the branch [\`$branch\`](https://github.com/sourcegraph/deploy-sourcegraph-docker/tree/$branch).
EOF

- name: 'Register on release registry'
cmd: |
echo "Registering internal deploy-sourcegraph-docker {{version}} release on release registry"
Expand All @@ -89,14 +89,14 @@ internal:

test:
steps:
- name: "Placeholder"
- name: 'Placeholder'
cmd: |
echo "Test"

promoteToPublic:
create:
steps:
- name: "git"
- name: 'git'
cmd: |
set -eu
branch="internal/release-{{version}}"
Expand Down Expand Up @@ -162,7 +162,7 @@ promoteToPublic:
Promoted release is **publicly available** through a git tag at [\`{{version}}\`](https://github.com/sourcegraph/deploy-sourcegraph-docker/tree/{{version}}).
EOF

- name: "update main with latest version"
- name: 'update main with latest version'
cmd: |
set -eu
branch="promote/release-{{version}}-update-main"
Expand All @@ -178,14 +178,14 @@ promoteToPublic:
registry=index.docker.io/sourcegraph
sg ops update-images --registry ${registry} --kind compose --pin-tag {{inputs.server.tag}} --docker-username $DOCKER_USERNAME --docker-password $DOCKER_PASSWORD ../docker-compose/

- name: "git:branch"
- name: 'git:branch'
cmd: |
set -eu
branch="promote/release-{{version}}-update-main"
git commit -am 'prep update main: {{version}}' -m 'update main with latest release'
git push origin "${branch}"

- name: "github:pr"
- name: 'github:pr'
cmd: |
set -eu
internal_branch="promote/release-{{version}}-update-main"
Expand Down
2 changes: 1 addition & 1 deletion docker-compose/db-only-migrate.docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# This file contains the pgsql service definition (the primary Sourcegraph database deployment),
# This file contains the pgsql service definition (the primary Sourcegraph database deployment),
# as well as the codeintel-db service definition (a separate postgres deployment). Both run postgres.
# This file is used for database-only operations such as backup and restore.
# See https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/operations
Expand Down
2 changes: 1 addition & 1 deletion docker-compose/dev/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ This folder includes a helper script, [run.sh](./run.sh), that will run the `doc
Example usage:

```sh
./run.sh up -d
./run.sh up -d
```

The above will deploy Sourcegraph on [http://localhost:8080](http://localhost:8080) with the indicated overlays.
Expand Down
28 changes: 14 additions & 14 deletions docker-compose/examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,34 @@ These best practices reduce the operational cost of keeping your Sourcegraph ins

- Your docker-compose.override.yaml file should be stored, maintained, and fetched from a Git repo / code version control system, with meaningful commit messages, so that future you, and future admins, can understand why changes needed to be made

- Bonus points if your override file is stored outside of your clone of this deploy-sourcegraph-docker repo, so that this deploy-sourcegraph-docker repo can be deleted / recreated / `git reset --hard` as needed, without losing your customizations
- Bonus points if your override file is stored outside of your clone of this deploy-sourcegraph-docker repo, so that this deploy-sourcegraph-docker repo can be deleted / recreated / `git reset --hard` as needed, without losing your customizations

- The examples in this directory use the current compose syntax, used by `docker compose`, the docker-compose-plugin. Some parts of this syntax may not be valid for the old `docker-compose` standalone binary. It is highly recommended to switch to using the new compose plugin. See Docker docs:

- Install Compose plugin
- https://docs.docker.com/compose/install
- Install Compose plugin
- https://docs.docker.com/compose/install

- Merging multiple compose files
- https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/
- https://docs.docker.com/reference/compose-file/merge/#reset-value
- Merging multiple compose files
- https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/
- https://docs.docker.com/reference/compose-file/merge/#reset-value

- Using multiple compose / override files

- Old method:
- Old method:

- Store your docker-compose.override.yaml file inside this deploy-sourcegraph-docker repo, under the /docker-compose directory
- Store your docker-compose.override.yaml file inside this deploy-sourcegraph-docker repo, under the /docker-compose directory

- `docker compose -f docker-compose.yaml -f executors/executor.docker-compose.yaml -f docker-compose.override.yaml up -d --remove-orphans`
- `docker compose -f docker-compose.yaml -f executors/executor.docker-compose.yaml -f docker-compose.override.yaml up -d --remove-orphans`

- New method:
- New method:

- Store your docker-compose.override.yaml file in the parent directory / beside this deploy-sourcegraph-docker repo
- Store your docker-compose.override.yaml file in the parent directory / beside this deploy-sourcegraph-docker repo

- Create an `.env` file, containing the below
- Create an `.env` file, containing the below

- `docker compose up -d --remove-orphans`
- `docker compose up -d --remove-orphans`

- Note that any file paths (ex. mounting TLS CAs) are relative to the first compose file, so the mount path will likely start with `../../cert-chain.pem` if `cert-chain.pem` is in the parent directory / beside this deploy-sourcegraph-docker repo
- Note that any file paths (ex. mounting TLS CAs) are relative to the first compose file, so the mount path will likely start with `../../cert-chain.pem` if `cert-chain.pem` is in the parent directory / beside this deploy-sourcegraph-docker repo

```env
SOURCEGRAPH=./deploy-sourcegraph-docker/docker-compose/docker-compose.yaml
Expand Down
8 changes: 2 additions & 6 deletions docker-compose/examples/aws-alb/docker-compose.override.yaml
Original file line number Diff line number Diff line change
@@ -1,18 +1,14 @@
# Docker Compose override snippet for customers using AWS ALBs to terminate TLS,
# and do not need TLS between the ALB and their Sourcegraph intance


## Overrides for services defined in the base docker-compose.yaml files
services:

sourcegraph-frontend-0:

# Add port 80, for ALB target group, in place of Caddy
ports:
- "0.0.0.0:80:3080"

- '0.0.0.0:80:3080'

## Disable services
## Disable services

# Disable Caddy, because we're using an ALB for TLS termination
caddy: !reset {}
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
# Docker Compose override file for customers using AWS IAM authentication to databases on RDS
# See https://sourcegraph.com/docs/self-hosted/external-services/postgres#usage-with-aws-rds-iam-auth



## Reusable blocks of environment variables

# For all containers, which connect to any of the databases
# Using AWS RDS IAM authentication, not a password, so instruct the psql library to refresh the token
x-database-env-var-rds-iam-auth: &database-env-var-rds-iam-auth

# Using AWS RDS IAM authentication, not a password, so instruct the psql library to refresh the token
PG_CONNECTION_UPDATER: EC2_ROLE_CREDENTIALS

# For both migrator and frontend
x-database-env-vars: &database-env-vars

<<: *database-env-var-rds-iam-auth

# NOTE: Must clear / override the PGDATASOURCE env vars, otherwise frontends don't start up
Expand All @@ -22,39 +18,33 @@ x-database-env-vars: &database-env-vars
PGSSLMODE: require
PGDATABASE: sg
PGUSER: sg
PGPASSWORD: ""
PGDATASOURCE: ""
PGPASSWORD: ''
PGDATASOURCE: ''

CODEINTEL_PGHOST: example-database-codeintel.cluster-example.us-west-2.rds.amazonaws.com
CODEINTEL_PGPORT: 5432
CODEINTEL_PGSSLMODE: require
CODEINTEL_PGDATABASE: codeintel
CODEINTEL_PGUSER: sg
CODEINTEL_PGPASSWORD: ""
CODEINTEL_PGDATASOURCE: ""
CODEINTEL_PGPASSWORD: ''
CODEINTEL_PGDATASOURCE: ''

CODEINSIGHTS_PGHOST: example-database-codeinsights.cluster-example.us-west-2.rds.amazonaws.com
CODEINSIGHTS_PGPORT: 5432
CODEINSIGHTS_PGSSLMODE: require
CODEINSIGHTS_PGDATABASE: codeinsights
CODEINSIGHTS_PGUSER: sg
CODEINSIGHTS_PGPASSWORD: ""
CODEINSIGHTS_PGDATASOURCE: ""


CODEINSIGHTS_PGPASSWORD: ''
CODEINSIGHTS_PGDATASOURCE: ''

## Overrides for services defined in the base docker-compose.yaml files
services:


gitserver-0:
# Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
environment:
<<: *database-env-var-rds-iam-auth


migrator:

# !reset drops the depends_on config entirely, so that migrator will start up without the database containers
depends_on: !reset []

Expand Down Expand Up @@ -85,25 +75,21 @@ services:
# Do not skip the drift check, or use other args, without instruction from Sourcegraph Customer Support Engineering
# If migrator fails to start up due to drift detected, or other issues, please open a support ticket by emailing support@sourcegraph.com


precise-code-intel-worker:
# Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
environment:
<<: *database-env-var-rds-iam-auth


searcher-0:
# Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
environment:
<<: *database-env-var-rds-iam-auth


sourcegraph-frontend-0:
# Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
environment:
<<: *database-env-vars


sourcegraph-frontend-internal:
# !override drops all depends_on from earlier compose files, then defines new ones,
# to ensure it doesn't wait for the database containers, but also doesn't skip the dependency on migrator
Expand All @@ -114,13 +100,11 @@ services:
environment:
<<: *database-env-vars


syntactic-code-intel-worker:
# Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
environment:
<<: *database-env-var-rds-iam-auth


worker:
# Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
environment:
Expand All @@ -130,8 +114,7 @@ services:
# TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_ORGANIZATIONS: true
# TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_USERS: true


## Disable services
## Disable services

# Drop built-in database services and volumes from the base compose file
pgsql: !reset {}
Expand Down
Loading
Loading