rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-25 00:50:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deploy storage into new account and migrate to management API v2 (#2619)

Browse Source 
Deploy storage into new account
Migrate safekeeper and pageserver initialisation to management api v2
This commit is contained in:
Sergey Melnikov
2022-10-18 15:52:15 +03:00
committed by GitHub
parent 59bc7e67e0
commit 546e9bdbec
11 changed files with 116 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/ansible/deploy.yaml vendored
View File

@@ -1,7 +1,7 @@
- name: Upload Neon binaries
hosts: storage
gather_facts: False
remote_user: admin
remote_user: "{{ remote_user }}"
tasks:
@@ -36,7 +36,7 @@
- name: Deploy pageserver
hosts: pageservers
gather_facts: False
remote_user: admin
remote_user: "{{ remote_user }}"
tasks:
@@ -124,7 +124,7 @@
- name: Deploy safekeeper
hosts: safekeepers
gather_facts: False
remote_user: admin
remote_user: "{{ remote_user }}"
tasks:

13
.github/ansible/neon-stress.hosts.yaml vendored
View File

@@ -12,19 +12,20 @@ storage:
bucket_name: "{{ bucket_name }}"
bucket_region: "{{ bucket_region }}"
prefix_in_bucket: "{{ inventory_hostname }}"
hostname_suffix: ".local"
remote_user: admin
children:
pageservers:
hosts:
neon-stress-ps-1:
console_region_id: 1
console_region_id: aws-eu-west-1
neon-stress-ps-2:
console_region_id: 1
console_region_id: aws-eu-west-1
safekeepers:
hosts:
neon-stress-sk-1:
console_region_id: 1
console_region_id: aws-eu-west-1
neon-stress-sk-2:
console_region_id: 1
console_region_id: aws-eu-west-1
neon-stress-sk-3:
console_region_id: 1
console_region_id: aws-eu-west-1

12
.github/ansible/production.hosts.yaml vendored
View File

@@ -12,20 +12,22 @@ storage:
bucket_name: "{{ bucket_name }}"
bucket_region: "{{ bucket_region }}"
prefix_in_bucket: "{{ inventory_hostname }}"
hostname_suffix: ".local"
remote_user: admin
children:
pageservers:
hosts:
zenith-1-ps-2:
console_region_id: 1
console_region_id: aws-us-west-2
zenith-1-ps-3:
console_region_id: 1
console_region_id: aws-us-west-2
safekeepers:
hosts:
zenith-1-sk-1:
console_region_id: 1
console_region_id: aws-us-west-2
zenith-1-sk-2:
console_region_id: 1
console_region_id: aws-us-west-2
zenith-1-sk-3:
console_region_id: 1
console_region_id: aws-us-west-2

9
.github/ansible/scripts/init_pageserver.sh vendored
View File

@@ -12,18 +12,19 @@ cat <<EOF | tee /tmp/payload
"version": 1,
"host": "${HOST}",
"port": 6400,
"region_id": {{ console_region_id }},
"region_id": "{{ console_region_id }}",
"instance_id": "${INSTANCE_ID}",
"http_host": "${HOST}",
"http_port": 9898
"http_port": 9898,
"active": false
}
EOF
# check if pageserver already registered or not
if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/pageservers/${INSTANCE_ID} -o /dev/null; then
if ! curl -sf -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/${INSTANCE_ID} -o /dev/null; then
# not registered, so register it now
ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/pageservers -d@/tmp/payload | jq -r '.ID')
ID=$(curl -sf -X POST -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers -d@/tmp/payload | jq -r '.id')
# init pageserver
sudo -u pageserver /usr/local/bin/pageserver -c "id=${ID}" -c "pg_distrib_dir='/usr/local'" --init -D /storage/pageserver/data

10
.github/ansible/scripts/init_safekeeper.sh vendored
View File

@@ -14,18 +14,18 @@ cat <<EOF | tee /tmp/payload
"host": "${HOST}",
"port": 6500,
"http_port": 7676,
"region_id": {{ console_region_id }},
"region_id": "{{ console_region_id }}",
"instance_id": "${INSTANCE_ID}",
"availability_zone_id": "${AZ_ID}"
"availability_zone_id": "${AZ_ID}",
"active": false
}
EOF
# check if safekeeper already registered or not
if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/safekeepers/${INSTANCE_ID} -o /dev/null; then
if ! curl -sf -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/${INSTANCE_ID} -o /dev/null; then
# not registered, so register it now
ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers -d@/tmp/payload | jq -r '.ID')
ID=$(curl -sf -X POST -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers -d@/tmp/payload | jq -r '.id')
# init safekeeper
sudo -u safekeeper /usr/local/bin/safekeeper --id ${ID} --init -D /storage/safekeeper/data
fi

3
.github/ansible/ssm_config vendored Normal file
View File

@@ -0,0 +1,3 @@
ansible_connection: aws_ssm
ansible_aws_ssm_bucket_name: neon-dev-bucket
ansible_python_interpreter: /usr/bin/python3

22
.github/ansible/staging.hosts.yaml vendored
View File

@@ -11,30 +11,24 @@ storage:
bucket_name: "{{ bucket_name }}"
bucket_region: "{{ bucket_region }}"
prefix_in_bucket: "{{ inventory_hostname }}"
hostname_suffix: ".local"
remote_user: admin
children:
pageservers:
hosts:
zenith-us-stage-ps-2:
console_region_id: 27
console_region_id: aws-us-east-1
zenith-us-stage-ps-3:
console_region_id: 27
console_region_id: aws-us-east-1
zenith-us-stage-ps-4:
console_region_id: 27
zenith-us-stage-test-ps-1:
console_region_id: 28
console_region_id: aws-us-east-1
safekeepers:
hosts:
zenith-us-stage-sk-4:
console_region_id: 27
console_region_id: aws-us-east-1
zenith-us-stage-sk-5:
console_region_id: 27
console_region_id: aws-us-east-1
zenith-us-stage-sk-6:
console_region_id: 27
zenith-us-stage-test-sk-1:
console_region_id: 28
zenith-us-stage-test-sk-2:
console_region_id: 28
zenith-us-stage-test-sk-3:
console_region_id: 28
console_region_id: aws-us-east-1

32
.github/ansible/staging.us-east-2.hosts.yaml vendored Normal file
View File

@@ -0,0 +1,32 @@
storage:
vars:
bucket_name: neon-staging-storage-us-east-2
bucket_region: us-east-2
console_mgmt_base_url: http://console-staging.local
env_name: us-stage
etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
pageserver_config_stub:
pg_distrib_dir: /usr/local
remote_storage:
bucket_name: "{{ bucket_name }}"
bucket_region: "{{ bucket_region }}"
prefix_in_bucket: "pageserver/v1"
hostname_suffix: ""
remote_user: ssm-user
ansible_aws_ssm_region: us-east-2
console_region_id: aws-us-east-2
children:
pageservers:
hosts:
pageserver-0.us-east-2.aws.neon.build:
ansible_host: i-0c3e70929edb5d691
safekeepers:
hosts:
safekeeper-0.us-east-2.aws.neon.build:
ansible_host: i-027662bd552bf5db0
safekeeper-1.us-east-2.aws.neon.build:
ansible_host: i-0171efc3604a7b907
safekeeper-2.us-east-2.aws.neon.build:
ansible_host: i-0de0b03a51676a6ce

2
.github/ansible/systemd/pageserver.service vendored
View File

@@ -1,5 +1,5 @@
[Unit]
Description=Zenith pageserver
Description=Neon pageserver
After=network.target auditd.service
[Service]

4
.github/ansible/systemd/safekeeper.service vendored
View File

@@ -1,12 +1,12 @@
[Unit]
Description=Zenith safekeeper
Description=Neon safekeeper
After=network.target auditd.service
[Service]
Type=simple
User=safekeeper
Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib
ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ env_name }}/wal"}'
ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ env_name }}/wal"}'
ExecReload=/bin/kill -HUP $MAINPID
KillMode=mixed
KillSignal=SIGINT