From 71d09c78d4ffd159cfcd83c4c1b919a4c7eef7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Sat, 19 Oct 2024 00:23:49 +0200 Subject: [PATCH 1/6] Accept basebackup --gzip requests (#9456) In #9453, we want to remove the non-gzipped basebackup code in the computes, and always request gzipped basebackups. However, right now the pageserver's page service only accepts basebackup requests in the following formats: * `basebackup `, lsn is determined by the pageserver as the most recent one (`timeline.get_last_record_rlsn()`) * `basebackup ` * `basebackup --gzip` We add a fourth case, `basebackup --gzip` to allow gzipping the request for the latest lsn as well. --- pageserver/src/page_service.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index afb2f92ff8..62b14cb83e 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -1326,22 +1326,22 @@ where .for_command(ComputeCommandKind::Basebackup) .inc(); - let lsn = if let Some(lsn_str) = params.get(2) { - Some( - Lsn::from_str(lsn_str) - .with_context(|| format!("Failed to parse Lsn from {lsn_str}"))?, - ) - } else { - None - }; - - let gzip = match params.get(3) { - Some(&"--gzip") => true, - None => false, - Some(third_param) => { - return Err(QueryError::Other(anyhow::anyhow!( - "Parameter in position 3 unknown {third_param}", - ))) + let (lsn, gzip) = match (params.get(2), params.get(3)) { + (None, _) => (None, false), + (Some(&"--gzip"), _) => (None, true), + (Some(lsn_str), gzip_str_opt) => { + let lsn = Lsn::from_str(lsn_str) + .with_context(|| format!("Failed to parse Lsn from {lsn_str}"))?; + let gzip = match gzip_str_opt { + Some(&"--gzip") => true, + None => false, + Some(third_param) => { + return Err(QueryError::Other(anyhow::anyhow!( + "Parameter in position 3 unknown {third_param}", + ))) + } + }; + (Some(lsn), gzip) } }; From cc25ef73423ea0108986436501481b0154443932 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Sun, 20 Oct 2024 13:42:50 +0100 Subject: [PATCH 2/6] bump pg-session-jwt version (#9455) forgot to bump this before --- proxy/src/serverless/local_conn_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/src/serverless/local_conn_pool.rs b/proxy/src/serverless/local_conn_pool.rs index beb2ad4e8f..e1ad46c751 100644 --- a/proxy/src/serverless/local_conn_pool.rs +++ b/proxy/src/serverless/local_conn_pool.rs @@ -39,7 +39,7 @@ use crate::usage_metrics::{Ids, MetricCounter, USAGE_METRICS}; use crate::{DbName, RoleName}; pub(crate) const EXT_NAME: &str = "pg_session_jwt"; -pub(crate) const EXT_VERSION: &str = "0.1.1"; +pub(crate) const EXT_VERSION: &str = "0.1.2"; pub(crate) const EXT_SCHEMA: &str = "auth"; struct ConnPoolEntry { From ed958da38a0edf7853ee999f43737ac2ff69f920 Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Mon, 21 Oct 2024 10:29:23 +0200 Subject: [PATCH 3/6] proxy: Make tests fail fast when test proxy exited early (#9432) This currently happens when proxy is not compiled with feature `testing`. Also fix an adjacent function. --- test_runner/fixtures/neon_fixtures.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 3cd8019e32..747c2c0d63 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3175,10 +3175,13 @@ class NeonProxy(PgProtocol): # two seconds. Raises subprocess.TimeoutExpired if the proxy does not exit in time. def wait_for_exit(self, timeout=2): if self._popen: - self._popen.wait(timeout=2) + self._popen.wait(timeout=timeout) @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=10) def _wait_until_ready(self): + assert ( + self._popen and self._popen.poll() is None + ), "Proxy exited unexpectedly. Check test log." requests.get(f"http://{self.host}:{self.http_port}/v1/status") def http_query(self, query, args, **kwargs): From 5b37485c99836abb060bed8eb1172870b31504b2 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 21 Oct 2024 09:51:12 +0100 Subject: [PATCH 4/6] Rename dockerfiles from `Dockerfile.` to `.Dockerfile` (#9446) ## Problem Our dockerfiles, for some historical reason, have unconventional names `Dockerfile.`, and some tools (like GitHub UI) fail to highlight the syntax in them. > Some projects may need distinct Dockerfiles for specific purposes. A common convention is to name these `.Dockerfile` From: https://docs.docker.com/build/concepts/dockerfile/#filename ## Summary of changes - Rename `Dockerfile.build-tools` -> `build-tools.Dockerfile` - Rename `compute/Dockerfile.compute-node` -> `compute/compute-node.Dockerfile` --- .github/workflows/_build-and-test-locally.yml | 8 ++++---- .github/workflows/build-build-tools-image.yml | 2 +- .github/workflows/build_and_test.yml | 6 +++--- .github/workflows/check-build-tools-image.yml | 2 +- .github/workflows/trigger-e2e-tests.yml | 2 +- Dockerfile.build-tools => build-tools.Dockerfile | 2 +- compute/README.md | 6 +++--- .../{Dockerfile.compute-node => compute-node.Dockerfile} | 0 docs/docker.md | 6 +++--- 9 files changed, 17 insertions(+), 17 deletions(-) rename Dockerfile.build-tools => build-tools.Dockerfile (99%) rename compute/{Dockerfile.compute-node => compute-node.Dockerfile} (100%) diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index 3aa671fab1..c0f59fbdd5 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -124,28 +124,28 @@ jobs: uses: actions/cache@v4 with: path: pg_install/v14 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }} + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} - name: Cache postgres v15 build id: cache_pg_15 uses: actions/cache@v4 with: path: pg_install/v15 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }} + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} - name: Cache postgres v16 build id: cache_pg_16 uses: actions/cache@v4 with: path: pg_install/v16 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }} + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} - name: Cache postgres v17 build id: cache_pg_17 uses: actions/cache@v4 with: path: pg_install/v17 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }} + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} - name: Build postgres v14 if: steps.cache_pg_14.outputs.cache-hit != 'true' diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml index 0f05276579..10750089b2 100644 --- a/.github/workflows/build-build-tools-image.yml +++ b/.github/workflows/build-build-tools-image.yml @@ -82,7 +82,7 @@ jobs: - uses: docker/build-push-action@v6 with: - file: Dockerfile.build-tools + file: build-tools.Dockerfile context: . provenance: false push: true diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b669eaeb11..1186b9927b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -683,7 +683,7 @@ jobs: provenance: false push: true pull: true - file: compute/Dockerfile.compute-node + file: compute/compute-node.Dockerfile cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-node-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }} tags: | @@ -703,7 +703,7 @@ jobs: provenance: false push: true pull: true - file: compute/Dockerfile.compute-node + file: compute/compute-node.Dockerfile target: neon-pg-ext-test cache-from: type=registry,ref=cache.neon.build/neon-test-extensions-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/neon-test-extensions-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }} @@ -728,7 +728,7 @@ jobs: provenance: false push: true pull: true - file: compute/Dockerfile.compute-node + file: compute/compute-node.Dockerfile cache-from: type=registry,ref=cache.neon.build/neon-test-extensions-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-tools-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }} tags: | diff --git a/.github/workflows/check-build-tools-image.yml b/.github/workflows/check-build-tools-image.yml index 807a9ef3bd..a7a15ad58b 100644 --- a/.github/workflows/check-build-tools-image.yml +++ b/.github/workflows/check-build-tools-image.yml @@ -31,7 +31,7 @@ jobs: id: get-build-tools-tag env: IMAGE_TAG: | - ${{ hashFiles('Dockerfile.build-tools', + ${{ hashFiles('build-tools.Dockerfile', '.github/workflows/check-build-tools-image.yml', '.github/workflows/build-build-tools-image.yml') }} run: | diff --git a/.github/workflows/trigger-e2e-tests.yml b/.github/workflows/trigger-e2e-tests.yml index 5c5423e252..1e7264c55a 100644 --- a/.github/workflows/trigger-e2e-tests.yml +++ b/.github/workflows/trigger-e2e-tests.yml @@ -112,7 +112,7 @@ jobs: # This isn't exhaustive, just the paths that are most directly compute-related. # For example, compute_ctl also depends on libs/utils, but we don't trigger # an e2e run on that. - vendor/*|pgxn/*|compute_tools/*|libs/vm_monitor/*|compute/Dockerfile.compute-node) + vendor/*|pgxn/*|compute_tools/*|libs/vm_monitor/*|compute/compute-node.Dockerfile) platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique') ;; *) diff --git a/Dockerfile.build-tools b/build-tools.Dockerfile similarity index 99% rename from Dockerfile.build-tools rename to build-tools.Dockerfile index f05c60661c..818cc1b6db 100644 --- a/Dockerfile.build-tools +++ b/build-tools.Dockerfile @@ -142,7 +142,7 @@ RUN wget -O /tmp/openssl-${OPENSSL_VERSION}.tar.gz https://www.openssl.org/sourc # Use the same version of libicu as the compute nodes so that # clusters created using inidb on pageserver can be used by computes. # -# TODO: at this time, Dockerfile.compute-node uses the debian bullseye libicu +# TODO: at this time, compute-node.Dockerfile uses the debian bullseye libicu # package, which is 67.1. We're duplicating that knowledge here, and also, technically, # Debian has a few patches on top of 67.1 that we're not adding here. ENV ICU_VERSION=67.1 diff --git a/compute/README.md b/compute/README.md index bb1e42ab53..61e0eee4be 100644 --- a/compute/README.md +++ b/compute/README.md @@ -1,7 +1,7 @@ This directory contains files that are needed to build the compute images, or included in the compute images. -Dockerfile.compute-node +compute-node.Dockerfile To build the compute image vm-image-spec.yaml @@ -14,8 +14,8 @@ etc/ patches/ Some extensions need to be patched to work with Neon. This directory contains such patches. They are applied to the extension - sources in Dockerfile.compute-node + sources in compute-node.Dockerfile In addition to these, postgres itself, the neon postgres extension, and compute_ctl are built and copied into the compute image by -Dockerfile.compute-node. +compute-node.Dockerfile. diff --git a/compute/Dockerfile.compute-node b/compute/compute-node.Dockerfile similarity index 100% rename from compute/Dockerfile.compute-node rename to compute/compute-node.Dockerfile diff --git a/docs/docker.md b/docs/docker.md index d16311c27b..0914a00082 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -5,7 +5,7 @@ Currently we build two main images: - [neondatabase/neon](https://hub.docker.com/repository/docker/neondatabase/neon) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile). -- [neondatabase/compute-node-v16](https://hub.docker.com/repository/docker/neondatabase/compute-node-v16) — compute node image with pre-built Postgres binaries from [neondatabase/postgres](https://github.com/neondatabase/postgres). Similar images exist for v15 and v14. Built from [/compute-node/Dockerfile](/compute/Dockerfile.compute-node). +- [neondatabase/compute-node-v16](https://hub.docker.com/repository/docker/neondatabase/compute-node-v16) — compute node image with pre-built Postgres binaries from [neondatabase/postgres](https://github.com/neondatabase/postgres). Similar images exist for v15 and v14. Built from [/compute-node/Dockerfile](/compute/compute-node.Dockerfile). And additional intermediate image: @@ -56,7 +56,7 @@ CREATE TABLE postgres=# insert into t values(1, 1); INSERT 0 1 postgres=# select * from t; - key | value + key | value -----+------- 1 | 1 (1 row) @@ -84,4 +84,4 @@ Access http://localhost:9001 and sign in. - Username: `minio` - Password: `password` -You can see durable pages and WAL data in `neon` bucket. \ No newline at end of file +You can see durable pages and WAL data in `neon` bucket. From 163beaf9ad8521ec28d451d1ea884039efcb8897 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 21 Oct 2024 12:14:19 +0100 Subject: [PATCH 5/6] CI: use build-tools on Debian 12 whenever we use Neon artifact (#9463) ## Problem ``` + /tmp/neon/pg_install/v16/bin/psql '***' -c 'SELECT version()' /tmp/neon/pg_install/v16/bin/psql: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.33' not found (required by /tmp/neon/pg_install/v16/bin/psql) /tmp/neon/pg_install/v16/bin/psql: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by /tmp/neon/pg_install/v16/bin/psql) /tmp/neon/pg_install/v16/bin/psql: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.32' not found (required by /tmp/neon/pg_install/v16/lib/libpq.so.5) /tmp/neon/pg_install/v16/bin/psql: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.33' not found (required by /tmp/neon/pg_install/v16/lib/libpq.so.5) /tmp/neon/pg_install/v16/bin/psql: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by /tmp/neon/pg_install/v16/lib/libpq.so.5) ``` ## Summary of changes - Use `build-tools:pinned-bookworm` whenever we download Neon artefact --- .../workflows/_benchmarking_preparation.yml | 2 +- .github/workflows/benchmarking.yml | 18 +++++++++--------- .github/workflows/cloud-regress.yml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/_benchmarking_preparation.yml b/.github/workflows/_benchmarking_preparation.yml index d60f97320b..5cdc16f248 100644 --- a/.github/workflows/_benchmarking_preparation.yml +++ b/.github/workflows/_benchmarking_preparation.yml @@ -27,7 +27,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: neondatabase/build-tools:pinned + image: neondatabase/build-tools:pinned-bookworm credentials: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 32806b89ab..5ccfe48684 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -83,7 +83,7 @@ jobs: runs-on: ${{ matrix.RUNNER }} container: - image: neondatabase/build-tools:pinned + image: neondatabase/build-tools:pinned-bookworm credentials: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} @@ -178,7 +178,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: neondatabase/build-tools:pinned + image: neondatabase/build-tools:pinned-bookworm credentials: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} @@ -280,7 +280,7 @@ jobs: region_id_default=${{ env.DEFAULT_REGION_ID }} runner_default='["self-hosted", "us-east-2", "x64"]' runner_azure='["self-hosted", "eastus2", "x64"]' - image_default="neondatabase/build-tools:pinned" + image_default="neondatabase/build-tools:pinned-bookworm" matrix='{ "pg_version" : [ 16 @@ -299,9 +299,9 @@ jobs: "include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" }, { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" }, { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }, - { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned" }, - { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned" }, - { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" }, { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }] }' @@ -665,7 +665,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: neondatabase/build-tools:pinned + image: neondatabase/build-tools:pinned-bookworm credentials: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} @@ -772,7 +772,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: neondatabase/build-tools:pinned + image: neondatabase/build-tools:pinned-bookworm credentials: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} @@ -877,7 +877,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: neondatabase/build-tools:pinned + image: neondatabase/build-tools:pinned-bookworm credentials: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/cloud-regress.yml b/.github/workflows/cloud-regress.yml index ecafe183f8..19ebf457b8 100644 --- a/.github/workflows/cloud-regress.yml +++ b/.github/workflows/cloud-regress.yml @@ -31,7 +31,7 @@ jobs: runs-on: us-east-2 container: - image: neondatabase/build-tools:pinned + image: neondatabase/build-tools:pinned-bookworm options: --init steps: From ababa50cce5e05df4d3d9fcf617a1b2625ed3b4a Mon Sep 17 00:00:00 2001 From: Ivan Efremov Date: Mon, 21 Oct 2024 16:20:39 +0300 Subject: [PATCH 6/6] Use '-f' for make clean in Makefile compute (#9464) Use '-f' instead of '--force' because it is impossible to clean the targets on MacOS --- compute/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/Makefile b/compute/Makefile index e2896fe390..08e3c7a68b 100644 --- a/compute/Makefile +++ b/compute/Makefile @@ -34,7 +34,7 @@ sql_exporter_autoscaling.yml: $(jsonnet_files) .PHONY: clean clean: - rm --force \ + rm -f \ etc/neon_collector.yml \ etc/neon_collector_autoscaling.yml \ etc/sql_exporter.yml \