diff --git a/.dockerignore b/.dockerignore index 2c78951923..9f8a22d598 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,7 @@ !pgxn/ !proxy/ !safekeeper/ -!vendor/postgres/ +!vendor/postgres-v14/ +!vendor/postgres-v15/ !workspace_hack/ !neon_local/ diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 01ddced313..f04f5d11b8 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -88,7 +88,7 @@ runs: # PLATFORM will be embedded in the perf test report # and it is needed to distinguish different environments export PLATFORM=${PLATFORM:-github-actions-selfhosted} - export POSTGRES_DISTRIB_DIR=${POSTGRES_DISTRIB_DIR:-/tmp/neon/pg_install} + export POSTGRES_DISTRIB_DIR=${POSTGRES_DISTRIB_DIR:-/tmp/neon/pg_install/v14} if [ "${BUILD_TYPE}" = "remote" ]; then export REMOTE_ENV=1 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6fae36c6e4..6eddbc3335 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -78,8 +78,8 @@ jobs: fetch-depth: 1 - name: Set pg revision for caching - id: pg_ver - run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres) + id: pg_v14_rev + run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres-v14) shell: bash -euxo pipefail {0} # Set some environment variables used by all the steps. @@ -124,12 +124,12 @@ jobs: v7-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }} v7-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}- - - name: Cache postgres build + - name: Cache postgres v14 build id: cache_pg uses: actions/cache@v3 with: - path: tmp_install/ - key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_ver.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + path: pg_install/v14 + key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Build postgres if: steps.cache_pg.outputs.cache-hit != 'true' @@ -192,7 +192,7 @@ jobs: shell: bash -euxo pipefail {0} - name: Install postgres binaries - run: cp -a tmp_install /tmp/neon/pg_install + run: cp -a pg_install /tmp/neon/pg_install shell: bash -euxo pipefail {0} - name: Upload Neon artifact @@ -447,7 +447,6 @@ jobs: compute-node-image: runs-on: dev container: gcr.io/kaniko-project/executor:v1.9.0-debug - steps: - name: Checkout uses: actions/checkout@v1 # v3 won't work with kaniko @@ -458,18 +457,57 @@ jobs: - name: Configure ECR login run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - - name: Kaniko build compute node with extensions - run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --dockerfile Dockerfile.compute-node --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID + # compute-node uses postgres 14, which is default now + # cloud repo depends on this image name, thus duplicating it + # remove compute-node when cloud repo is updated + - name: Kaniko build compute node with extensions v14 (compatibility) + run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID + + compute-node-image-v14: + runs-on: dev + container: gcr.io/kaniko-project/executor:v1.9.0-debug + steps: + - name: Checkout + uses: actions/checkout@v1 # v3 won't work with kaniko + with: + submodules: true + fetch-depth: 0 + + - name: Configure ECR login + run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json + + - name: Kaniko build compute node with extensions v14 + run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:$GITHUB_RUN_ID + + + compute-node-image-v15: + runs-on: dev + container: gcr.io/kaniko-project/executor:v1.9.0-debug + steps: + - name: Checkout + uses: actions/checkout@v1 # v3 won't work with kaniko + with: + submodules: true + fetch-depth: 0 + + - name: Configure ECR login + run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json + + - name: Kaniko build compute node with extensions v15 + run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --dockerfile Dockerfile.compute-node-v15 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:$GITHUB_RUN_ID promote-images: runs-on: dev - needs: [ neon-image, compute-node-image, compute-tools-image ] + needs: [ neon-image, compute-node-image, compute-node-image-v14, compute-tools-image ] if: github.event_name != 'workflow_dispatch' container: amazon/aws-cli strategy: fail-fast: false matrix: - name: [ neon, compute-node, compute-tools ] + # compute-node uses postgres 14, which is default now + # cloud repo depends on this image name, thus duplicating it + # remove compute-node when cloud repo is updated + name: [ neon, compute-node, compute-node-v14, compute-tools ] steps: - name: Promote image to latest @@ -501,6 +539,9 @@ jobs: - name: Pull compute node image from ECR run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:latest compute-node + - name: Pull compute node v14 image from ECR + run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest compute-node-v14 + - name: Pull rust image from ECR run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned rust @@ -519,6 +560,9 @@ jobs: - name: Push compute node image to Docker Hub run: crane push compute-node neondatabase/compute-node:${{needs.tag.outputs.build-tag}} + - name: Push compute node v14 image to Docker Hub + run: crane push compute-node-v14 neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}} + - name: Push rust image to Docker Hub run: crane push rust neondatabase/rust:pinned @@ -530,6 +574,7 @@ jobs: crane tag neondatabase/neon:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/compute-tools:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/compute-node:${{needs.tag.outputs.build-tag}} latest + crane tag neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}} latest calculate-deploy-targets: runs-on: [ self-hosted, Linux, k8s-runner ] diff --git a/.github/workflows/codestyle.yml b/.github/workflows/codestyle.yml index b64ea8a01f..a5e31d49ee 100644 --- a/.github/workflows/codestyle.yml +++ b/.github/workflows/codestyle.yml @@ -27,8 +27,10 @@ jobs: # Rust toolchains (e.g. nightly or 1.37.0), add them here. rust_toolchain: [1.58] os: [ubuntu-latest, macos-latest] + # To support several Postgres versions, add them here. + postgres_version: [v14, v15] timeout-minutes: 60 - name: run regression test suite + name: check codestyle rust and postgres runs-on: ${{ matrix.os }} steps: @@ -61,14 +63,14 @@ jobs: - name: Set pg revision for caching id: pg_ver - run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres) + run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres-${{matrix.postgres_version}}) - - name: Cache postgres build + - name: Cache postgres ${{matrix.postgres_version}} build id: cache_pg uses: actions/cache@v3 with: path: | - tmp_install/ + pg_install/${{matrix.postgres_version}} key: ${{ runner.os }}-pg-${{ steps.pg_ver.outputs.pg_rev }} - name: Set extra env for macOS @@ -90,10 +92,10 @@ jobs: if: failure() continue-on-error: true run: | - echo '' && echo '=== config.log ===' && echo '' - cat tmp_install/build/config.log - echo '' && echo '=== configure.log ===' && echo '' - cat tmp_install/build/configure.log + echo '' && echo '=== Postgres ${{matrix.postgres_version}} config.log ===' && echo '' + cat pg_install/build/${{matrix.postgres_version}}/config.log + echo '' && echo '=== Postgres ${{matrix.postgres_version}} configure.log ===' && echo '' + cat pg_install/build/${{matrix.postgres_version}}/configure.log - name: Cache cargo deps id: cache_cargo diff --git a/.github/workflows/pg_clients.yml b/.github/workflows/pg_clients.yml index 95052619cd..bf14865db2 100644 --- a/.github/workflows/pg_clients.yml +++ b/.github/workflows/pg_clients.yml @@ -52,7 +52,7 @@ jobs: REMOTE_ENV: 1 BENCHMARK_CONNSTR: "${{ secrets.BENCHMARK_STAGING_CONNSTR }}" - POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install/v14 shell: bash -euxo pipefail {0} run: | # Test framework expects we have psql binary; diff --git a/.gitignore b/.gitignore index ed718c8c79..618ff2c5b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ +/pg_install /target /tmp_check -/tmp_install /tmp_check_cli __pycache__/ test_output/ diff --git a/.gitmodules b/.gitmodules index 8975c6e2fa..23765194c1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,8 @@ -[submodule "vendor/postgres"] - path = vendor/postgres - url = https://github.com/zenithdb/postgres +[submodule "vendor/postgres-v14"] + path = vendor/postgres-v14 + url = https://github.com/neondatabase/postgres.git branch = main +[submodule "vendor/postgres-v15"] + path = vendor/postgres-v15 + url = https://github.com/neondatabase/postgres.git + branch = REL_15_STABLE_neon diff --git a/Dockerfile b/Dockerfile index aa31e227da..d379c05051 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,20 +5,24 @@ ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com ARG IMAGE=rust ARG TAG=pinned +# ARGs don't get replaced in RUN commands in Kaniko +# so use hardcoded value below +# ARG PG_VERSION=v14 # Build Postgres FROM $REPOSITORY/$IMAGE:$TAG AS pg-build WORKDIR /home/nonroot -COPY --chown=nonroot vendor/postgres vendor/postgres +ARG PG_VERSION=v14 +COPY --chown=nonroot vendor/postgres-v14 vendor/postgres-v14 COPY --chown=nonroot pgxn pgxn COPY --chown=nonroot Makefile Makefile ENV BUILD_TYPE release RUN set -e \ - && mold -run make -j $(nproc) -s neon-pg-ext \ - && rm -rf tmp_install/build \ - && tar -C tmp_install -czf /home/nonroot/postgres_install.tar.gz . + && mold -run make -j $(nproc) -s neon-pg-ext-v14 \ + && rm -rf pg_install/v14/build \ + && tar -C pg_install/v14 -czf /home/nonroot/postgres_install.tar.gz . # Build zenith binaries FROM $REPOSITORY/$IMAGE:$TAG AS build @@ -35,7 +39,8 @@ ARG CACHEPOT_BUCKET=neon-github-dev #ARG AWS_ACCESS_KEY_ID #ARG AWS_SECRET_ACCESS_KEY -COPY --from=pg-build /home/nonroot/tmp_install/include/postgresql/server tmp_install/include/postgresql/server +ARG PG_VERSION=v14 +COPY --from=pg-build /home/nonroot/pg_install/v14/include/postgresql/server pg_install/v14/include/postgresql/server COPY . . # Show build caching stats to check if it was used in the end. @@ -64,7 +69,9 @@ COPY --from=build --chown=zenith:zenith /home/nonroot/target/release/pageserver COPY --from=build --chown=zenith:zenith /home/nonroot/target/release/safekeeper /usr/local/bin COPY --from=build --chown=zenith:zenith /home/nonroot/target/release/proxy /usr/local/bin -COPY --from=pg-build /home/nonroot/tmp_install/ /usr/local/ +# v14 is default for now +ARG PG_VERSION=v14 +COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/ COPY --from=pg-build /home/nonroot/postgres_install.tar.gz /data/ # By default, pageserver uses `.neon/` working directory in WORKDIR, so create one and fill it with the dummy config. diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node-v14 similarity index 98% rename from Dockerfile.compute-node rename to Dockerfile.compute-node-v14 index 3298032030..8ddf752191 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node-v14 @@ -2,6 +2,7 @@ ARG TAG=pinned # apparently, ARGs don't get replaced in RUN commands in kaniko # ARG POSTGIS_VERSION=3.3.0 # ARG PLV8_VERSION=3.1.4 +# ARG PG_VERSION=v14 # # Layer "build-deps" @@ -16,7 +17,7 @@ RUN apt update && \ # Build Postgres from the neon postgres repository. # FROM build-deps AS pg-build -COPY vendor/postgres postgres +COPY vendor/postgres-v14 postgres RUN cd postgres && \ ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ @@ -28,8 +29,8 @@ RUN cd postgres && \ # # Layer "postgis-build" # Build PostGIS from the upstream PostGIS mirror. -# -# PostGIS compiles against neon postgres sources without changes. Perhaps we +# +# PostGIS compiles against neon postgres sources without changes. Perhaps we # could even use the upstream binaries, compiled against vanilla Postgres, but # it would require some investigation to check that it works, and also keeps # working in the future. So for now, we compile our own binaries. diff --git a/Dockerfile.compute-node-v15 b/Dockerfile.compute-node-v15 new file mode 100644 index 0000000000..f949ef7680 --- /dev/null +++ b/Dockerfile.compute-node-v15 @@ -0,0 +1,172 @@ +# +# This file is identical to the Dockerfile.compute-node-v14 file +# except for the version of Postgres that is built. +# + +ARG TAG=pinned +# apparently, ARGs don't get replaced in RUN commands in kaniko +# ARG POSTGIS_VERSION=3.3.0 +# ARG PLV8_VERSION=3.1.4 +# ARG PG_VERSION=v15 + +# +# Layer "build-deps" +# +FROM debian:bullseye-slim AS build-deps +RUN apt update && \ + apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \ + libcurl4-openssl-dev libossp-uuid-dev + +# +# Layer "pg-build" +# Build Postgres from the neon postgres repository. +# +FROM build-deps AS pg-build +COPY vendor/postgres-v15 postgres +RUN cd postgres && \ + ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \ + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \ + # Install headers + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \ + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install + +# +# Layer "postgis-build" +# Build PostGIS from the upstream PostGIS mirror. +# +# PostGIS compiles against neon postgres sources without changes. Perhaps we +# could even use the upstream binaries, compiled against vanilla Postgres, but +# it would require some investigation to check that it works, and also keeps +# working in the future. So for now, we compile our own binaries. +FROM build-deps AS postgis-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ +RUN apt update && \ + apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc wget + +RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.0.tar.gz && \ + tar xvzf postgis-3.3.0.tar.gz && \ + cd postgis-3.3.0 && \ + ./autogen.sh && \ + export PATH="/usr/local/pgsql/bin:$PATH" && \ + ./configure && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + cd extensions/postgis && \ + make clean && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_raster.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control + +# +# Layer "plv8-build" +# Build plv8 +# +FROM build-deps AS plv8-build +COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ +RUN apt update && \ + apt install -y git curl wget make ninja-build build-essential libncurses5 python3-dev pkg-config libc++-dev libc++abi-dev libglib2.0-dev + +# https://github.com/plv8/plv8/issues/475 +# Debian bullseye provides binutils 2.35 when >= 2.38 is necessary +RUN echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \ + echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \ + apt update && \ + apt install -y --no-install-recommends -t testing binutils + +RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \ + tar xvzf v3.1.4.tar.gz && \ + cd plv8-3.1.4 && \ + export PATH="/usr/local/pgsql/bin:$PATH" && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + rm -rf /plv8-* && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control + +# +# Layer "neon-pg-ext-build" +# compile neon extensions +# +FROM build-deps AS neon-pg-ext-build +COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY pgxn/ pgxn/ + +RUN make -j $(getconf _NPROCESSORS_ONLN) \ + PG_CONFIG=/usr/local/pgsql/bin/pg_config \ + -C pgxn/neon \ + -s install + +# Compile and run the Neon-specific `compute_ctl` binary +FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools +USER nonroot +# Copy entire project to get Cargo.* files with proper dependencies for the whole project +COPY --chown=nonroot . . +RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto + +# +# Clean up postgres folder before inclusion +# +FROM neon-pg-ext-build AS postgres-cleanup-layer +COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql + +# Remove binaries from /bin/ that we won't use (or would manually copy & install otherwise) +RUN cd /usr/local/pgsql/bin && rm ecpg raster2pgsql shp2pgsql pgtopo_export pgtopo_import pgsql2shp + +# Remove headers that we won't need anymore - we've completed installation of all extensions +RUN rm -r /usr/local/pgsql/include + +# Remove now-useless PGXS src infrastructure +RUN rm -r /usr/local/pgsql/lib/pgxs/src + +# Remove static postgresql libraries - all compilation is finished, so we +# can now remove these files - they must be included in other binaries by now +# if they were to be used by other libraries. +RUN rm /usr/local/pgsql/lib/lib*.a + +# +# Final layer +# Put it all together into the final image +# +FROM debian:bullseye-slim +# Add user postgres +RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \ + echo "postgres:test_console_pass" | chpasswd && \ + mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \ + chown -R postgres:postgres /var/db/postgres && \ + chmod 0750 /var/db/postgres/compute && \ + echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig + +# TODO: Check if we can make the extension setup more modular versus a linear build +# currently plv8-build copies the output /usr/local/pgsql from postgis-build, etc# +COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local +COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl + +# Install: +# libreadline8 for psql +# libossp-uuid16 for extension ossp-uuid +# libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS +# GLIBC 2.34 for plv8. +# Debian bullseye provides GLIBC 2.31, so we install the library from testing +# +# Lastly, link compute_ctl into zenith_ctl while we're at it, +# so that we don't need to put this in another layer. +RUN apt update && \ + apt install --no-install-recommends -y \ + libreadline8 \ + libossp-uuid16 \ + libgeos-c1v5 \ + libgdal28 \ + libproj19 \ + libprotobuf-c1 && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + echo "Installing GLIBC 2.34" && \ + echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \ + echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \ + apt update && \ + apt install -y --no-install-recommends -t testing libc6 && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl + +USER postgres +ENTRYPOINT ["/usr/local/bin/compute_ctl"] diff --git a/Dockerfile.compute-node.legacy b/Dockerfile.compute-node.legacy index ba34e2486f..7689167156 100644 --- a/Dockerfile.compute-node.legacy +++ b/Dockerfile.compute-node.legacy @@ -37,7 +37,8 @@ RUN adduser postgres RUN mkdir /pg && chown postgres:postgres /pg # Copy source files -COPY ./vendor/postgres /pg/ +# version 14 is default for now +COPY ./vendor/postgres-v14 /pg/ COPY ./pgxn /pg/ # Build and install Postgres locally diff --git a/Makefile b/Makefile index 9d7e1497e5..0b2b097ebc 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,7 @@ ROOT_PROJECT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -# Where to install Postgres, default is ./tmp_install, maybe useful for package managers -POSTGRES_INSTALL_DIR ?= $(ROOT_PROJECT_DIR)/tmp_install - -# Seccomp BPF is only available for Linux -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) - SECCOMP = --with-libseccomp -else - SECCOMP = -endif +# Where to install Postgres, default is ./pg_install, maybe useful for package managers +POSTGRES_INSTALL_DIR ?= $(ROOT_PROJECT_DIR)/pg_install/ # # We differentiate between release / debug build types using the BUILD_TYPE @@ -28,6 +20,13 @@ else $(error Bad build type '$(BUILD_TYPE)', see Makefile for options) endif +# Seccomp BPF is only available for Linux +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + PG_CONFIGURE_OPTS += --with-libseccomp +endif + + # macOS with brew-installed openssl requires explicit paths # It can be configured with OPENSSL_PREFIX variable UNAME_S := $(shell uname -s) @@ -48,75 +47,136 @@ CARGO_CMD_PREFIX += $(if $(filter n,$(MAKEFLAGS)),,+) CARGO_CMD_PREFIX += CARGO_TERM_PROGRESS_WHEN=never CI=1 # -# Top level Makefile to build Zenith and PostgreSQL +# Top level Makefile to build Neon and PostgreSQL # .PHONY: all -all: zenith postgres neon-pg-ext +all: neon postgres neon-pg-ext -### Zenith Rust bits +### Neon Rust bits # # The 'postgres_ffi' depends on the Postgres headers. -.PHONY: zenith -zenith: postgres-headers - +@echo "Compiling Zenith" +.PHONY: neon +neon: postgres-v14-headers + +@echo "Compiling Neon" $(CARGO_CMD_PREFIX) cargo build $(CARGO_BUILD_FLAGS) ### PostgreSQL parts -$(POSTGRES_INSTALL_DIR)/build/config.status: - +@echo "Configuring postgres build" - mkdir -p $(POSTGRES_INSTALL_DIR)/build - (cd $(POSTGRES_INSTALL_DIR)/build && \ - $(ROOT_PROJECT_DIR)/vendor/postgres/configure CFLAGS='$(PG_CFLAGS)' \ +# The rules are duplicated for Postgres v14 and 15. We may want to refactor +# to avoid the duplication in the future, but it's tolerable for now. +# +$(POSTGRES_INSTALL_DIR)/build/v14/config.status: + +@echo "Configuring Postgres v14 build" + mkdir -p $(POSTGRES_INSTALL_DIR)/build/v14 + (cd $(POSTGRES_INSTALL_DIR)/build/v14 && \ + $(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure CFLAGS='$(PG_CFLAGS)' \ $(PG_CONFIGURE_OPTS) \ - $(SECCOMP) \ - --prefix=$(abspath $(POSTGRES_INSTALL_DIR)) > configure.log) + --prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v14 > configure.log) -# nicer alias for running 'configure' -.PHONY: postgres-configure -postgres-configure: $(POSTGRES_INSTALL_DIR)/build/config.status +$(POSTGRES_INSTALL_DIR)/build/v15/config.status: + +@echo "Configuring Postgres v15 build" + mkdir -p $(POSTGRES_INSTALL_DIR)/build/v15 + (cd $(POSTGRES_INSTALL_DIR)/build/v15 && \ + $(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure CFLAGS='$(PG_CFLAGS)' \ + $(PG_CONFIGURE_OPTS) \ + --prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v15 > configure.log) -# Install the PostgreSQL header files into $(POSTGRES_INSTALL_DIR)/include -.PHONY: postgres-headers -postgres-headers: postgres-configure - +@echo "Installing PostgreSQL headers" - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/src/include MAKELEVEL=0 install +# nicer alias to run 'configure' +.PHONY: postgres-v14-configure +postgres-v14-configure: $(POSTGRES_INSTALL_DIR)/build/v14/config.status -# Compile and install PostgreSQL and contrib/neon -.PHONY: postgres -postgres: postgres-configure \ - postgres-headers # to prevent `make install` conflicts with zenith's `postgres-headers` - +@echo "Compiling PostgreSQL" - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build MAKELEVEL=0 install - +@echo "Compiling libpq" - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/src/interfaces/libpq install - +@echo "Compiling pg_buffercache" - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/pg_buffercache install - +@echo "Compiling pageinspect" - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/pageinspect install +.PHONY: postgres-v15-configure +postgres-v15-configure: $(POSTGRES_INSTALL_DIR)/build/v15/config.status -.PHONY: postgres-clean -postgres-clean: - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build MAKELEVEL=0 clean - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/pg_buffercache clean - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/pageinspect clean - $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/src/interfaces/libpq clean +# Install the PostgreSQL header files into $(POSTGRES_INSTALL_DIR)//include +.PHONY: postgres-v14-headers +postgres-v14-headers: postgres-v14-configure + +@echo "Installing PostgreSQL v14 headers" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/include MAKELEVEL=0 install -neon-pg-ext: postgres - +@echo "Compiling neon" - $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/bin/pg_config \ - -C $(ROOT_PROJECT_DIR)/pgxn/neon install - +@echo "Compiling neon_test_utils" - $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/bin/pg_config \ - -C $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils install +.PHONY: postgres-v15-headers +postgres-v15-headers: postgres-v15-configure + +@echo "Installing PostgreSQL v15 headers" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/include MAKELEVEL=0 install + +# Compile and install PostgreSQL +.PHONY: postgres-v14 +postgres-v14: postgres-v14-configure \ + postgres-v14-headers # to prevent `make install` conflicts with neon's `postgres-headers` + +@echo "Compiling PostgreSQL v14" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14 MAKELEVEL=0 install + +@echo "Compiling libpq v14" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/interfaces/libpq install + +@echo "Compiling pg_buffercache v14" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_buffercache install + +@echo "Compiling pageinspect v14" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pageinspect install + +.PHONY: postgres-v15 +postgres-v15: postgres-v15-configure \ + postgres-v15-headers # to prevent `make install` conflicts with neon's `postgres-headers` + +@echo "Compiling PostgreSQL v15" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15 MAKELEVEL=0 install + +@echo "Compiling libpq v15" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/interfaces/libpq install + +@echo "Compiling pg_buffercache v15" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_buffercache install + +@echo "Compiling pageinspect v15" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pageinspect install + +# shorthand to build all Postgres versions +postgres: postgres-v14 postgres-v15 + +.PHONY: postgres-v14-clean +postgres-v14-clean: + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14 MAKELEVEL=0 clean + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_buffercache clean + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pageinspect clean + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/interfaces/libpq clean + +.PHONY: postgres-v15-clean +postgres-v15-clean: + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15 MAKELEVEL=0 clean + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_buffercache clean + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pageinspect clean + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/interfaces/libpq clean + +neon-pg-ext-v14: postgres-v14 + +@echo "Compiling neon v14" + mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-v14 + (cd $(POSTGRES_INSTALL_DIR)/build/neon-v14 && \ + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install) + +@echo "Compiling neon_test_utils" v14 + mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14 + (cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14 && \ + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile install) + +neon-pg-ext-v15: postgres-v15 + +@echo "Compiling neon v15" + mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-v15 + (cd $(POSTGRES_INSTALL_DIR)/build/neon-v15 && \ + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install) + +@echo "Compiling neon_test_utils" v15 + mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15 + (cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15 && \ + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile install) .PHONY: neon-pg-ext-clean $(MAKE) -C $(ROOT_PROJECT_DIR)/pgxn/neon clean $(MAKE) -C $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils clean +neon-pg-ext: neon-pg-ext-v14 neon-pg-ext-v15 +postgres-headers: postgres-v14-headers postgres-v15-headers +postgres-clean: postgres-v14-clean postgres-v15-clean + # This doesn't remove the effects of 'configure'. .PHONY: clean clean: - cd $(POSTGRES_INSTALL_DIR)/build && $(MAKE) clean + cd $(POSTGRES_INSTALL_DIR)/build/v14 && $(MAKE) clean + cd $(POSTGRES_INSTALL_DIR)/build/v15 && $(MAKE) clean $(CARGO_CMD_PREFIX) cargo clean cd pgxn/neon && $(MAKE) clean cd pgxn/neon_test_utils && $(MAKE) clean diff --git a/NOTICE b/NOTICE index 47cc4e798f..4fbec9763b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Neon Copyright 2022 Neon Inc. -The PostgreSQL submodule in vendor/postgres is licensed under the -PostgreSQL license. See vendor/postgres/COPYRIGHT. +The PostgreSQL submodules in vendor/postgres-v14 and vendor/postgres-v15 are licensed under the +PostgreSQL license. See vendor/postgres-v14/COPYRIGHT and vendor/postgres-v15/COPYRIGHT. diff --git a/README.md b/README.md index f557b19987..57d0a144cb 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Pageserver consists of: - WAL receiver - service that receives WAL from WAL service and stores it in the repository. - Page service - service that communicates with compute nodes and responds with pages from the repository. - WAL redo - service that builds pages from base images and WAL records on Page service request + ## Running local installation @@ -101,7 +102,7 @@ make -j`sysctl -n hw.logicalcpu` ``` #### Dependency installation notes -To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively. +To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `pg_install/bin` and `pg_install/lib`, respectively. To run the integration tests or Python scripts (not required to use the code), install Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (requires [poetry](https://python-poetry.org/)) in the project directory. @@ -208,7 +209,7 @@ Ensure your dependencies are installed as described [here](https://github.com/ne ```sh git clone --recursive https://github.com/neondatabase/neon.git -make # builds also postgres and installs it to ./tmp_install +make # builds also postgres and installs it to ./pg_install ./scripts/pytest ``` diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 75e552f6cc..c4a61dbd7b 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -289,13 +289,13 @@ impl LocalEnv { let mut env: LocalEnv = toml::from_str(toml)?; // Find postgres binaries. - // Follow POSTGRES_DISTRIB_DIR if set, otherwise look in "tmp_install". + // Follow POSTGRES_DISTRIB_DIR if set, otherwise look in "pg_install/v14". if env.pg_distrib_dir == Path::new("") { if let Some(postgres_bin) = env::var_os("POSTGRES_DISTRIB_DIR") { env.pg_distrib_dir = postgres_bin.into(); } else { let cwd = env::current_dir()?; - env.pg_distrib_dir = cwd.join("tmp_install") + env.pg_distrib_dir = cwd.join("pg_install/v14") } } diff --git a/docs/settings.md b/docs/settings.md index 5a0e976b47..30db495dbe 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -157,7 +157,7 @@ for other files and for sockets for incoming connections. A directory with Postgres installation to use during pageserver activities. Inside that dir, a `bin/postgres` binary should be present. -The default distrib dir is `./tmp_install/`. +The default distrib dir is `./pg_install/`. #### workdir (-D) diff --git a/docs/sourcetree.md b/docs/sourcetree.md index 88f4b0e559..f3bc9230e2 100644 --- a/docs/sourcetree.md +++ b/docs/sourcetree.md @@ -40,15 +40,15 @@ and create new databases and accounts (control plane API in our case). Integration tests, written in Python using the `pytest` framework. -`/vendor/postgres`: +`/vendor/postgres-v14`: PostgreSQL source tree, with the modifications needed for Neon. -`/vendor/postgres/contrib/neon`: +`/pgxn/neon`: PostgreSQL extension that implements storage manager API and network communications with remote page server. -`/vendor/postgres/contrib/neon_test_utils`: +`/pgxn/neon_test_utils`: PostgreSQL extension that contains functions needed for testing and debugging. diff --git a/libs/postgres_ffi/build.rs b/libs/postgres_ffi/build.rs index 69b2711c22..19507f0557 100644 --- a/libs/postgres_ffi/build.rs +++ b/libs/postgres_ffi/build.rs @@ -47,14 +47,17 @@ fn main() { println!("cargo:rerun-if-changed=bindgen_deps.h"); // Finding the location of C headers for the Postgres server: - // - if POSTGRES_INSTALL_DIR is set look into it, otherwise look into `/tmp_install` - // - if there's a `bin/pg_config` file use it for getting include server, otherwise use `/tmp_install/include/postgresql/server` + // - if POSTGRES_INSTALL_DIR is set look into it, otherwise look into `/pg_install` + // - if there's a `bin/pg_config` file use it for getting include server, otherwise use `/pg_install/v14/include/postgresql/server` let mut pg_install_dir = if let Some(postgres_install_dir) = env::var_os("POSTGRES_INSTALL_DIR") { postgres_install_dir.into() } else { - PathBuf::from("tmp_install") + PathBuf::from("pg_install") }; + // Currently, we only expect to find PostgreSQL v14 sources, in "pg_install/v14". In the + // future, we will run this for all supported PostgreSQL versions. + pg_install_dir.push("v14"); if pg_install_dir.is_relative() { let cwd = env::current_dir().unwrap(); diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index e7838c3f2c..0d9aaa4708 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -449,7 +449,7 @@ mod tests { .join("..") .join(".."); let cfg = Conf { - pg_distrib_dir: top_path.join("tmp_install"), + pg_distrib_dir: top_path.join("pg_install/v14"), datadir: top_path.join(format!("test_output/{}", test_name)), }; if cfg.datadir.exists() { diff --git a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs index 938f8f421b..2a607db6dc 100644 --- a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs +++ b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs @@ -37,7 +37,7 @@ fn main() -> Result<()> { Arg::new("pg-distrib-dir") .long("pg-distrib-dir") .takes_value(true) - .help("Directory with Postgres distribution (bin and lib directories, e.g. tmp_install)") + .help("Directory with Postgres distribution (bin and lib directories, e.g. pg_install/v14)") .default_value("/usr/local") ) ) diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index c1c4169e14..fb70ea327d 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -205,7 +205,7 @@ impl Default for PageServerConfigBuilder { workdir: Set(PathBuf::new()), pg_distrib_dir: Set(env::current_dir() .expect("cannot access current directory") - .join("tmp_install")), + .join("pg_install/v14")), auth_type: Set(AuthType::Trust), auth_validation_public_key_path: Set(None), remote_storage_config: Set(None), diff --git a/pgxn/neon/inmem_smgr.c b/pgxn/neon/inmem_smgr.c index 7840292b08..13fd4d50b6 100644 --- a/pgxn/neon/inmem_smgr.c +++ b/pgxn/neon/inmem_smgr.c @@ -29,6 +29,10 @@ #include "storage/relfilenode.h" #include "storage/smgr.h" +#if PG_VERSION_NUM >= 150000 +#include "access/xlogutils.h" +#endif + /* Size of the in-memory smgr */ #define MAX_PAGES 64 diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index 21d6dfec52..e3f083fd43 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -64,6 +64,11 @@ #include "catalog/pg_tablespace_d.h" #include "postmaster/autovacuum.h" +#if PG_VERSION_NUM >= 150000 +#include "access/xlogutils.h" +#include "access/xlogrecovery.h" +#endif + /* * If DEBUG_COMPARE_LOCAL is defined, we pass through all the SMGR API * calls to md.c, and *also* do the calls to the Page Server. On every @@ -645,7 +650,11 @@ zenith_get_request_lsn(bool *latest, RelFileNode rnode, ForkNumber forknum, Bloc * _bt_blwritepage logs the full page without flushing WAL before * smgrextend (files are fsynced before build ends). */ +#if PG_VERSION_NUM >= 150000 + flushlsn = GetFlushRecPtr(NULL); +#else flushlsn = GetFlushRecPtr(); +#endif if (lsn > flushlsn) { elog(DEBUG5, "last-written LSN %X/%X is ahead of last flushed LSN %X/%X", diff --git a/pgxn/neon/relsize_cache.c b/pgxn/neon/relsize_cache.c index 8dfcffe1d1..31021f3e41 100644 --- a/pgxn/neon/relsize_cache.c +++ b/pgxn/neon/relsize_cache.c @@ -24,6 +24,9 @@ #include "utils/dynahash.h" #include "utils/guc.h" +#if PG_VERSION_NUM >= 150000 +#include "miscadmin.h" +#endif typedef struct { @@ -41,6 +44,10 @@ static HTAB *relsize_hash; static LWLockId relsize_lock; static int relsize_hash_size; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void relsize_shmem_request(void); +#endif /* * Size of a cache entry is 20 bytes. So this default will take about 1.2 MB, @@ -158,10 +165,31 @@ relsize_hash_init(void) if (relsize_hash_size > 0) { +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = relsize_shmem_request; +#else RequestAddinShmemSpace(hash_estimate_size(relsize_hash_size, sizeof(RelSizeEntry))); RequestNamedLWLockTranche("neon_relsize", 1); +#endif prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = zenith_smgr_shmem_startup; } } + +#if PG_VERSION_NUM >= 150000 +/* + * shmem_request hook: request additional shared resources. We'll allocate or + * attach to the shared resources in zenith_smgr_shmem_startup(). + */ +static void +relsize_shmem_request(void) +{ + if (prev_shmem_request_hook) + prev_shmem_request_hook(); + + RequestAddinShmemSpace(hash_estimate_size(relsize_hash_size, sizeof(RelSizeEntry))); + RequestNamedLWLockTranche("neon_relsize", 1); +} +#endif diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index 3baa4802b0..a769a5216b 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -39,6 +39,10 @@ #include "access/xact.h" #include "access/xlogdefs.h" #include "access/xlogutils.h" +#include "access/xloginsert.h" +#if PG_VERSION_NUM >= 150000 +#include "access/xlogrecovery.h" +#endif #include "storage/latch.h" #include "miscadmin.h" #include "pgstat.h" @@ -165,7 +169,10 @@ static bool backpressure_throttling_impl(void); static process_interrupts_callback_t PrevProcessInterruptsCallback; static shmem_startup_hook_type prev_shmem_startup_hook_type; - +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void walproposer_shmem_request(void); +#endif void pg_init_walproposer(void) @@ -221,19 +228,38 @@ static void nwp_register_gucs(void) GUC_UNIT_MS, NULL, NULL, NULL ); - + } /* shmem handling */ static void nwp_prepare_shmem(void) { +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = walproposer_shmem_request; +#else RequestAddinShmemSpace(WalproposerShmemSize()); - +#endif prev_shmem_startup_hook_type = shmem_startup_hook; shmem_startup_hook = nwp_shmem_startup_hook; } +#if PG_VERSION_NUM >= 150000 +/* + * shmem_request hook: request additional shared resources. We'll allocate or + * attach to the shared resources in nwp_shmem_startup_hook(). + */ +static void +walproposer_shmem_request(void) +{ + if (prev_shmem_request_hook) + prev_shmem_request_hook(); + + RequestAddinShmemSpace(WalproposerShmemSize()); +} +#endif + static void nwp_shmem_startup_hook(void) { if (prev_shmem_startup_hook_type) @@ -248,6 +274,10 @@ static void nwp_shmem_startup_hook(void) void WalProposerMain(Datum main_arg) { +#if PG_VERSION_NUM >= 150000 + TimeLineID tli; +#endif + /* Establish signal handlers. */ pqsignal(SIGUSR1, procsignal_sigusr1_handler); pqsignal(SIGHUP, SignalHandlerForConfigReload); @@ -255,9 +285,14 @@ WalProposerMain(Datum main_arg) BackgroundWorkerUnblockSignals(); +#if PG_VERSION_NUM >= 150000 + // FIXME pass proper tli to WalProposerInit ? + GetXLogReplayRecPtr(&tli); + WalProposerInit(GetFlushRecPtr(NULL), GetSystemIdentifier()); +#else GetXLogReplayRecPtr(&ThisTimeLineID); - WalProposerInit(GetFlushRecPtr(), GetSystemIdentifier()); +#endif last_reconnect_attempt = GetCurrentTimestamp(); @@ -468,7 +503,12 @@ WalProposerInitImpl(XLogRecPtr flushRecPtr, uint64 systemId) !HexDecodeString(greetRequest.ztenantid, zenith_tenant_walproposer, 16)) elog(FATAL, "Could not parse neon.tenant_id, %s", zenith_tenant_walproposer); +#if PG_VERSION_NUM >= 150000 +// FIXME don't use hardcoded timeline id + greetRequest.timeline = 1; +#else greetRequest.timeline = ThisTimeLineID; +#endif greetRequest.walSegSize = wal_segment_size; InitEventSet(); @@ -1702,7 +1742,12 @@ SendAppendRequests(Safekeeper *sk) &sk->outbuf.data[sk->outbuf.len], req->beginLsn, req->endLsn - req->beginLsn, + #if PG_VERSION_NUM >= 150000 + // FIXME don't use hardcoded timelineid here + 1, + #else ThisTimeLineID, + #endif &errinfo)) { WALReadRaiseError(&errinfo); @@ -2373,8 +2418,11 @@ backpressure_lag_impl(void) XLogRecPtr writePtr; XLogRecPtr flushPtr; XLogRecPtr applyPtr; +#if PG_VERSION_NUM >= 150000 + XLogRecPtr myFlushLsn = GetFlushRecPtr(NULL); +#else XLogRecPtr myFlushLsn = GetFlushRecPtr(); - +#endif replication_feedback_get_lsns(&writePtr, &flushPtr, &applyPtr); #define MB ((XLogRecPtr)1024*1024) diff --git a/pgxn/neon/walproposer_utils.c b/pgxn/neon/walproposer_utils.c index 7b96fd580c..417a8c4586 100644 --- a/pgxn/neon/walproposer_utils.c +++ b/pgxn/neon/walproposer_utils.c @@ -21,6 +21,11 @@ #include #include +#if PG_VERSION_NUM >= 150000 +#include "access/xlogutils.h" +#include "access/xlogrecovery.h" +#endif + /* * These variables are used similarly to openLogFile/SegNo, * but for walproposer to write the XLOG during recovery. walpropFileTLI is the TimeLineID @@ -85,7 +90,11 @@ static volatile sig_atomic_t replication_active = false; typedef void (*WalSndSendDataCallback) (void); static void WalSndLoop(WalSndSendDataCallback send_data); static void XLogSendPhysical(void); +#if PG_VERSION_NUM >= 150000 +static XLogRecPtr GetStandbyFlushRecPtr(TimeLineID *tli); +#else static XLogRecPtr GetStandbyFlushRecPtr(void); +#endif static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); @@ -222,10 +231,10 @@ SafekeeperStateDesiredEvents(SafekeeperState state) result = WL_SOCKET_READABLE; break; - /* + /* * Flush states require write-ready for flushing. * Active state does both reading and writing. - * + * * TODO: SS_ACTIVE sometimes doesn't need to be write-ready. We should * check sk->flushWrite here to set WL_SOCKET_WRITEABLE. */ @@ -398,12 +407,21 @@ XLogWalPropWrite(char *buf, Size nbytes, XLogRecPtr recptr) if (walpropFile < 0) { + #if PG_VERSION_NUM >= 150000 + // FIXME Is it ok to use hardcoded value here? + TimeLineID tli = 1; + #else bool use_existent = true; - + #endif /* Create/use new log file */ XLByteToSeg(recptr, walpropSegNo, wal_segment_size); + #if PG_VERSION_NUM >= 150000 + walpropFile = XLogFileInit(walpropSegNo, tli); + walpropFileTLI = tli; + #else walpropFile = XLogFileInit(walpropSegNo, &use_existent, false); walpropFileTLI = ThisTimeLineID; + #endif } /* Calculate the start offset of the received logs */ @@ -488,11 +506,14 @@ void StartProposerReplication(StartReplicationCmd *cmd) { XLogRecPtr FlushPtr; + TimeLineID currTLI; + #if PG_VERSION_NUM < 150000 if (ThisTimeLineID == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION"))); + #endif /* create xlogreader for physical replication */ xlogreader = @@ -534,10 +555,19 @@ StartProposerReplication(StartReplicationCmd *cmd) * Select the timeline. If it was given explicitly by the client, use * that. Otherwise use the timeline of the last replayed record, which is * kept in ThisTimeLineID. - * + * * Neon doesn't currently use PG Timelines, but it may in the future, so * we keep this code around to lighten the load for when we need it. */ +#if PG_VERSION_NUM >= 150000 + if (am_cascading_walsender) + { + /* this also updates ThisTimeLineID */ + FlushPtr = GetStandbyFlushRecPtr(&currTLI); + } + else + FlushPtr = GetFlushRecPtr(&currTLI); +#else if (am_cascading_walsender) { /* this also updates ThisTimeLineID */ @@ -546,12 +576,16 @@ StartProposerReplication(StartReplicationCmd *cmd) else FlushPtr = GetFlushRecPtr(); + currTLI = ThisTimeLineID; +#endif + + if (cmd->timeline != 0) { XLogRecPtr switchpoint; sendTimeLine = cmd->timeline; - if (sendTimeLine == ThisTimeLineID) + if (sendTimeLine == currTLI) { sendTimeLineIsHistoric = false; sendTimeLineValidUpto = InvalidXLogRecPtr; @@ -566,7 +600,7 @@ StartProposerReplication(StartReplicationCmd *cmd) * Check that the timeline the client requested exists, and the * requested start location is on that timeline. */ - timeLineHistory = readTimeLineHistory(ThisTimeLineID); + timeLineHistory = readTimeLineHistory(currTLI); switchpoint = tliSwitchPoint(cmd->timeline, timeLineHistory, &sendTimeLineNextTLI); list_free_deep(timeLineHistory); @@ -605,7 +639,7 @@ StartProposerReplication(StartReplicationCmd *cmd) } else { - sendTimeLine = ThisTimeLineID; + sendTimeLine = currTLI; sendTimeLineValidUpto = InvalidXLogRecPtr; sendTimeLineIsHistoric = false; } @@ -710,6 +744,34 @@ StartProposerReplication(StartReplicationCmd *cmd) EndReplicationCommand("START_STREAMING"); } +#if PG_VERSION_NUM >= 150000 +static XLogRecPtr +GetStandbyFlushRecPtr(TimeLineID *tli) +{ + XLogRecPtr replayPtr; + TimeLineID replayTLI; + XLogRecPtr receivePtr; + TimeLineID receiveTLI; + XLogRecPtr result; + + /* + * We can safely send what's already been replayed. Also, if walreceiver + * is streaming WAL from the same timeline, we can send anything that it + * has streamed, but hasn't been replayed yet. + */ + + receivePtr = GetWalRcvFlushRecPtr(NULL, &receiveTLI); + replayPtr = GetXLogReplayRecPtr(&replayTLI); + + *tli = replayTLI; + + result = replayPtr; + if (receiveTLI == replayTLI && receivePtr > replayPtr) + result = receivePtr; + + return result; +} +#else /* * Returns the latest point in WAL that has been safely flushed to disk, and * can be sent to the standby. This should only be called when in recovery, @@ -744,6 +806,9 @@ GetStandbyFlushRecPtr(void) return result; } +#endif + + /* XLogReaderRoutine->segment_open callback */ static void @@ -878,6 +943,7 @@ XLogSendPhysical(void) XLogRecPtr startptr; XLogRecPtr endptr; Size nbytes PG_USED_FOR_ASSERTS_ONLY; + TimeLineID currTLI; /* If requested switch the WAL sender to the stopping state. */ if (got_STOPPING) @@ -919,9 +985,12 @@ XLogSendPhysical(void) * FlushPtr that was calculated before it became historic. */ bool becameHistoric = false; - +#if PG_VERSION_NUM >= 150000 + SendRqstPtr = GetStandbyFlushRecPtr(&currTLI); +#else SendRqstPtr = GetStandbyFlushRecPtr(); - + currTLI = ThisTimeLineID; +#endif if (!RecoveryInProgress()) { /* @@ -935,10 +1004,10 @@ XLogSendPhysical(void) { /* * Still a cascading standby. But is the timeline we're sending - * still the one recovery is recovering from? ThisTimeLineID was + * still the one recovery is recovering from? currTLI was * updated by the GetStandbyFlushRecPtr() call above. */ - if (sendTimeLine != ThisTimeLineID) + if (sendTimeLine != currTLI) becameHistoric = true; } @@ -951,7 +1020,7 @@ XLogSendPhysical(void) */ List *history; - history = readTimeLineHistory(ThisTimeLineID); + history = readTimeLineHistory(currTLI); sendTimeLineValidUpto = tliSwitchPoint(sendTimeLine, history, &sendTimeLineNextTLI); Assert(sendTimeLine < sendTimeLineNextTLI); @@ -974,7 +1043,11 @@ XLogSendPhysical(void) * primary: if the primary subsequently crashes and restarts, standbys * must not have applied any WAL that got lost on the primary. */ + #if PG_VERSION_NUM >= 150000 + SendRqstPtr = GetFlushRecPtr(NULL); + #else SendRqstPtr = GetFlushRecPtr(); + #endif } /* diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 8ffb2eb829..b47e560325 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -59,7 +59,7 @@ Env = Dict[str, str] Fn = TypeVar("Fn", bound=Callable[..., Any]) DEFAULT_OUTPUT_DIR = "test_output" -DEFAULT_POSTGRES_DIR = "tmp_install" +DEFAULT_POSTGRES_DIR = "pg_install/v14" DEFAULT_BRANCH_NAME = "main" BASE_PORT = 15000 @@ -188,7 +188,7 @@ def can_bind(host: str, port: int) -> bool: Check whether a host:port is available to bind for listening Inspired by the can_bind() perl function used in Postgres tests, in - vendor/postgres/src/test/perl/PostgresNode.pm + vendor/postgres-v14/src/test/perl/PostgresNode.pm """ with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: # TODO: The pageserver and safekeepers don't use SO_REUSEADDR at the diff --git a/test_runner/regress/test_pg_regress.py b/test_runner/regress/test_pg_regress.py index 119528b8f9..aa5a65f446 100644 --- a/test_runner/regress/test_pg_regress.py +++ b/test_runner/regress/test_pg_regress.py @@ -26,8 +26,8 @@ def test_pg_regress(neon_simple_env: NeonEnv, test_output_dir: Path, pg_bin, cap (runpath / "testtablespace").mkdir(parents=True) # Compute all the file locations that pg_regress will need. - build_path = os.path.join(pg_distrib_dir, "build/src/test/regress") - src_path = os.path.join(base_dir, "vendor/postgres/src/test/regress") + build_path = os.path.join(pg_distrib_dir, "../build/v14/src/test/regress") + src_path = os.path.join(base_dir, "vendor/postgres-v14/src/test/regress") bindir = os.path.join(pg_distrib_dir, "bin") schedule = os.path.join(src_path, "parallel_schedule") pg_regress = os.path.join(build_path, "pg_regress") @@ -80,8 +80,8 @@ def test_isolation(neon_simple_env: NeonEnv, test_output_dir: Path, pg_bin, caps (runpath / "testtablespace").mkdir(parents=True) # Compute all the file locations that pg_isolation_regress will need. - build_path = os.path.join(pg_distrib_dir, "build/src/test/isolation") - src_path = os.path.join(base_dir, "vendor/postgres/src/test/isolation") + build_path = os.path.join(pg_distrib_dir, "../build/v14/src/test/isolation") + src_path = os.path.join(base_dir, "vendor/postgres-v14/src/test/isolation") bindir = os.path.join(pg_distrib_dir, "bin") schedule = os.path.join(src_path, "isolation_schedule") pg_isolation_regress = os.path.join(build_path, "pg_isolation_regress") @@ -124,7 +124,7 @@ def test_sql_regress(neon_simple_env: NeonEnv, test_output_dir: Path, pg_bin, ca # Compute all the file locations that pg_regress will need. # This test runs neon specific tests - build_path = os.path.join(pg_distrib_dir, "build/src/test/regress") + build_path = os.path.join(pg_distrib_dir, "../build/v14/src/test/regress") src_path = os.path.join(base_dir, "test_runner/sql_regress") bindir = os.path.join(pg_distrib_dir, "bin") schedule = os.path.join(src_path, "parallel_schedule") diff --git a/vendor/postgres b/vendor/postgres-v14 similarity index 100% rename from vendor/postgres rename to vendor/postgres-v14 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 new file mode 160000 index 0000000000..26c6466873 --- /dev/null +++ b/vendor/postgres-v15 @@ -0,0 +1 @@ +Subproject commit 26c64668736b729a3e4c02c6fc0a84544118df26