add missing expected file

.circleci/config.yml

@@ -37,7 +37,7 @@ jobs:
           command: |
             if [ ! -e tmp_install/bin/postgres ]; then
               sudo apt update
-              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison
+              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
             fi
 
         # Build postgres if the restore_cache didn't find a build.
@@ -119,7 +119,8 @@ jobs:
             - target
 
         # Run rust unit tests
-      - run: cargo test
+        # FIXME: remove -p zenith_utils once integration tests are moved to python
+      - run: cargo test -p zenith_utils
 
         # Install the rust binaries, for use by test jobs
         # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
@@ -191,12 +192,7 @@ jobs:
           condition: << parameters.needs_postgres_source >>
           steps:
             - run: git submodule update --init --depth 1
-      - run:
-          name: Install pipenv & deps
-          working_directory: test_runner
-          command: |
-            pip install pipenv
-            pipenv install
+      - run: pip install pytest psycopg2
       - run:
           name: Run pytest
           working_directory: test_runner
@@ -215,21 +211,25 @@ jobs:
             #
             # The junit.xml file allows CircleCI to display more fine-grained test information
             # in its "Tests" tab in the results page.
-            # -s prevents pytest from capturing output, which helps to see
-            # what's going on if the test hangs
-            # --verbose prints name of each test (helpful when there are
-            # multiple tests in one file)
-            # -rA prints summary in the end
-            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
+            pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short $TEST_SELECTION $EXTRA_PARAMS
       - run:
           # CircleCI artifacts are preserved one file at a time, so skipping
           # this step isn't a good idea. If you want to extract the
           # pageserver state, perhaps a tarball would be a better idea.
-          name: Delete all data but logs
+          name: Delete pageserver data
           when: always
           command: |
             du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" -delete
+            for DIR in /tmp/test_output/*; do
+              mv $DIR/repo/pageserver.log $DIR/ || true # ignore errors
+              for PGDIR in $DIR/repo/pgdatadirs/pg?; do
+                echo "PGDIR: $PGDIR"
+                NEW_LOG="${PGDIR##*/}_log"
+                mv $PGDIR/log "$DIR/$NEW_LOG" || true # ignore errors
+              done
+              echo "rm $DIR/repo"
+              rm -rf $DIR/repo
+            done
             du -sh /tmp/test_output/*
       - store_artifacts:
           path: /tmp/test_output

.dockerignore

@@ -1,13 +0,0 @@
-**/.git/
-**/__pycache__
-**/.pytest_cache
-
-/target
-/tmp_check
-/tmp_install
-/tmp_check_cli
-/test_output
-/.vscode
-/.zenith
-/integration_tests/.zenith
-/Dockerfile

.github/workflows/testing.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Install postgres dependencies
         run: |
           sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
 
       - name: Set pg revision for caching
         id: pg_ver

CONTRIBUTING.md

@@ -1,31 +0,0 @@
-# How to contribute
-
-Howdy! Usual good software engineering practices apply. Write
-tests. Write comments. Follow standard Rust coding practices where
-possible. Use 'cargo fmt' and 'clippy' to tidy up formatting.
-
-There are soft spots in the code, which could use cleanup,
-refactoring, additional comments, and so forth. Let's try to raise the
-bar, and clean things up as we go. Try to leave code in a better shape
-than it was before.
-
-## Submitting changes
-
-1. Make a PR for every change.
-
-   Even seemingly trivial patches can break things in surprising ways.
-Use of common sense is OK. If you're only fixing a typo in a comment,
-it's probably fine to just push it. But if in doubt, open a PR.
-
-2. Get at least one +1 on your PR before you push.
-
-   For simple patches, it will only take a minute for someone to review
-it.
-
-3. Always keep the CI green.
-
-   Do not push, if the CI failed on your PR. Even if you think it's not
-your patch's fault. Help to fix the root cause if something else has
-broken the CI, before pushing.
-
-*Happy Hacking!*

COPYRIGHT

@@ -1,20 +0,0 @@
-This software is licensed under the Apache 2.0 License:
-
-----------------------------------------------------------------------------
-Copyright 2021 Zenith Labs, Inc
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-----------------------------------------------------------------------------
-
-The PostgreSQL submodule in vendor/postgres is licensed under the
-PostgreSQL license. See vendor/postgres/COPYRIGHT.

Cargo.lock

@@ -181,9 +181,6 @@ name = "bytes"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
-dependencies = [
- "serde",
-]
 
 [[package]]
 name = "cassowary"
@@ -786,6 +783,20 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "integration_tests"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "control_plane",
+ "lazy_static",
+ "nix",
+ "pageserver",
+ "postgres",
+ "rand",
+ "walkeeper",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.3.0"
@@ -914,15 +925,6 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
 
-[[package]]
-name = "memoffset"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -1194,7 +1196,6 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-stream",
- "toml",
  "tui",
  "walkdir",
  "workspace_hack",
@@ -1360,7 +1361,6 @@ dependencies = [
  "hex",
  "lazy_static",
  "log",
- "memoffset",
  "rand",
  "regex",
  "thiserror",
@@ -2254,7 +2254,6 @@ dependencies = [
  "regex",
  "rust-s3",
  "serde",
- "serde_json",
  "slog",
  "slog-async",
  "slog-scope",
@@ -2466,12 +2465,9 @@ dependencies = [
 name = "zenith_utils"
 version = "0.1.0"
 dependencies = [
- "anyhow",
  "bincode",
- "byteorder",
  "bytes",
  "hex-literal",
- "log",
  "serde",
  "thiserror",
  "workspace_hack",

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "integration_tests",
     "pageserver",
     "walkeeper",
     "zenith",

Dockerfile

@@ -1,97 +0,0 @@
-#
-# Docker image for console integration testing.
-#
-# We may also reuse it in CI to unify installation process and as a general binaries building
-# tool for production servers.
-#
-# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
-# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
-# images which are statically linked and have guards against any dlopen. I would rather
-# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
-# we will have our own storage and drop rockdb dependency.
-#
-# Cargo-chef is used to separate dependencies building from main binaries building. This
-# way `docker build` will download and install dependencies only of there are changes to
-# out Cargo.toml files.
-#
-
-
-#
-# build postgres separately -- this layer will be rebuilt only if one of
-# mentioned paths will get any changes
-#
-FROM alpine:3.13 as pg-build
-RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
-                     make bison flex readline-dev zlib-dev perl linux-headers
-WORKDIR zenith
-COPY ./vendor/postgres vendor/postgres
-COPY ./Makefile Makefile
-# Build using clang and lld
-RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
-
-#
-# Calculate cargo dependencies.
-# This will always run, but only generate recipe.json with list of dependencies without
-# installing them.
-#
-FROM alpine:20210212 as cargo-deps-inspect
-RUN apk add --update rust cargo
-RUN cargo install cargo-chef
-WORKDIR zenith
-COPY . .
-RUN cargo chef prepare --recipe-path recipe.json
-
-#
-# Build cargo dependencies.
-# This temp cantainner would be build only if recipe.json was changed.
-#
-FROM alpine:20210212 as deps-build
-RUN apk add --update rust cargo openssl-dev clang build-base
-# rust-rocksdb can be built against system-wide rocksdb -- that saves about
-# 10 minutes during build. Rocksdb apk package is in testing now, but use it
-# anyway. In case of any troubles we can download and build rocksdb here manually
-# (to cache it as a docker layer).
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
-COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
-RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
-
-#
-# Build zenith binaries
-#
-FROM alpine:20210212 as build
-RUN apk add --update rust cargo openssl-dev clang build-base
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
-COPY . .
-# Copy cached dependencies
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=deps-build /zenith/target target
-COPY --from=deps-build /root/.cargo /root/.cargo
-RUN cargo build --release
-
-#
-# Copy binaries to resulting image.
-# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
-# out how to statically link rocksdb or avoid it at all).
-#
-FROM alpine:3.13
-RUN apk add --update openssl build-base
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
-COPY --from=build /zenith/target/release/pageserver /usr/local/bin
-COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
-COPY --from=pg-build /zenith/tmp_install /usr/local
-COPY docker-entrypoint.sh /docker-entrypoint.sh
-
-RUN addgroup zenith && adduser -h /data -D -G zenith zenith
-VOLUME ["/data"]
-WORKDIR /data
-USER zenith
-ENV ZENITH_REPO_DIR /data/
-ENV POSTGRES_DISTRIB_DIR /usr/local
-
-EXPOSE 6400
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["pageserver"]

LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

Makefile

@@ -21,8 +21,8 @@ tmp_install/build/config.status:
 	+@echo "Configuring postgres build"
 	mkdir -p tmp_install/build
 	(cd tmp_install/build && \
-	../../vendor/postgres/configure CFLAGS='-O0 $(CFLAGS)' --enable-debug --enable-cassert \
-	    --enable-depend --prefix=$(abspath tmp_install) > configure.log)
+	../../vendor/postgres/configure CFLAGS='-O0' --enable-debug --enable-cassert \
+	    --enable-depend --with-libxml --prefix=$(abspath tmp_install) > configure.log)
 
 # nicer alias for running 'configure'
 postgres-configure: tmp_install/build/config.status
@@ -38,7 +38,8 @@ postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
 	+@echo "Compiling contrib/zenith"
-	$(MAKE) -C tmp_install/build/contrib/zenith install
+	(cd vendor/postgres/contrib/zenith && \
+	$(MAKE) PG_CONFIG=$(abspath tmp_install)/bin/pg_config install USE_PGXS=1)
 
 postgres-clean:
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean

README.md

@@ -9,7 +9,7 @@ Zenith substitutes PostgreSQL storage layer and redistributes data across a clus
 On Ubuntu or Debian this set of packages should be sufficient to build the code:
 ```text
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison \
-libssl-dev clang
+libxml2-dev libcurl4-openssl-dev libssl-dev clang
 ```
 
 [Rust] 1.48 or later is also required.
@@ -105,38 +105,11 @@ cd test_runner
 pytest
 ```
 
-## Documentation
-
-Now we use README files to cover design ideas and overall architecture for each module.
-And rustdoc style documentation comments.
-
-To view your documentation in a browser, try running `cargo doc --no-deps --open`
-
 ## Source tree layout
 
-`/control_plane`:
-
-Local control plane.
-Functions to start, cofigure and stop pageserver and postgres instances running as a local processes.
-Intended to be used in integration tests and in CLI tools for local installations.
-
-`/zenith`
-
-Main entry point for the 'zenith' CLI utility.
-TODO: Doesn't it belong to control_plane?
-
-`/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/zenith_utils`:
-
-Helpers that are shared between other crates in this repository.
-
 `/walkeeper`:
 
-WAL safekeeper (also known as WAL acceptor). Written in Rust.
+WAL safekeeper. Written in Rust.
 
 `/pageserver`:
 
@@ -148,20 +121,14 @@ Depends on the modified 'postgres' binary for WAL redo.
 
 PostgreSQL source tree, with the modifications needed for Zenith.
 
-`/vendor/postgres/contrib/zenith`:
+`/vendor/postgres/src/bin/safekeeper`:
 
-PostgreSQL extension that implements storage manager API and network communications with remote page server.
+Extension (safekeeper_proxy) that runs in the compute node, and connects to the WAL safekeepers
+and streams the WAL
 
 `/test_runner`:
 
 Integration tests, written in Python using the `pytest` framework.
 
-`test_runner/zenith_regress`:
-
-Quick way to add new SQL regression test to integration tests set.
-
-`/integration_tests`:
-
-Another pack of integration tests. Written in Rust.
 
 [Rust]: https://www.rust-lang.org/learn/get-started

control_plane/Cargo.toml

@@ -16,8 +16,11 @@ toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
 anyhow = "1.0"
+# hex = "0.4.3"
 bytes = "1.0.1"
+# fs_extra = "1.2.0"
 nix = "0.20"
+# thiserror = "1"
 url = "2.2.2"
 
 pageserver = { path = "../pageserver" }

control_plane/src/compute.rs

@@ -97,6 +97,51 @@ impl ComputeControlPlane {
         Ok(node)
     }
 
+    pub fn new_test_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
+        let timeline_id = self
+            .pageserver
+            .branch_get_by_name(branch_name)
+            .expect("failed to get timeline_id")
+            .timeline_id;
+
+        let node = self.new_from_page_server(true, timeline_id, branch_name);
+        let node = node.unwrap();
+
+        // Configure the node to stream WAL directly to the pageserver
+        node.append_conf(
+            "postgresql.conf",
+            format!(
+                "shared_preload_libraries = zenith\n\
+                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
+                node.connstr()
+            )
+            .as_str(),
+        )
+        .unwrap();
+
+        node
+    }
+
+    pub fn new_test_master_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
+        let timeline_id = self
+            .pageserver
+            .branch_get_by_name(branch_name)
+            .expect("failed to get timeline_id")
+            .timeline_id;
+
+        let node = self
+            .new_from_page_server(true, timeline_id, branch_name)
+            .unwrap();
+
+        node.append_conf(
+            "postgresql.conf",
+            "synchronous_standby_names = 'safekeeper_proxy'\n",
+        )
+        .unwrap();
+
+        node
+    }
+
     pub fn new_node(&mut self, branch_name: &str) -> Result<Arc<PostgresNode>> {
         let timeline_id = self.pageserver.branch_get_by_name(branch_name)?.timeline_id;
 
@@ -106,11 +151,8 @@ impl ComputeControlPlane {
         node.append_conf(
             "postgresql.conf",
             format!(
-                concat!(
-                    "shared_preload_libraries = zenith\n",
-                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
-                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
-                ),
+                "shared_preload_libraries = zenith\n\
+                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
                 node.connstr()
             )
             .as_str(),
@@ -259,13 +301,11 @@ impl PostgresNode {
         ar.unpack(&pgdata)
             .with_context(|| "extracting page backup failed")?;
 
-        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
-        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
+        // listen for selected port
         self.append_conf(
             "postgresql.conf",
             &format!(
                 "max_wal_senders = 10\n\
-                 wal_log_hints = on\n\
                  max_replication_slots = 10\n\
                  hot_standby = on\n\
                  shared_buffers = 1MB\n\
@@ -339,8 +379,7 @@ impl PostgresNode {
                         "-D",
                         self.pgdata().to_str().unwrap(),
                         "-l",
-                        self.pgdata().join("pg.log").to_str().unwrap(),
-                        "-w", //wait till pg_ctl actually does what was asked
+                        self.pgdata().join("log").to_str().unwrap(),
                     ],
                     args,
                 ]

control_plane/src/storage.rs

@@ -44,15 +44,17 @@ impl PageServerNode {
     pub fn init(&self) -> Result<()> {
         let mut cmd = Command::new(self.env.pageserver_bin()?);
         let status = cmd
-            .args(&[
-                "--init",
-                "-D",
-                self.env.base_data_dir.to_str().unwrap(),
-                "--postgres-distrib",
-                self.env.pg_distrib_dir.to_str().unwrap(),
-            ])
+            .args(&["--init", "-D", self.env.base_data_dir.to_str().unwrap()])
             .env_clear()
             .env("RUST_BACKTRACE", "1")
+            .env(
+                "POSTGRES_DISTRIB_DIR",
+                self.env.pg_distrib_dir.to_str().unwrap(),
+            )
+            .env("ZENITH_REPO_DIR", self.repo_path())
+            .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
+            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
             .status()
             .expect("pageserver init failed");
 
@@ -79,10 +81,23 @@ impl PageServerNode {
         );
 
         let mut cmd = Command::new(self.env.pageserver_bin()?);
-        cmd.args(&["-D", self.repo_path().to_str().unwrap()])
-            .arg("-d")
-            .env_clear()
-            .env("RUST_BACKTRACE", "1");
+        cmd.args(&[
+            "-l",
+            self.address().to_string().as_str(),
+            "-D",
+            self.repo_path().to_str().unwrap(),
+        ])
+        .arg("-d")
+        .env_clear()
+        .env("RUST_BACKTRACE", "1")
+        .env(
+            "POSTGRES_DISTRIB_DIR",
+            self.env.pg_distrib_dir.to_str().unwrap(),
+        )
+        .env("ZENITH_REPO_DIR", self.repo_path())
+        .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
+        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
 
         if !cmd.status()?.success() {
             bail!(
@@ -210,6 +225,22 @@ impl PageServerNode {
 
         Ok(branch.clone())
     }
+
+    pub fn system_id_get(&self) -> Result<u64> {
+        let mut client = self.page_server_psql_client()?;
+        let query_result = client
+            .simple_query("identify_system")?
+            .first()
+            .map(|msg| match msg {
+                postgres::SimpleQueryMessage::Row(row) => row.get(0),
+                _ => None,
+            })
+            .flatten()
+            .ok_or_else(|| anyhow!("failed to get system_id"))?
+            .parse::<u64>()?;
+
+        Ok(query_result)
+    }
 }
 
 impl Drop for PageServerNode {

docker-entrypoint.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-if [ "$1" = 'pageserver' ]; then
-    if [ ! -d "/data/timelines" ]; then
-        echo "Initializing pageserver data directory"
-        pageserver --init --workdir $ZENITH_REPO_DIR
-    fi
-    echo "Staring pageserver at 0.0.0.0:6400"
-    pageserver -l 0.0.0.0:6400 --workdir $ZENITH_REPO_DIR
-else
-    "$@"
-fi

integration_tests/.gitignore

@@ -0,0 +1 @@
+tmp_check/

integration_tests/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "integration_tests"
+version = "0.1.0"
+authors = ["Stas Kelvich <stas@zenith.tech>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+lazy_static = "1.4.0"
+rand = "0.8.3"
+anyhow = "1.0"
+nix = "0.20"
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+
+pageserver = { path = "../pageserver" }
+walkeeper = { path = "../walkeeper" }
+control_plane = { path = "../control_plane" }

integration_tests/src/lib.rs

@@ -0,0 +1,414 @@
+use anyhow::{bail, Result};
+use nix::sys::signal::{kill, Signal};
+use nix::unistd::Pid;
+use std::collections::BTreeMap;
+use std::convert::TryInto;
+use std::fs::{self, File, OpenOptions};
+use std::io::Read;
+use std::net::SocketAddr;
+use std::path::{Path, PathBuf};
+use std::process::{Command, ExitStatus};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use control_plane::compute::PostgresNode;
+use control_plane::read_pidfile;
+use control_plane::{local_env::LocalEnv, storage::PageServerNode};
+
+// Find the directory where the binaries were put (i.e. target/debug/)
+fn cargo_bin_dir() -> PathBuf {
+    let mut pathbuf = std::env::current_exe().unwrap();
+
+    pathbuf.pop();
+    if pathbuf.ends_with("deps") {
+        pathbuf.pop();
+    }
+
+    pathbuf
+}
+
+// local compute env for tests
+pub fn create_test_env(testname: &str) -> LocalEnv {
+    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../tmp_check/")
+        .join(testname);
+
+    let base_path_str = base_path.to_str().unwrap();
+
+    // Remove remnants of old test repo
+    let _ = fs::remove_dir_all(&base_path);
+
+    fs::create_dir_all(&base_path)
+        .unwrap_or_else(|_| panic!("could not create directory for {}", base_path_str));
+
+    let pgdatadirs_path = base_path.join("pgdatadirs");
+    fs::create_dir(&pgdatadirs_path)
+        .unwrap_or_else(|_| panic!("could not create directory {:?}", pgdatadirs_path));
+
+    LocalEnv {
+        pageserver_connstring: "postgresql://127.0.0.1:64000".to_string(),
+        pg_distrib_dir: Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install"),
+        zenith_distrib_dir: Some(cargo_bin_dir()),
+        base_data_dir: base_path,
+        remotes: BTreeMap::default(),
+    }
+}
+
+//
+// Collection of several example deployments useful for tests.
+//
+// I'm intendedly modelling storage and compute control planes as a separate entities
+// as it is closer to the actual setup.
+//
+pub struct TestStorageControlPlane {
+    pub wal_acceptors: Vec<WalAcceptorNode>,
+    pub pageserver: Arc<PageServerNode>,
+    pub test_done: AtomicBool,
+}
+
+impl TestStorageControlPlane {
+    // postgres <-> page_server
+    //
+    // Initialize a new repository and configure a page server to run in it
+    //
+    pub fn one_page_server(local_env: &LocalEnv) -> TestStorageControlPlane {
+        let pserver = Arc::new(PageServerNode {
+            env: local_env.clone(),
+            kill_on_exit: true,
+            listen_address: None,
+        });
+        pserver.init().unwrap();
+        pserver.start().unwrap();
+
+        TestStorageControlPlane {
+            wal_acceptors: Vec::new(),
+            pageserver: pserver,
+            test_done: AtomicBool::new(false),
+        }
+    }
+
+    // postgres <-> {wal_acceptor1, wal_acceptor2, ...}
+    pub fn fault_tolerant(local_env: &LocalEnv, redundancy: usize) -> TestStorageControlPlane {
+        let mut cplane = TestStorageControlPlane {
+            wal_acceptors: Vec::new(),
+            pageserver: Arc::new(PageServerNode {
+                env: local_env.clone(),
+                kill_on_exit: true,
+                listen_address: None,
+            }),
+            test_done: AtomicBool::new(false),
+            // repopath,
+        };
+        cplane.pageserver.init().unwrap();
+        cplane.pageserver.start().unwrap();
+
+        let systemid = cplane.pageserver.system_id_get().unwrap();
+
+        const WAL_ACCEPTOR_PORT: usize = 54321;
+
+        let datadir_base = local_env.base_data_dir.join("safekeepers");
+        fs::create_dir_all(&datadir_base).unwrap();
+
+        for i in 0..redundancy {
+            let wal_acceptor = WalAcceptorNode {
+                listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i)
+                    .parse()
+                    .unwrap(),
+                data_dir: datadir_base.join(format!("wal_acceptor_{}", i)),
+                systemid,
+                env: local_env.clone(),
+                pass_to_pageserver: true,
+            };
+            wal_acceptor.init();
+            wal_acceptor.start();
+            cplane.wal_acceptors.push(wal_acceptor);
+        }
+        cplane
+    }
+
+    pub fn stop(&self) {
+        for wa in self.wal_acceptors.iter() {
+            let _ = wa.stop();
+        }
+        self.test_done.store(true, Ordering::Relaxed);
+    }
+
+    pub fn get_wal_acceptor_conn_info(&self) -> String {
+        self.wal_acceptors
+            .iter()
+            .map(|wa| wa.listen.to_string())
+            .collect::<Vec<String>>()
+            .join(",")
+    }
+
+    pub fn is_running(&self) -> bool {
+        self.test_done.load(Ordering::Relaxed)
+    }
+}
+
+impl Drop for TestStorageControlPlane {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// PostgresNodeExt
+//
+///////////////////////////////////////////////////////////////////////////////
+
+///
+/// Testing utilities for PostgresNode type
+///
+pub trait PostgresNodeExt {
+    fn pg_regress(&self) -> ExitStatus;
+    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus;
+    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode;
+    fn open_psql(&self, db: &str) -> postgres::Client;
+    fn dump_log_file(&self);
+    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row>;
+}
+
+impl PostgresNodeExt for PostgresNode {
+    fn pg_regress(&self) -> ExitStatus {
+        self.safe_psql("postgres", "CREATE DATABASE regression");
+
+        let regress_run_path = self.env.base_data_dir.join("regress");
+        fs::create_dir_all(&regress_run_path).unwrap();
+        fs::create_dir_all(regress_run_path.join("testtablespace")).unwrap();
+        std::env::set_current_dir(regress_run_path).unwrap();
+
+        let regress_build_path =
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install/build/src/test/regress");
+        let regress_src_path =
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("../vendor/postgres/src/test/regress");
+
+        let regress_check = Command::new(regress_build_path.join("pg_regress"))
+            .args(&[
+                "--bindir=''",
+                "--use-existing",
+                format!("--bindir={}", self.env.pg_bin_dir().to_str().unwrap()).as_str(),
+                format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(),
+                format!(
+                    "--schedule={}",
+                    regress_src_path.join("parallel_schedule").to_str().unwrap()
+                )
+                .as_str(),
+                format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(),
+            ])
+            .env_clear()
+            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("PGPORT", self.address.port().to_string())
+            .env("PGUSER", self.whoami())
+            .env("PGHOST", self.address.ip().to_string())
+            .status()
+            .expect("pg_regress failed");
+        if !regress_check.success() {
+            if let Ok(mut file) = File::open("regression.diffs") {
+                let mut buffer = String::new();
+                file.read_to_string(&mut buffer).unwrap();
+                println!("--------------- regression.diffs:\n{}", buffer);
+            }
+            self.dump_log_file();
+        }
+        regress_check
+    }
+
+    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus {
+        let port = self.address.port().to_string();
+        let clients = clients.to_string();
+        let seconds = seconds.to_string();
+        let _pg_bench_init = Command::new(self.env.pg_bin_dir().join("pgbench"))
+            .args(&["-i", "-p", port.as_str(), "postgres"])
+            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .status()
+            .expect("pgbench -i");
+        let pg_bench_run = Command::new(self.env.pg_bin_dir().join("pgbench"))
+            .args(&[
+                "-p",
+                port.as_str(),
+                "-T",
+                seconds.as_str(),
+                "-P",
+                "1",
+                "-c",
+                clients.as_str(),
+                "-M",
+                "prepared",
+                "postgres",
+            ])
+            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .status()
+            .expect("pgbench run");
+        pg_bench_run
+    }
+
+    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode {
+        let proxy_path = self.env.pg_bin_dir().join("safekeeper_proxy");
+        match Command::new(proxy_path.as_path())
+            .args(&["--ztimelineid", &self.timelineid.to_string()])
+            .args(&["-s", wal_acceptors])
+            .args(&["-h", &self.address.ip().to_string()])
+            .args(&["-p", &self.address.port().to_string()])
+            .arg("-v")
+            .stderr(
+                OpenOptions::new()
+                    .create(true)
+                    .append(true)
+                    .open(self.pgdata().join("safekeeper_proxy.log"))
+                    .unwrap(),
+            )
+            .spawn()
+        {
+            Ok(child) => WalProposerNode { pid: child.id() },
+            Err(e) => panic!("Failed to launch {:?}: {}", proxy_path, e),
+        }
+    }
+
+    fn dump_log_file(&self) {
+        if let Ok(mut file) = File::open(self.env.pageserver_data_dir().join("pageserver.log")) {
+            let mut buffer = String::new();
+            file.read_to_string(&mut buffer).unwrap();
+            println!("--------------- pageserver.log:\n{}", buffer);
+        }
+    }
+
+    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row> {
+        let connstring = format!(
+            "host={} port={} dbname={} user={}",
+            self.address.ip(),
+            self.address.port(),
+            db,
+            self.whoami()
+        );
+        let mut client = postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap();
+
+        println!("Running {}", sql);
+        let result = client.query(sql, &[]);
+        if result.is_err() {
+            self.dump_log_file();
+        }
+        result.unwrap()
+    }
+
+    fn open_psql(&self, db: &str) -> postgres::Client {
+        let connstring = format!(
+            "host={} port={} dbname={} user={}",
+            self.address.ip(),
+            self.address.port(),
+            db,
+            self.whoami()
+        );
+        postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap()
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// WalAcceptorNode
+//
+///////////////////////////////////////////////////////////////////////////////
+
+//
+// Control routines for WalAcceptor.
+//
+// Now used only in test setups.
+//
+pub struct WalAcceptorNode {
+    listen: SocketAddr,
+    data_dir: PathBuf,
+    systemid: u64,
+    env: LocalEnv,
+    pass_to_pageserver: bool,
+}
+
+impl WalAcceptorNode {
+    pub fn init(&self) {
+        if self.data_dir.exists() {
+            fs::remove_dir_all(self.data_dir.clone()).unwrap();
+        }
+        fs::create_dir_all(self.data_dir.clone()).unwrap();
+    }
+
+    pub fn start(&self) {
+        println!(
+            "Starting wal_acceptor in {} listening '{}'",
+            self.data_dir.to_str().unwrap(),
+            self.listen
+        );
+
+        let ps_arg = if self.pass_to_pageserver {
+            // Tell page server it can receive WAL from this WAL safekeeper
+            ["--pageserver", "127.0.0.1:64000"].to_vec()
+        } else {
+            [].to_vec()
+        };
+
+        let status = Command::new(
+            self.env
+                .zenith_distrib_dir
+                .as_ref()
+                .unwrap()
+                .join("wal_acceptor"),
+        )
+        .args(&["-D", self.data_dir.to_str().unwrap()])
+        .args(&["-l", self.listen.to_string().as_str()])
+        .args(&["--systemid", self.systemid.to_string().as_str()])
+        .args(&ps_arg)
+        .arg("-d")
+        .arg("-n")
+        .status()
+        .expect("failed to start wal_acceptor");
+
+        if !status.success() {
+            panic!("wal_acceptor start failed");
+        }
+    }
+
+    pub fn stop(&self) -> Result<()> {
+        println!("Stopping wal acceptor on {}", self.listen);
+        let pidfile = self.data_dir.join("wal_acceptor.pid");
+        let pid = read_pidfile(&pidfile)?;
+        let pid = Pid::from_raw(pid);
+        if kill(pid, Signal::SIGTERM).is_err() {
+            bail!("Failed to kill wal_acceptor with pid {}", pid);
+        }
+        Ok(())
+    }
+}
+
+impl Drop for WalAcceptorNode {
+    fn drop(&mut self) {
+        // Ignore errors.
+        let _ = self.stop();
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// WalProposerNode
+//
+///////////////////////////////////////////////////////////////////////////////
+
+pub struct WalProposerNode {
+    pub pid: u32,
+}
+
+impl WalProposerNode {
+    pub fn stop(&self) {
+        // std::process::Child::id() returns u32, we need i32.
+        let pid: i32 = self.pid.try_into().unwrap();
+        let pid = Pid::from_raw(pid);
+        kill(pid, Signal::SIGTERM).expect("failed to execute kill");
+    }
+}
+
+impl Drop for WalProposerNode {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}

integration_tests/tests/test_wal_acceptor.rs

@@ -0,0 +1,330 @@
+use rand::Rng;
+use std::sync::Arc;
+use std::time::SystemTime;
+use std::{thread, time};
+
+use control_plane::compute::{ComputeControlPlane, PostgresNode};
+
+use integration_tests::PostgresNodeExt;
+use integration_tests::TestStorageControlPlane;
+
+const DOWNTIME: u64 = 2;
+
+fn start_node_with_wal_proposer(
+    timeline: &str,
+    compute_cplane: &mut ComputeControlPlane,
+    wal_acceptors: &str,
+) -> Arc<PostgresNode> {
+    let node = compute_cplane.new_test_master_node(timeline);
+    let _node = node.append_conf(
+        "postgresql.conf",
+        &format!("wal_acceptors='{}'\n", wal_acceptors),
+    );
+    node.start().unwrap();
+    node
+}
+
+#[test]
+fn test_embedded_wal_proposer() {
+    let local_env = integration_tests::create_test_env("test_embedded_wal_proposer");
+
+    const REDUNDANCY: usize = 3;
+    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // start postgres
+    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    node.safe_psql(
+        "postgres",
+        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
+    );
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 5000050000);
+    // check wal files equality
+}
+
+#[test]
+fn test_acceptors_normal_work() {
+    let local_env = integration_tests::create_test_env("test_acceptors_normal_work");
+
+    const REDUNDANCY: usize = 3;
+    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // start postgres
+    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    node.safe_psql(
+        "postgres",
+        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
+    );
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 5000050000);
+    // check wal files equality
+}
+
+// Run page server and multiple safekeepers, and multiple compute nodes running
+// against different timelines.
+#[test]
+fn test_many_timelines() {
+    // Initialize a new repository, and set up WAL safekeepers and page server.
+    const REDUNDANCY: usize = 3;
+    const N_TIMELINES: usize = 5;
+    let local_env = integration_tests::create_test_env("test_many_timelines");
+    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // Create branches
+    let mut timelines: Vec<String> = vec!["main".to_string()];
+
+    for i in 1..N_TIMELINES {
+        let branchname = format!("experimental{}", i);
+        storage_cplane
+            .pageserver
+            .branch_create(&branchname, "main")
+            .unwrap();
+        timelines.push(branchname);
+    }
+
+    // start postgres on each timeline
+    let mut nodes = Vec::new();
+    for tli_name in timelines {
+        let node = start_node_with_wal_proposer(&tli_name, &mut compute_cplane, &wal_acceptors);
+        nodes.push(node.clone());
+    }
+
+    // create schema
+    for node in &nodes {
+        node.safe_psql(
+            "postgres",
+            "CREATE TABLE t(key int primary key, value text)",
+        );
+    }
+
+    // Populate data
+    for node in &nodes {
+        node.safe_psql(
+            "postgres",
+            "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
+        );
+    }
+
+    // Check data
+    for node in &nodes {
+        let count: i64 = node
+            .safe_psql("postgres", "SELECT sum(key) FROM t")
+            .first()
+            .unwrap()
+            .get(0);
+        println!("sum = {}", count);
+        assert_eq!(count, 5000050000);
+    }
+}
+
+// Majority is always alive
+#[test]
+fn test_acceptors_restarts() {
+    let local_env = integration_tests::create_test_env("test_acceptors_restarts");
+
+    // Start pageserver that reads WAL directly from that postgres
+    const REDUNDANCY: usize = 3;
+    const FAULT_PROBABILITY: f32 = 0.01;
+
+    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+    let mut rng = rand::thread_rng();
+
+    // start postgres
+    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
+
+    let mut failed_node: Option<usize> = None;
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    let mut psql = node.open_psql("postgres");
+    for i in 1..=1000 {
+        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
+            .unwrap();
+        let prob: f32 = rng.gen();
+        if prob <= FAULT_PROBABILITY {
+            if let Some(node) = failed_node {
+                storage_cplane.wal_acceptors[node].start();
+                failed_node = None;
+            } else {
+                let node: usize = rng.gen_range(0..REDUNDANCY);
+                failed_node = Some(node);
+                storage_cplane.wal_acceptors[node].stop().unwrap();
+            }
+        }
+    }
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 500500);
+}
+
+fn start_acceptor(cplane: &Arc<TestStorageControlPlane>, no: usize) {
+    let cp = cplane.clone();
+    thread::spawn(move || {
+        thread::sleep(time::Duration::from_secs(DOWNTIME));
+        cp.wal_acceptors[no].start();
+    });
+}
+
+// Stop majority of acceptors while compute is under the load. Boot
+// them again and check that nothing was losed. Repeat.
+// N_CRASHES env var
+#[test]
+fn test_acceptors_unavailability() {
+    let local_env = integration_tests::create_test_env("test_acceptors_unavailability");
+
+    // Start pageserver that reads WAL directly from that postgres
+    const REDUNDANCY: usize = 2;
+
+    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // start postgres
+    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    let mut psql = node.open_psql("postgres");
+    psql.execute("INSERT INTO t values (1, 'payload')", &[])
+        .unwrap();
+
+    // Shut down all wal acceptors
+    storage_cplane.wal_acceptors[0].stop().unwrap();
+    let cp = Arc::new(storage_cplane);
+    start_acceptor(&cp, 0);
+    let now = SystemTime::now();
+    psql.execute("INSERT INTO t values (2, 'payload')", &[])
+        .unwrap();
+    // Here we check that the query above was hanging
+    // while wal_acceptor was unavailiable
+    assert!(now.elapsed().unwrap().as_secs() >= DOWNTIME);
+    psql.execute("INSERT INTO t values (3, 'payload')", &[])
+        .unwrap();
+
+    cp.wal_acceptors[1].stop().unwrap();
+    start_acceptor(&cp, 1);
+    psql.execute("INSERT INTO t values (4, 'payload')", &[])
+        .unwrap();
+    // Here we check that the query above was hanging
+    // while wal_acceptor was unavailiable
+    assert!(now.elapsed().unwrap().as_secs() >= 2 * DOWNTIME);
+
+    psql.execute("INSERT INTO t values (5, 'payload')", &[])
+        .unwrap();
+
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    // Ensure that all inserts succeeded.
+    // Including ones that were waiting for wal acceptor restart.
+    assert_eq!(count, 15);
+}
+
+fn simulate_failures(cplane: Arc<TestStorageControlPlane>) {
+    let mut rng = rand::thread_rng();
+    let n_acceptors = cplane.wal_acceptors.len();
+    let failure_period = time::Duration::from_secs(1);
+    while cplane.is_running() {
+        thread::sleep(failure_period);
+        let mask: u32 = rng.gen_range(0..(1 << n_acceptors));
+        for i in 0..n_acceptors {
+            if (mask & (1 << i)) != 0 {
+                cplane.wal_acceptors[i].stop().unwrap();
+            }
+        }
+        thread::sleep(failure_period);
+        for i in 0..n_acceptors {
+            if (mask & (1 << i)) != 0 {
+                cplane.wal_acceptors[i].start();
+            }
+        }
+    }
+}
+
+// Race condition test
+#[test]
+fn test_race_conditions() {
+    let local_env = integration_tests::create_test_env("test_race_conditions");
+
+    // Start pageserver that reads WAL directly from that postgres
+    const REDUNDANCY: usize = 3;
+
+    let storage_cplane = Arc::new(TestStorageControlPlane::fault_tolerant(
+        &local_env, REDUNDANCY,
+    ));
+    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // start postgres
+    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+
+    let cp = storage_cplane.clone();
+    let failures_thread = thread::spawn(move || {
+        simulate_failures(cp);
+    });
+
+    let mut psql = node.open_psql("postgres");
+    for i in 1..=1000 {
+        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
+            .unwrap();
+    }
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 500500);
+
+    storage_cplane.stop();
+    failures_thread.join().unwrap();
+}

pageserver/Cargo.toml

@@ -10,7 +10,7 @@ edition = "2018"
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
-bytes = { version = "1.0.1", features = ['serde'] }
+bytes = "1.0.1"
 byteorder = "1.4.3"
 futures = "0.3.13"
 lazy_static = "1.4.0"
@@ -30,8 +30,7 @@ tokio-stream = { version = "0.1.4" }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
-rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
+rocksdb = "0.16.0"
 anyhow = "1.0"
 crc32c = "0.6.0"
 walkdir = "2"
@@ -42,7 +41,6 @@ parse_duration = "2.1.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 fs_extra = "1.2.0"
-toml = "0.5"
 
 postgres_ffi = { path = "../postgres_ffi" }
 zenith_utils = { path = "../zenith_utils" }

pageserver/README

@@ -1,4 +1,82 @@
-## Page server architecture
+Page Server
+===========
+
+
+How to test
+-----------
+
+
+1. Compile and install Postgres from this repository (there are
+   modifications, so vanilla Postgres won't do)
+
+    ./configure --prefix=/home/heikki/zenith-install
+
+2. Compile the page server
+
+    cd pageserver
+    cargo build
+
+3. Create another "dummy" cluster that will be used by the page server when it applies
+   the WAL records. (shouldn't really need this, getting rid of it is a TODO):
+
+    /home/heikki/zenith-install/bin/initdb -D /data/zenith-dummy
+
+
+4. Initialize and start a new postgres cluster
+
+    /home/heikki/zenith-install/bin/initdb -D /data/zenith-test-db --username=postgres
+    /home/heikki/zenith-install/bin/postgres -D /data/zenith-test-db
+
+5. In another terminal, start the page server.
+
+    PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
+
+   It should connect to the postgres instance using streaming replication, and print something
+   like this:
+
+    $ PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
+    Starting WAL receiver
+    connecting...
+    Starting page server on 127.0.0.1:5430
+    connected!
+    page cache is empty
+
+6. You can now open another terminal and issue DDL commands. Generated WAL records will
+   be streamed to the page servers, and attached to blocks that they apply to in its
+   page cache
+
+    $ psql postgres -U postgres
+    psql (14devel)
+    Type "help" for help.
+    
+    postgres=# create table mydata (i int4);
+    CREATE TABLE
+    postgres=# insert into mydata select g from generate_series(1,100) g;
+    INSERT 0 100
+    postgres=# 
+
+7. The GetPage@LSN interface to the compute nodes isn't working yet, but to simulate
+   that, the page server generates a test GetPage@LSN call every 5 seconds on a random
+   block that's in the page cache. In a few seconds, you should see output from that:
+
+    testing GetPage@LSN for block 0
+    WAL record at LSN 23584576 initializes the page
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF40
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF80
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DFC0
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E018
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E058
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E098
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E0D8
+    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E118
+    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E158
+    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E198
+    applied 10 WAL records to produce page image at LSN 18446744073709547246
+
+
+
+Architecture
+============
 
 The Page Server is responsible for all operations on a number of
 "chunks" of relation data. A chunk corresponds to a PostgreSQL
@@ -6,10 +84,8 @@ relation segment (i.e. one max. 1 GB file in the data directory), but
 it holds all the different versions of every page in the segment that
 are still needed by the system.
 
-Currently we do not specifically organize data in chunks.
-All page images and corresponding WAL records are stored as entries in a key-value storage,
-where StorageKey is a zenith_timeline_id + BufferTag + LSN.
-
+Determining which chunk each Page Server holds is handled elsewhere. (TODO:
+currently, there is only one Page Server which holds all chunks)
 
 The Page Server has a few different duties:
 
@@ -78,33 +154,11 @@ and stores them to the page cache.
 Page Cache
 ----------
 
-The Page Cache is a switchboard to access different Repositories.
+The Page Cache is a data structure, to hold all the different page versions.
+It is accessed by all the other threads, to perform their duties.
 
-#### Repository
-Repository corresponds to one .zenith directory.
-Repository is needed to manage Timelines.
-
-#### Timeline
-Timeline is a page cache workhorse that accepts page changes
-and serves get_page_at_lsn() and get_rel_size() requests.
-Note: this has nothing to do with PostgreSQL WAL timeline.
-
-#### Branch
-We can create branch at certain LSN.
-Each Branch lives in a corresponding timeline and has an ancestor.
-
-To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
-
-#### ObjectRepository
-ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
-
-#### ObjectStore
-ObjectStore is an interface for key-value store for page images and wal records.
-Currently it has one implementation - RocksDB.
-
-#### WAL redo service
-WAL redo service - service that runs PostgreSQL in a special wal_redo mode
-to apply given WAL records over an old page image and return new page image.
+Currently, the page cache is implemented fully in-memory. TODO: Store it
+on disk. Define a file format.
 
 
 TODO: Garbage Collection / Compaction
@@ -123,7 +177,3 @@ The backup service is responsible for periodically pushing the chunks to S3.
 TODO: How/when do restore from S3? Whenever we get a GetPage@LSN request for
 a chunk we don't currently have? Or when an external Control Plane tells us?
 
-TODO: Sharding
---------------------
-
-We should be able to run multiple Page Servers that handle sharded data.

pageserver/launch.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+#
+# Set up a simple Compute Node + Page Server combination locally.
+#
+# NOTE: This doesn't clean up between invocations. You'll need to manually:
+#
+# - Kill any previous 'postgres' and 'pageserver' processes
+# - Clear the S3 bucket
+# - Remove the 'zenith-pgdata' directory
+
+
+set -e
+
+# Set up some config.
+#
+# CHANGE THESE ACCORDING TO YOUR S3 INSTALLATION
+export S3_REGION=auto
+export S3_ENDPOINT=https://localhost:9000
+export S3_ACCESSKEY=minioadmin
+export S3_SECRET=pikkunen
+export S3_BUCKET=zenith-testbucket
+
+
+COMPUTE_NODE_PGDATA=zenith-pgdata
+
+
+# 1. Initialize a cluster.
+initdb -D $COMPUTE_NODE_PGDATA -U zenith
+
+echo "port=65432" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+echo "log_connections=on" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+# Use a small shared_buffers, so that we hit the Page Server more
+# easily.
+echo "shared_buffers = 1MB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+# TODO: page server should use a replication slot, or some other mechanism
+# to make sure that the primary doesn't lose data that the page server still
+# needs. (The WAL safekeepers should ensure that)
+echo "wal_keep_size=10GB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+# Tell the Postgres server how to connect to the Page Server
+echo "page_server_connstring='host=localhost port=5430'" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+
+# 2. Run zenith_push to push a base backup fo the database to an S3 bucket. The
+# Page Server will read it from there
+zenith_push -D $COMPUTE_NODE_PGDATA
+
+
+# 3. Launch page server
+rm -rf /tmp/pgdata-dummy
+initdb -N -D /tmp/pgdata-dummy
+PGDATA=/tmp/pgdata-dummy ./target/debug/pageserver  &
+
+# 4. Start up the Postgres server
+postgres -D $COMPUTE_NODE_PGDATA &
+
+
+echo "ALL SET! You can now connect to Postgres with something like:"
+echo ""
+echo 'psql "dbname=postgres host=localhost user=zenith port=65432"'

pageserver/src/basebackup.rs

@@ -1,9 +1,3 @@
-//!
-//! Generate a tarball with files needed to bootstrap ComputeNode.
-//!
-//! TODO: this module has nothing to do with PostgreSQL pg_basebackup.
-//! It could use a better name.
-//!
 use crate::ZTimelineId;
 use log::*;
 use std::io::Write;

pageserver/src/bin/pageserver.rs

@@ -3,107 +3,29 @@
 //
 
 use log::*;
-use serde::{Deserialize, Serialize};
+use parse_duration::parse;
+use std::io;
+use std::process::exit;
+use std::thread;
+use std::time::Duration;
+use std::{env, path::PathBuf};
 use std::{
-    env,
     fs::{File, OpenOptions},
-    io,
-    net::{SocketAddr, TcpListener},
-    path::{Path, PathBuf},
-    process::exit,
-    thread,
-    time::Duration,
+    net::TcpListener,
 };
 
 use anyhow::{Context, Result};
-use clap::{App, Arg, ArgMatches};
+use clap::{App, Arg};
 use daemonize::Daemonize;
 
 use slog::{Drain, FnValue};
 
 use pageserver::{branches, page_cache, page_service, tui, PageServerConf};
 
-const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
-
 const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(10);
-
-/// String arguments that can be declared via CLI or config file
-#[derive(Serialize, Deserialize)]
-struct CfgFileParams {
-    listen_addr: Option<String>,
-    gc_horizon: Option<String>,
-    gc_period: Option<String>,
-    pg_distrib_dir: Option<String>,
-}
-
-impl CfgFileParams {
-    /// Extract string arguments from CLI
-    fn from_args(arg_matches: &ArgMatches) -> Self {
-        let get_arg = |arg_name: &str| -> Option<String> {
-            arg_matches.value_of(arg_name).map(str::to_owned)
-        };
-
-        Self {
-            listen_addr: get_arg("listen"),
-            gc_horizon: get_arg("gc_horizon"),
-            gc_period: get_arg("gc_period"),
-            pg_distrib_dir: get_arg("postgres-distrib"),
-        }
-    }
-
-    /// Fill missing values in `self` with `other`
-    fn or(self, other: CfgFileParams) -> Self {
-        // TODO cleaner way to do this
-        Self {
-            listen_addr: self.listen_addr.or(other.listen_addr),
-            gc_horizon: self.gc_horizon.or(other.gc_horizon),
-            gc_period: self.gc_period.or(other.gc_period),
-            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
-        }
-    }
-
-    /// Create a PageServerConf from these string parameters
-    fn try_into_config(&self) -> Result<PageServerConf> {
-        let listen_addr: SocketAddr = self
-            .listen_addr
-            .as_deref()
-            .unwrap_or(DEFAULT_LISTEN_ADDR)
-            .parse()?;
-
-        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
-            Some(horizon_str) => horizon_str.parse()?,
-            None => DEFAULT_GC_HORIZON,
-        };
-
-        let gc_period: Duration = match self.gc_period.as_ref() {
-            Some(period_str) => parse_duration::parse(period_str)?,
-            None => DEFAULT_GC_PERIOD,
-        };
-
-        let pg_distrib_dir = match self.pg_distrib_dir.as_ref() {
-            Some(pg_distrib_dir_str) => PathBuf::from(pg_distrib_dir_str),
-            None => env::current_dir()?.join("tmp_install"),
-        };
-
-        if !pg_distrib_dir.join("bin/postgres").exists() {
-            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
-        }
-
-        Ok(PageServerConf {
-            daemonize: false,
-            interactive: false,
-
-            listen_addr,
-            gc_horizon,
-            gc_period,
-
-            workdir: PathBuf::from("."),
-
-            pg_distrib_dir,
-        })
-    }
-}
+const DEFAULT_GC_PERIOD_SEC: u64 = 10;
+//const DEFAULT_GC_HORIZON: u64 = 1024 * 1024 * 1024;
+//const DEFAULT_GC_PERIOD_SEC: u64 = 600;
 
 fn main() -> Result<()> {
     let arg_matches = App::new("Zenith page server")
@@ -154,39 +76,47 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Working directory for the pageserver"),
         )
-        .arg(
-            Arg::with_name("postgres-distrib")
-                .long("postgres-distrib")
-                .takes_value(true)
-                .help("Postgres distribution directory"),
-        )
         .get_matches();
 
-    let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
-    let cfg_file_path = workdir.canonicalize()?.join("pageserver.toml");
-
-    let args_params = CfgFileParams::from_args(&arg_matches);
-
-    let init = arg_matches.is_present("init");
-    let params = if init {
-        // We're initializing the repo, so there's no config file yet
-        args_params
+    let workdir = if let Some(workdir_arg) = arg_matches.value_of("workdir") {
+        PathBuf::from(workdir_arg)
+    } else if let Some(workdir_arg) = std::env::var_os("ZENITH_REPO_DIR") {
+        PathBuf::from(workdir_arg.to_str().unwrap())
     } else {
-        // Supplement the CLI arguments with the config file
-        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)?;
-        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents)?;
-        args_params.or(file_params)
+        PathBuf::from(".zenith")
     };
 
-    // Ensure the config is valid, even if just init-ing
-    let mut conf = params.try_into_config()?;
+    let pg_distrib_dir: PathBuf = {
+        if let Some(postgres_bin) = env::var_os("POSTGRES_DISTRIB_DIR") {
+            postgres_bin.into()
+        } else {
+            let cwd = env::current_dir()?;
+            cwd.join("tmp_install")
+        }
+    };
 
-    conf.daemonize = arg_matches.is_present("daemonize");
-    conf.interactive = arg_matches.is_present("interactive");
+    if !pg_distrib_dir.join("bin/postgres").exists() {
+        anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
+    }
 
-    if init && (conf.daemonize || conf.interactive) {
-        eprintln!("--daemonize and --interactive may not be used with --init");
-        exit(1);
+    let mut conf = PageServerConf {
+        daemonize: false,
+        interactive: false,
+        gc_horizon: DEFAULT_GC_HORIZON,
+        gc_period: Duration::from_secs(DEFAULT_GC_PERIOD_SEC),
+        listen_addr: "127.0.0.1:64000".parse().unwrap(),
+        // we will change the current working directory to the repository below,
+        // so always set 'workdir' to '.'
+        workdir: PathBuf::from("."),
+        pg_distrib_dir,
+    };
+
+    if arg_matches.is_present("daemonize") {
+        conf.daemonize = true;
+    }
+
+    if arg_matches.is_present("interactive") {
+        conf.interactive = true;
     }
 
     if conf.daemonize && conf.interactive {
@@ -194,19 +124,26 @@ fn main() -> Result<()> {
         exit(1);
     }
 
+    if let Some(addr) = arg_matches.value_of("listen") {
+        conf.listen_addr = addr.parse()?;
+    }
+
+    if let Some(horizon) = arg_matches.value_of("gc_horizon") {
+        conf.gc_horizon = horizon.parse()?;
+    }
+
+    if let Some(period) = arg_matches.value_of("gc_period") {
+        conf.gc_period = parse(period)?;
+    }
+
     // The configuration is all set up now. Turn it into a 'static
     // that can be freely stored in structs and passed across threads
     // as a ref.
     let conf: &'static PageServerConf = Box::leak(Box::new(conf));
 
     // Create repo and exit if init was requested
-    if init {
+    if arg_matches.is_present("init") {
         branches::init_repo(conf, &workdir)?;
-
-        // write the config file
-        let cfg_file_contents = toml::to_string_pretty(&params)?;
-        std::fs::write(&cfg_file_path, cfg_file_contents)?;
-
         return Ok(());
     }
 

pageserver/src/branches.rs

@@ -1,12 +1,13 @@
-//!
-//! Branch management code
-//!
+//
+// Branch management code
+//
 // TODO: move all paths construction to conf impl
 //
 
 use anyhow::{anyhow, bail, Context, Result};
+use bytes::Bytes;
 use fs::File;
-use postgres_ffi::{pg_constants, xlog_utils, ControlFileData};
+use postgres_ffi::{pg_constants, xlog_utils};
 use rand::Rng;
 use serde::{Deserialize, Serialize};
 use std::env;
@@ -73,16 +74,13 @@ pub fn init_repo(conf: &'static PageServerConf, repo_dir: &Path) -> Result<()> {
         .output()
         .with_context(|| "failed to execute initdb")?;
     if !initdb_otput.status.success() {
-        anyhow::bail!(
-            "initdb failed: '{}'",
-            String::from_utf8_lossy(&initdb_otput.stderr)
-        );
+        anyhow::bail!("initdb failed");
     }
     println!("initdb succeeded");
 
     // Read control file to extract the LSN and system id
     let controlfile_path = tmppath.join("global").join("pg_control");
-    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
+    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
     // let systemid = controlfile.system_identifier;
     let lsn = controlfile.checkPoint;
     let lsnstr = format!("{:016X}", lsn);
@@ -98,13 +96,10 @@ pub fn init_repo(conf: &'static PageServerConf, repo_dir: &Path) -> Result<()> {
     // and it kept running even after the "zenith init" had exited. In tests, we started the
     // page server immediately after that, so that initdb was still running in the background,
     // and we failed to run initdb again in the same directory. This has been solved for the
-    // rapid init+start case now, but the general race condition remains if you restart the
+    // rapid init+start case now, but the general race condition remains if you restart the the
     // server quickly.
-    let storage = crate::rocksdb_storage::RocksObjectStore::create(conf)?;
-
-    let repo = crate::object_repository::ObjectRepository::new(
+    let repo = crate::repository::rocksdb::RocksRepository::new(
         conf,
-        std::sync::Arc::new(storage),
         std::sync::Arc::new(crate::walredo::DummyRedoManager {}),
     );
     let timeline = repo.create_empty_timeline(tli, Lsn(lsn))?;
@@ -144,9 +139,10 @@ pub fn init_repo(conf: &'static PageServerConf, repo_dir: &Path) -> Result<()> {
     Ok(())
 }
 
-pub(crate) fn get_branches(conf: &PageServerConf) -> Result<Vec<BranchInfo>> {
-    let repo = page_cache::get_repository();
-
+pub(crate) fn get_branches(
+    conf: &PageServerConf,
+    repository: &dyn Repository,
+) -> Result<Vec<BranchInfo>> {
     // Each branch has a corresponding record (text file) in the refs/branches
     // with timeline_id.
     let branches_dir = std::path::Path::new("refs").join("branches");
@@ -157,7 +153,7 @@ pub(crate) fn get_branches(conf: &PageServerConf) -> Result<Vec<BranchInfo>> {
             let name = dir_entry.file_name().to_str().unwrap().to_string();
             let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
 
-            let latest_valid_lsn = repo
+            let latest_valid_lsn = repository
                 .get_timeline(timeline_id)
                 .map(|timeline| timeline.get_last_valid_lsn())
                 .ok();
@@ -214,7 +210,7 @@ pub(crate) fn get_system_id(conf: &PageServerConf) -> Result<u64> {
 
     let (_, main_snap_dir) = find_latest_snapshot(conf, *main_tli)?;
     let controlfile_path = main_snap_dir.join("global").join("pg_control");
-    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
+    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
     Ok(controlfile.system_identifier)
 }
 
@@ -223,8 +219,6 @@ pub(crate) fn create_branch(
     branchname: &str,
     startpoint_str: &str,
 ) -> Result<BranchInfo> {
-    let repo = page_cache::get_repository();
-
     if conf.branch_path(&branchname).exists() {
         anyhow::bail!("branch {} already exists", branchname);
     }
@@ -233,9 +227,7 @@ pub(crate) fn create_branch(
 
     if startpoint.lsn == Lsn(0) {
         // Find end of WAL on the old timeline
-        let end_of_wal = repo
-            .get_timeline(startpoint.timelineid)?
-            .get_last_record_lsn();
+        let end_of_wal = find_end_of_wal(conf, startpoint.timelineid)?;
         println!("branching at end of WAL: {}", end_of_wal);
         startpoint.lsn = end_of_wal;
     }
@@ -245,6 +237,7 @@ pub(crate) fn create_branch(
     let newtimelinedir = conf.timeline_path(newtli);
 
     // Let the Repository backend do its initialization
+    let repo = page_cache::get_repository();
     repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
 
     // Copy the latest snapshot (TODO: before the startpoint) and all WAL
@@ -357,11 +350,15 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
 fn force_crash_recovery(datadir: &Path) -> Result<()> {
     // Read in the control file
     let controlfilepath = datadir.to_path_buf().join("global").join("pg_control");
-    let mut controlfile = ControlFileData::decode(&fs::read(controlfilepath.as_path())?)?;
+    let mut controlfile =
+        postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfilepath.as_path())?))?;
 
     controlfile.state = postgres_ffi::DBState_DB_IN_PRODUCTION;
 
-    fs::write(controlfilepath.as_path(), controlfile.encode())?;
+    fs::write(
+        controlfilepath.as_path(),
+        postgres_ffi::encode_pg_control(controlfile),
+    )?;
 
     Ok(())
 }
@@ -433,6 +430,13 @@ fn copy_wal(src_dir: &Path, dst_dir: &Path, upto: Lsn, wal_seg_size: usize) -> R
     Ok(())
 }
 
+// Find the end of valid WAL in a wal directory
+pub fn find_end_of_wal(conf: &PageServerConf, timeline: ZTimelineId) -> Result<Lsn> {
+    let waldir = conf.timeline_path(timeline).join("wal");
+    let (lsn, _tli) = xlog_utils::find_end_of_wal(&waldir, pg_constants::WAL_SEGMENT_SIZE, true);
+    Ok(Lsn(lsn))
+}
+
 // Find the latest snapshot for a timeline
 fn find_latest_snapshot(conf: &PageServerConf, timeline: ZTimelineId) -> Result<(Lsn, PathBuf)> {
     let snapshotsdir = conf.snapshots_path(timeline);

pageserver/src/lib.rs

@@ -8,13 +8,10 @@ use std::time::Duration;
 
 pub mod basebackup;
 pub mod branches;
-pub mod object_repository;
-pub mod object_store;
 pub mod page_cache;
 pub mod page_service;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod rocksdb_storage;
 pub mod tui;
 pub mod tui_event;
 mod tui_logger;
@@ -112,9 +109,6 @@ impl FromStr for ZTimelineId {
     fn from_str(s: &str) -> Result<ZTimelineId, Self::Err> {
         let timelineid = hex::decode(s)?;
 
-	if timelineid.len() != 16 {
-	    return Err(hex::FromHexError::InvalidStringLength);
-	}
         let mut buf: [u8; 16] = [0u8; 16];
         buf.copy_from_slice(timelineid.as_slice());
         Ok(ZTimelineId(buf))

pageserver/src/object_store.rs

@@ -1,77 +0,0 @@
-//! Low-level key-value storage abstraction.
-//!
-use crate::repository::{BufferTag, RelTag};
-use crate::ZTimelineId;
-use anyhow::Result;
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::iter::Iterator;
-use zenith_utils::lsn::Lsn;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ObjectKey {
-    pub timeline: ZTimelineId,
-    pub buf_tag: BufferTag,
-}
-
-///
-/// Low-level storage abstraction.
-///
-/// All the data in the repository is stored in a key-value store. This trait
-/// abstracts the details of the key-value store.
-///
-/// A simple key-value store would support just GET and PUT operations with
-/// a key, but the upper layer needs slightly complicated read operations
-///
-/// The most frequently used function is 'object_versions'. It is used
-/// to look up a page version. It is LSN aware, in that the caller
-/// specifies an LSN, and the function returns all values for that
-/// block with the same or older LSN.
-///
-pub trait ObjectStore: Send + Sync {
-    ///
-    /// Store a value with given key.
-    ///
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
-
-    /// Read entry with the exact given key.
-    ///
-    /// This is used for retrieving metadata with special key that doesn't
-    /// correspond to any real relation.
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
-
-    /// Iterate through all page versions of one object.
-    ///
-    /// Returns all page versions in descending LSN order, along with the LSN
-    /// of each page version.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(BufferTag, Lsn, Vec<u8>)>> + 'a>>;
-
-    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
-    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
-    ///
-    /// This is used to implement 'create database'
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>>;
-
-    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
-}

pageserver/src/page_cache.rs

@@ -3,33 +3,30 @@
 //! isn't much here. If we implement multi-tenancy, this will probably be changed into
 //! a hash map, keyed by the tenant ID.
 
-use crate::object_repository::ObjectRepository;
+use crate::repository::rocksdb::RocksRepository;
 use crate::repository::Repository;
-use crate::rocksdb_storage::RocksObjectStore;
 use crate::walredo::PostgresRedoManager;
 use crate::PageServerConf;
 use lazy_static::lazy_static;
 use std::sync::{Arc, Mutex};
 
 lazy_static! {
-    pub static ref REPOSITORY: Mutex<Option<Arc<dyn Repository>>> = Mutex::new(None);
+    pub static ref REPOSITORY: Mutex<Option<Arc<dyn Repository + Send + Sync>>> = Mutex::new(None);
 }
 
 pub fn init(conf: &'static PageServerConf) {
     let mut m = REPOSITORY.lock().unwrap();
 
-    let obj_store = RocksObjectStore::open(conf).unwrap();
-
     // Set up a WAL redo manager, for applying WAL records.
     let walredo_mgr = PostgresRedoManager::new(conf);
 
     // we have already changed current dir to the repository.
-    let repo = ObjectRepository::new(conf, Arc::new(obj_store), Arc::new(walredo_mgr));
+    let repo = RocksRepository::new(conf, Arc::new(walredo_mgr));
 
     *m = Some(Arc::new(repo));
 }
 
-pub fn get_repository() -> Arc<dyn Repository> {
+pub fn get_repository() -> Arc<dyn Repository + Send + Sync> {
     let o = &REPOSITORY.lock().unwrap();
     Arc::clone(o.as_ref().unwrap())
 }

pageserver/src/repository.rs

@@ -1,9 +1,13 @@
+pub mod rocksdb;
+
+use crate::waldecoder::{DecodedWALRecord, Oid, XlCreateDatabase, XlSmgrTruncate};
 use crate::ZTimelineId;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use log::*;
+use postgres_ffi::pg_constants;
 use postgres_ffi::relfile_utils::forknumber_to_name;
 use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
 use std::fmt;
 use std::sync::Arc;
 use zenith_utils::lsn::Lsn;
@@ -11,7 +15,7 @@ use zenith_utils::lsn::Lsn;
 ///
 /// A repository corresponds to one .zenith directory. One repository holds multiple
 /// timelines, forked off from the same initial call to 'initdb'.
-pub trait Repository: Send + Sync {
+pub trait Repository {
     /// Get Timeline handle for given zenith timeline ID.
     fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
 
@@ -28,7 +32,7 @@ pub trait Repository: Send + Sync {
     //fn get_stats(&self) -> RepositoryStats;
 }
 
-pub trait Timeline: Send + Sync {
+pub trait Timeline {
     //------------------------------------------------------------------------------
     // Public GET functions
     //------------------------------------------------------------------------------
@@ -37,13 +41,10 @@ pub trait Timeline: Send + Sync {
     fn get_page_at_lsn(&self, tag: BufferTag, lsn: Lsn) -> Result<Bytes>;
 
     /// Get size of relation
-    fn get_rel_size(&self, tag: RelTag, lsn: Lsn) -> Result<u32>;
+    fn get_relsize(&self, tag: RelTag, lsn: Lsn) -> Result<u32>;
 
     /// Does relation exist?
-    fn get_rel_exists(&self, tag: RelTag, lsn: Lsn) -> Result<bool>;
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>>;
+    fn get_relsize_exists(&self, tag: RelTag, lsn: Lsn) -> Result<bool>;
 
     //------------------------------------------------------------------------------
     // Public PUT functions, to update the repository with new page versions.
@@ -63,6 +64,116 @@ pub trait Timeline: Send + Sync {
     /// Truncate relation
     fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()>;
 
+    /// Drop relation or file segment
+    fn put_drop(&self, tag: BufferTag, lsn: Lsn) -> Result<()>;
+
+    /// Create a new database from a template database
+    ///
+    /// In PostgreSQL, CREATE DATABASE works by scanning the data directory and
+    /// copying all relation files from the template database. This is the equivalent
+    /// of that.
+    fn put_create_database(
+        &self,
+        lsn: Lsn,
+        db_id: Oid,
+        tablespace_id: Oid,
+        src_db_id: Oid,
+        src_tablespace_id: Oid,
+    ) -> Result<()>;
+
+    ///
+    /// Helper function to parse a WAL record and call the above functions for all the
+    /// relations/pages that the record affects.
+    ///
+    fn save_decoded_record(
+        &self,
+        decoded: DecodedWALRecord,
+        recdata: Bytes,
+        lsn: Lsn,
+    ) -> Result<()> {
+        // Figure out which blocks the record applies to, and "put" a separate copy
+        // of the record for each block.
+        for blk in decoded.blocks.iter() {
+            let tag = BufferTag {
+                rel: RelTag {
+                    spcnode: blk.rnode_spcnode,
+                    dbnode: blk.rnode_dbnode,
+                    relnode: blk.rnode_relnode,
+                    forknum: blk.forknum as u8,
+                },
+                blknum: blk.blkno,
+            };
+
+            if blk.will_drop {
+                self.put_drop(tag, lsn)?;
+            } else {
+                let rec = WALRecord {
+                    lsn,
+                    will_init: blk.will_init || blk.apply_image,
+                    rec: recdata.clone(),
+                    main_data_offset: decoded.main_data_offset as u32,
+                };
+
+                self.put_wal_record(tag, rec)?;
+            }
+        }
+
+        // Handle a few special record types
+        if decoded.xl_rmid == pg_constants::RM_SMGR_ID
+            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
+                == pg_constants::XLOG_SMGR_TRUNCATE
+        {
+            let truncate = XlSmgrTruncate::decode(&decoded);
+            let mut rel = RelTag {
+                spcnode: truncate.rnode.spcnode,
+                dbnode: truncate.rnode.dbnode,
+                relnode: truncate.rnode.relnode,
+                forknum: pg_constants::MAIN_FORKNUM,
+            };
+            if (truncate.flags & pg_constants::SMGR_TRUNCATE_HEAP) != 0 {
+                self.put_truncation(rel, lsn, truncate.blkno)?;
+            }
+            if (truncate.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 {
+                if truncate.blkno == 0 {
+                    rel.forknum = pg_constants::FSM_FORKNUM;
+                    self.put_truncation(rel, lsn, truncate.blkno)?;
+                } else {
+                    // TODO: handle partial truncation of FSM:
+                    // need to map heap block number to FSM block number
+                    // and clear bits in the tail block
+                    info!("Partial truncation of FSM is not supported");
+                }
+            }
+            if (truncate.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
+                if truncate.blkno == 0 {
+                    rel.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
+                    self.put_truncation(rel, lsn, truncate.blkno)?;
+                } else {
+                    // TODO: handle partial truncation of VM:
+                    // need to map heap block number to VM block number
+                    // and clear bits in the tail block
+                    info!("Partial truncation of VM is not supported");
+                }
+            }
+        } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID
+            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
+                == pg_constants::XLOG_DBASE_CREATE
+        {
+            let createdb = XlCreateDatabase::decode(&decoded);
+            self.put_create_database(
+                lsn,
+                createdb.db_id,
+                createdb.tablespace_id,
+                createdb.src_db_id,
+                createdb.src_tablespace_id,
+            )?;
+        }
+        // Now that this record has been handled, let the repository know that
+        // it is up-to-date to this LSN
+        self.advance_last_record_lsn(lsn);
+        Ok(())
+    }
+
     /// Remember the all WAL before the given LSN has been processed.
     ///
     /// The WAL receiver calls this after the put_* functions, to indicate that
@@ -86,34 +197,6 @@ pub trait Timeline: Send + Sync {
     /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
     /// know anything about them here in the repository.
     fn checkpoint(&self) -> Result<()>;
-
-    /// Events for all relations in the timeline.
-    /// Contains updates from start up to the last valid LSN
-    /// at time of history() call. This lsn can be read via the lsn() function.
-    ///
-    /// Relation size is increased implicitly and decreased with Truncate updates.
-    // TODO ordering guarantee?
-    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>>;
-}
-
-pub trait History: Iterator<Item = Result<RelationUpdate>> {
-    /// The last_valid_lsn at the time of history() call.
-    fn lsn(&self) -> Lsn;
-}
-
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct RelationUpdate {
-    pub rel: RelTag,
-    pub lsn: Lsn,
-    pub update: Update,
-}
-
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub enum Update {
-    Page { blknum: u32, img: Bytes },
-    WALRecord { blknum: u32, rec: WALRecord },
-    Truncate { n_blocks: u32 },
-    Unlink,
 }
 
 #[derive(Clone)]
@@ -151,15 +234,6 @@ pub struct RelTag {
     pub relnode: u32,
 }
 
-impl RelTag {
-    pub const ZEROED: Self = Self {
-        forknum: 0,
-        spcnode: 0,
-        dbnode: 0,
-        relnode: 0,
-    };
-}
-
 /// Display RelTag in the same format that's used in most PostgreSQL debug messages:
 ///
 /// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
@@ -191,14 +265,7 @@ pub struct BufferTag {
     pub blknum: u32,
 }
 
-impl BufferTag {
-    pub const ZEROED: Self = Self {
-        rel: RelTag::ZEROED,
-        blknum: 0,
-    };
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone)]
 pub struct WALRecord {
     pub lsn: Lsn, // LSN at the *end* of the record
     pub will_init: bool,
@@ -238,8 +305,6 @@ impl WALRecord {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::object_repository::ObjectRepository;
-    use crate::rocksdb_storage::RocksObjectStore;
     use crate::walredo::{WalRedoError, WalRedoManager};
     use crate::PageServerConf;
     use postgres_ffi::pg_constants;
@@ -294,11 +359,9 @@ mod tests {
         // OK in a test.
         let conf: &'static PageServerConf = Box::leak(Box::new(conf));
 
-        let obj_store = RocksObjectStore::create(conf)?;
-
         let walredo_mgr = TestRedoManager {};
 
-        let repo = ObjectRepository::new(conf, Arc::new(obj_store), Arc::new(walredo_mgr));
+        let repo = rocksdb::RocksRepository::new(conf, Arc::new(walredo_mgr));
 
         Ok(Box::new(repo))
     }
@@ -323,13 +386,16 @@ mod tests {
 
         tline.advance_last_valid_lsn(Lsn(5));
 
-        // The relation was created at LSN 2, not visible at LSN 1 yet.
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(1))?, false);
-        assert!(tline.get_rel_size(TESTREL_A, Lsn(1)).is_err());
+        // FIXME: The rocksdb implementation erroneously returns 'true' here, even
+        // though the relation was created only at a later LSN
+        // rocksdb implementation erroneosly returns 'true' here
+        assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(1))?, true); // CORRECT: false
+                                                                        // And this probably should throw an error, becaue the relation doesn't exist at Lsn(1) yet
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(1))?, 0); // CORRECT: throw error
 
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(2))?, 1);
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(5))?, 3);
+        assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(2))?, true);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(2))?, 1);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 3);
 
         // Check page contents at each LSN
         assert_eq!(
@@ -369,7 +435,7 @@ mod tests {
         tline.advance_last_valid_lsn(Lsn(6));
 
         // Check reported size and contents after truncation
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(6))?, 2);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(6))?, 2);
         assert_eq!(
             tline.get_page_at_lsn(TEST_BUF(0), Lsn(6))?,
             TEST_IMG("foo blk 0 at 3")
@@ -380,7 +446,7 @@ mod tests {
         );
 
         // should still see the truncated block with older LSN
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(5))?, 3);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 3);
         assert_eq!(
             tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
             TEST_IMG("foo blk 2 at 5")
@@ -411,7 +477,7 @@ mod tests {
         tline.advance_last_valid_lsn(Lsn(lsn));
 
         assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
             pg_constants::RELSEG_SIZE + 1
         );
 
@@ -420,7 +486,7 @@ mod tests {
         tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?;
         tline.advance_last_valid_lsn(Lsn(lsn));
         assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
             pg_constants::RELSEG_SIZE
         );
 
@@ -429,64 +495,13 @@ mod tests {
         tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?;
         tline.advance_last_valid_lsn(Lsn(lsn));
         assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
             pg_constants::RELSEG_SIZE - 1
         );
 
         Ok(())
     }
 
-    #[test]
-    fn test_history() -> Result<()> {
-        let repo = get_test_repo("test_snapshot")?;
-
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        let mut snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(0));
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // add a page and advance the last valid LSN
-        let buf = TEST_BUF(1);
-        tline.put_page_image(buf, Lsn(1), TEST_IMG("blk 1 @ lsn 1"))?;
-        tline.advance_last_valid_lsn(Lsn(1));
-        let mut snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let expected_page = RelationUpdate {
-            rel: buf.rel,
-            lsn: Lsn(1),
-            update: Update::Page {
-                blknum: buf.blknum,
-                img: TEST_IMG("blk 1 @ lsn 1"),
-            },
-        };
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // truncate to zero, but don't advance the last valid LSN
-        tline.put_truncation(buf.rel, Lsn(2), 0)?;
-        let mut snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // advance the last valid LSN and the truncation should be observable
-        tline.advance_last_valid_lsn(Lsn(2));
-        let mut snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(2));
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        let expected_truncate = RelationUpdate {
-            rel: buf.rel,
-            lsn: Lsn(2),
-            update: Update::Truncate { n_blocks: 0 },
-        };
-        assert_eq!(Some(expected_truncate), snapshot.next().transpose()?); // TODO ordering not guaranteed by API
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        Ok(())
-    }
-
     // Mock WAL redo manager that doesn't do much
     struct TestRedoManager {}
 

pageserver/src/restore_local_repo.rs

@@ -1,9 +1,9 @@
 //!
 //! Import data and WAL from a PostgreSQL data directory and WAL segments into
-//! zenith Timeline.
+//! zenith repository
 //!
 use log::*;
-use std::cmp::{max, min};
+use std::cmp::max;
 use std::fs;
 use std::fs::File;
 use std::io::Read;
@@ -14,9 +14,8 @@ use std::path::{Path, PathBuf};
 use anyhow::Result;
 use bytes::Bytes;
 
-use crate::repository::{BufferTag, RelTag, Timeline, WALRecord};
-use crate::waldecoder::{decode_wal_record, DecodedWALRecord, Oid, WalStreamDecoder};
-use crate::waldecoder::{XlCreateDatabase, XlSmgrTruncate};
+use crate::repository::{BufferTag, RelTag, Timeline};
+use crate::waldecoder::{decode_wal_record, Oid, WalStreamDecoder};
 use crate::PageServerConf;
 use crate::ZTimelineId;
 use postgres_ffi::pg_constants;
@@ -221,7 +220,7 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:
             }
             if let Some((lsn, recdata)) = rec.unwrap() {
                 let decoded = decode_wal_record(recdata.clone());
-                save_decoded_record(timeline, &decoded, recdata, lsn)?;
+                timeline.save_decoded_record(decoded, recdata, lsn)?;
                 last_lsn = lsn;
             } else {
                 break;
@@ -242,182 +241,3 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:
     info!("reached end of WAL at {}", last_lsn);
     Ok(())
 }
-
-///
-/// Helper function to parse a WAL record and call the Timeline's PUT functions for all the
-/// relations/pages that the record affects.
-///
-pub fn save_decoded_record(
-    timeline: &dyn Timeline,
-    decoded: &DecodedWALRecord,
-    recdata: Bytes,
-    lsn: Lsn,
-) -> Result<()> {
-    // Iterate through all the blocks that the record modifies, and
-    // "put" a separate copy of the record for each block.
-    for blk in decoded.blocks.iter() {
-        let tag = BufferTag {
-            rel: RelTag {
-                spcnode: blk.rnode_spcnode,
-                dbnode: blk.rnode_dbnode,
-                relnode: blk.rnode_relnode,
-                forknum: blk.forknum as u8,
-            },
-            blknum: blk.blkno,
-        };
-
-        let rec = WALRecord {
-            lsn,
-            will_init: blk.will_init || blk.apply_image,
-            rec: recdata.clone(),
-            main_data_offset: decoded.main_data_offset as u32,
-        };
-
-        timeline.put_wal_record(tag, rec)?;
-    }
-
-    // Handle a few special record types
-    if decoded.xl_rmid == pg_constants::RM_SMGR_ID
-        && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_SMGR_TRUNCATE
-    {
-        let truncate = XlSmgrTruncate::decode(&decoded);
-        save_xlog_smgr_truncate(timeline, lsn, &truncate)?;
-    } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID
-        && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_DBASE_CREATE
-    {
-        let createdb = XlCreateDatabase::decode(&decoded);
-        save_xlog_dbase_create(timeline, lsn, &createdb)?;
-    }
-
-    // Now that this record has been handled, let the repository know that
-    // it is up-to-date to this LSN
-    timeline.advance_last_record_lsn(lsn);
-    Ok(())
-}
-
-/// Subroutine of save_decoded_record(), to handle an XLOG_DBASE_CREATE record.
-fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatabase) -> Result<()> {
-    let db_id = rec.db_id;
-    let tablespace_id = rec.tablespace_id;
-    let src_db_id = rec.src_db_id;
-    let src_tablespace_id = rec.src_tablespace_id;
-
-    // Creating a database is implemented by copying the template (aka. source) database.
-    // To copy all the relations, we need to ask for the state as of the same LSN, but we
-    // cannot pass 'lsn' to the Timeline.get_* functions, or they will block waiting for
-    // the last valid LSN to advance up to it. So we use the previous record's LSN in the
-    // get calls instead.
-    let req_lsn = min(timeline.get_last_record_lsn(), lsn);
-
-    let rels = timeline.list_rels(src_tablespace_id, src_db_id, req_lsn)?;
-
-    trace!("save_create_database: {} rels", rels.len());
-
-    let mut num_rels_copied = 0;
-    let mut num_blocks_copied = 0;
-    for src_rel in rels {
-        assert_eq!(src_rel.spcnode, src_tablespace_id);
-        assert_eq!(src_rel.dbnode, src_db_id);
-
-        let nblocks = timeline.get_rel_size(src_rel, req_lsn)?;
-        let dst_rel = RelTag {
-            spcnode: tablespace_id,
-            dbnode: db_id,
-            relnode: src_rel.relnode,
-            forknum: src_rel.forknum,
-        };
-
-        // Copy content
-        for blknum in 0..nblocks {
-            let src_key = BufferTag {
-                rel: src_rel,
-                blknum,
-            };
-            let dst_key = BufferTag {
-                rel: dst_rel,
-                blknum,
-            };
-
-            let content = timeline.get_page_at_lsn(src_key, req_lsn)?;
-
-            info!("copying block {:?} to {:?}", src_key, dst_key);
-
-            timeline.put_page_image(dst_key, lsn, content)?;
-            num_blocks_copied += 1;
-        }
-
-        if nblocks == 0 {
-            // make sure we have some trace of the relation, even if it's empty
-            timeline.put_truncation(dst_rel, lsn, 0)?;
-        }
-
-        num_rels_copied += 1;
-    }
-    info!(
-        "Created database {}/{}, copied {} blocks in {} rels at {}",
-        tablespace_id, db_id, num_blocks_copied, num_rels_copied, lsn
-    );
-    Ok(())
-}
-
-/// Subroutine of save_decoded_record(), to handle an XLOG_SMGR_TRUNCATE record.
-///
-/// This is the same logic as in PostgreSQL's smgr_redo() function.
-fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTruncate) -> Result<()> {
-    let spcnode = rec.rnode.spcnode;
-    let dbnode = rec.rnode.dbnode;
-    let relnode = rec.rnode.relnode;
-
-    if (rec.flags & pg_constants::SMGR_TRUNCATE_HEAP) != 0 {
-        let rel = RelTag {
-            spcnode,
-            dbnode,
-            relnode,
-            forknum: pg_constants::MAIN_FORKNUM,
-        };
-        timeline.put_truncation(rel, lsn, rec.blkno)?;
-    }
-    if (rec.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 {
-        let rel = RelTag {
-            spcnode,
-            dbnode,
-            relnode,
-            forknum: pg_constants::FSM_FORKNUM,
-        };
-
-        // FIXME: 'blkno' stored in the WAL record is the new size of the
-        // heap. The formula for calculating the new size of the FSM is
-        // pretty complicated (see FreeSpaceMapPrepareTruncateRel() in
-        // PostgreSQL), and we should also clear bits in the tail FSM block,
-        // and update the upper level FSM pages. None of that has been
-        // implemented. What we do instead, is always just truncate the FSM
-        // to zero blocks. That's bad for performance, but safe. (The FSM
-        // isn't needed for correctness, so we could also leave garbage in
-        // it. Seems more tidy to zap it away.)
-        if rec.blkno != 0 {
-            info!("Partial truncation of FSM is not supported");
-        }
-        let num_fsm_blocks = 0;
-        timeline.put_truncation(rel, lsn, num_fsm_blocks)?;
-    }
-    if (rec.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
-        let rel = RelTag {
-            spcnode,
-            dbnode,
-            relnode,
-            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-        };
-
-        // FIXME: Like with the FSM above, the logic to truncate the VM
-        // correctly has not been implemented. Just zap it away completely,
-        // always. Unlike the FSM, the VM must never have bits incorrectly
-        // set. From a correctness point of view, it's always OK to clear
-        // bits or remove it altogether, though.
-        if rec.blkno != 0 {
-            info!("Partial truncation of VM is not supported");
-        }
-        let num_vm_blocks = 0;
-        timeline.put_truncation(rel, lsn, num_vm_blocks)?;
-    }
-    Ok(())
-}

pageserver/src/rocksdb_storage.rs

@@ -1,343 +0,0 @@
-//!
-//! An implementation of the ObjectStore interface, backed by RocksDB
-//!
-use crate::object_store::{ObjectKey, ObjectStore};
-use crate::repository::{BufferTag, RelTag};
-use crate::PageServerConf;
-use crate::ZTimelineId;
-use anyhow::{bail, Result};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::sync::{Arc, Mutex};
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct StorageKey {
-    obj_key: ObjectKey,
-    lsn: Lsn,
-}
-
-impl StorageKey {
-    /// The first key for a given timeline
-    fn timeline_start(timeline: ZTimelineId) -> Self {
-        Self {
-            obj_key: ObjectKey {
-                timeline,
-                buf_tag: BufferTag::ZEROED,
-            },
-            lsn: Lsn(0),
-        }
-    }
-}
-
-///
-/// RocksDB very inefficiently delete random record. Instead of it we have to use merge
-/// filter, which allows to throw away records at LSM merge phase.
-/// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
-/// at merge time. Version ca be removed if:
-/// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
-/// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
-///
-/// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
-/// for deletion. To mark object for deletion we can either set some flag in object itself.
-/// But it is complicated with new object value format, because RocksDB storage knows nothing about
-/// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
-/// This is why we keep keys of marked for deletion versions in HashSet in memory.
-/// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
-///
-struct GarbageCollector {
-    garbage: Mutex<HashSet<Vec<u8>>>,
-}
-
-impl GarbageCollector {
-    fn new() -> GarbageCollector {
-        GarbageCollector {
-            garbage: Mutex::new(HashSet::new()),
-        }
-    }
-
-    /// Called by GC to mark version as delete
-    fn mark_for_deletion(&self, key: &[u8]) {
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.insert(key.to_vec());
-    }
-
-    /// Called by LSM merge filter. If it finds key in the set, then
-    /// it doesn't merge it and removes from this set.
-    fn was_deleted(&self, key: &[u8]) -> bool {
-        let key = key.to_vec();
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.remove(&key)
-    }
-}
-
-pub struct RocksObjectStore {
-    _conf: &'static PageServerConf,
-
-    // RocksDB handle
-    db: rocksdb::DB,
-    gc: Arc<GarbageCollector>,
-}
-
-impl ObjectStore for RocksObjectStore {
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
-        let val = self.db.get(StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?)?;
-        if let Some(val) = val {
-            Ok(val)
-        } else {
-            bail!("could not find page {:?}", key);
-        }
-    }
-
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
-        self.db.put(
-            StorageKey::ser(&StorageKey {
-                obj_key: key.clone(),
-                lsn,
-            })?,
-            value,
-        )?;
-        Ok(())
-    }
-
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
-        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?);
-        Ok(())
-    }
-
-    /// Iterate through page versions of given page, starting from the given LSN.
-    /// The versions are walked in descending LSN order.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
-        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all entries in the given database. In practice, this
-    /// is used for CREATE DATABASE, and usually the template database is small.
-    /// But if it's not, this will be slow.
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>> {
-        // FIXME: This scans everything. Very slow
-
-        let mut rels: HashSet<RelTag> = HashSet::new();
-
-        let mut search_key = StorageKey {
-            obj_key: ObjectKey {
-                timeline: timelineid,
-                buf_tag: BufferTag {
-                    rel: RelTag {
-                        spcnode,
-                        dbnode,
-                        relnode: 0,
-                        forknum: 0u8,
-                    },
-                    blknum: 0,
-                },
-            },
-            lsn: Lsn(0),
-        };
-        let mut iter = self.db.raw_iterator();
-        loop {
-            iter.seek(search_key.ser()?);
-            if !iter.valid() {
-                break;
-            }
-            let key = StorageKey::des(iter.key().unwrap())?;
-            if (spcnode != 0 && key.obj_key.buf_tag.rel.spcnode != spcnode)
-                || (dbnode != 0 && key.obj_key.buf_tag.rel.dbnode != dbnode)
-            {
-                break;
-            }
-
-            if key.obj_key.buf_tag.rel.relnode != 0 // skip non-relational records (like timeline metadata)
-                && key.lsn < lsn
-            // visible in this snapshot
-            {
-                rels.insert(key.obj_key.buf_tag.rel);
-            }
-            search_key = key.clone();
-            search_key.obj_key.buf_tag.rel.relnode += 1; // skip to next relation
-        }
-
-        Ok(rels)
-    }
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(BufferTag, Lsn, Vec<u8>)>> + 'a>> {
-        let start_key = StorageKey::timeline_start(timeline);
-        let start_key_bytes = StorageKey::ser(&start_key)?;
-        let iter = self.db.iterator(rocksdb::IteratorMode::From(
-            &start_key_bytes,
-            rocksdb::Direction::Forward,
-        ));
-
-        Ok(Box::new(RocksObjects {
-            iter,
-            timeline,
-            lsn,
-        }))
-    }
-}
-
-impl RocksObjectStore {
-    /// Open a RocksDB database.
-    pub fn open(conf: &'static PageServerConf) -> Result<RocksObjectStore> {
-        let opts = Self::get_rocksdb_opts();
-        let obj_store = Self::new(conf, opts)?;
-        Ok(obj_store)
-    }
-
-    /// Create a new, empty RocksDB database.
-    pub fn create(conf: &'static PageServerConf) -> Result<RocksObjectStore> {
-        let path = conf.workdir.join("rocksdb-storage");
-        std::fs::create_dir(&path)?;
-
-        let mut opts = Self::get_rocksdb_opts();
-        opts.create_if_missing(true);
-        opts.set_error_if_exists(true);
-        let obj_store = Self::new(conf, opts)?;
-        Ok(obj_store)
-    }
-
-    fn new(conf: &'static PageServerConf, mut opts: rocksdb::Options) -> Result<RocksObjectStore> {
-        let path = conf.workdir.join("rocksdb-storage");
-        let gc = Arc::new(GarbageCollector::new());
-        let gc_ref = gc.clone();
-        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
-            if gc_ref.was_deleted(key) {
-                rocksdb::compaction_filter::Decision::Remove
-            } else {
-                rocksdb::compaction_filter::Decision::Keep
-            }
-        });
-        let db = rocksdb::DB::open(&opts, &path)?;
-        let obj_store = RocksObjectStore {
-            _conf: conf,
-            db,
-            gc,
-        };
-        Ok(obj_store)
-    }
-
-    /// common options used by `open` and `create`
-    fn get_rocksdb_opts() -> rocksdb::Options {
-        let mut opts = rocksdb::Options::default();
-        opts.set_use_fsync(true);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        opts
-    }
-}
-
-///
-/// Iterator for `object_versions`. Returns all page versions of a given block, in
-/// reverse LSN order.
-///
-struct RocksObjectVersionIter<'a> {
-    obj_key: ObjectKey,
-    dbiter: rocksdb::DBRawIterator<'a>,
-    first_call: bool,
-}
-impl<'a> RocksObjectVersionIter<'a> {
-    fn new(
-        db: &'a rocksdb::DB,
-        obj_key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<RocksObjectVersionIter<'a>> {
-        let key = StorageKey {
-            obj_key: obj_key.clone(),
-            lsn,
-        };
-        let mut dbiter = db.raw_iterator();
-        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
-        Ok(RocksObjectVersionIter {
-            first_call: true,
-            obj_key: obj_key.clone(),
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectVersionIter<'a> {
-    type Item = (Lsn, Vec<u8>);
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        if self.first_call {
-            self.first_call = false;
-        } else {
-            self.dbiter.prev(); // walk backwards
-        }
-
-        if !self.dbiter.valid() {
-            return None;
-        }
-        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-        if key.obj_key.buf_tag != self.obj_key.buf_tag {
-            return None;
-        }
-        let val = self.dbiter.value().unwrap();
-        let result = val.to_vec();
-
-        Some((key.lsn, result))
-    }
-}
-
-struct RocksObjects<'r> {
-    iter: rocksdb::DBIterator<'r>,
-    timeline: ZTimelineId,
-    lsn: Lsn,
-}
-
-impl<'r> Iterator for RocksObjects<'r> {
-    // TODO consider returning Box<[u8]>
-    type Item = Result<(BufferTag, Lsn, Vec<u8>)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next_result().transpose()
-    }
-}
-
-impl<'r> RocksObjects<'r> {
-    fn next_result(&mut self) -> Result<Option<(BufferTag, Lsn, Vec<u8>)>> {
-        for (key_bytes, v) in &mut self.iter {
-            let key = StorageKey::des(&key_bytes)?;
-
-            if key.obj_key.timeline != self.timeline {
-                return Ok(None);
-            }
-
-            if key.lsn > self.lsn {
-                // TODO can speed up by seeking iterator
-                continue;
-            }
-
-            return Ok(Some((key.obj_key.buf_tag, key.lsn, v.to_vec())));
-        }
-
-        Ok(None)
-    }
-}

pageserver/src/tui_event.rs

@@ -54,14 +54,14 @@ impl Events {
             thread::spawn(move || {
                 let stdin = io::stdin();
                 for evt in stdin.keys() {
-                    // This will panic if stdin returns EOF.
-                    let key = evt.unwrap();
-                    if let Err(err) = tx.send(Event::Input(key)) {
-                        eprintln!("{}", err);
-                        return;
-                    }
-                    if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
-                        return;
+                    if let Ok(key) = evt {
+                        if let Err(err) = tx.send(Event::Input(key)) {
+                            eprintln!("{}", err);
+                            return;
+                        }
+                        if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
+                            return;
+                        }
                     }
                 }
             })

pageserver/src/waldecoder.rs

@@ -1,15 +1,7 @@
-//!
-//! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
-//! the record affects, to add the records to the page cache.
-//!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use log::*;
 use postgres_ffi::pg_constants;
-use postgres_ffi::xlog_utils::*;
-use postgres_ffi::XLogLongPageHeaderData;
-use postgres_ffi::XLogPageHeaderData;
-use postgres_ffi::XLogRecord;
-
+use postgres_ffi::xlog_utils::XLogRecord;
 use std::cmp::min;
 use std::str;
 use thiserror::Error;
@@ -23,6 +15,35 @@ pub type MultiXactId = TransactionId;
 pub type MultiXactOffset = u32;
 pub type MultiXactStatus = u32;
 
+// From PostgreSQL headers
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct XLogPageHeaderData {
+    xlp_magic: u16,    /* magic value for correctness checks */
+    xlp_info: u16,     /* flag bits, see below */
+    xlp_tli: u32,      /* TimeLineID of first record on page */
+    xlp_pageaddr: u64, /* XLOG address of this page */
+    xlp_rem_len: u32,  /* total len of remaining data for record */
+}
+
+// FIXME: this assumes MAXIMUM_ALIGNOF 8. There are 4 padding bytes at end
+#[allow(non_upper_case_globals)]
+const SizeOfXLogShortPHD: usize = 2 + 2 + 4 + 8 + 4 + 4;
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct XLogLongPageHeaderData {
+    std: XLogPageHeaderData, /* standard header fields */
+    xlp_sysid: u64,          /* system identifier from pg_control */
+    xlp_seg_size: u32,       /* just as a cross-check */
+    xlp_xlog_blcksz: u32,    /* just as a cross-check */
+}
+
+// FIXME: this assumes MAXIMUM_ALIGNOF 8.
+#[allow(non_upper_case_globals)]
+const SizeOfXLogLongPHD: usize = (2 + 2 + 4 + 8 + 4) + 4 + 8 + 4 + 4;
+
 #[allow(dead_code)]
 pub struct WalStreamDecoder {
     lsn: Lsn,
@@ -69,7 +90,7 @@ impl WalStreamDecoder {
     /// decoder so far.
     ///
     /// Returns one of the following:
-    ///     Ok((Lsn, Bytes)): a tuple containing the LSN of next record, and the record itself
+    ///     Ok((u64, Bytes)): a tuple containing the LSN of next record, and the record itself
     ///     Ok(None): there is not enough data in the input buffer. Feed more by calling the `feed_bytes` function
     ///     Err(WalDecodeError): an error occured while decoding, meaning the input was invalid.
     ///
@@ -79,12 +100,11 @@ impl WalStreamDecoder {
             if self.lsn.segment_offset(pg_constants::WAL_SEGMENT_SIZE) == 0 {
                 // parse long header
 
-                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_LONG_PHD {
+                if self.inputbuf.remaining() < SizeOfXLogLongPHD {
                     return Ok(None);
                 }
 
-                let hdr = XLogLongPageHeaderData::from_bytes(&mut self.inputbuf);
-
+                let hdr = self.decode_XLogLongPageHeaderData();
                 if hdr.std.xlp_pageaddr != self.lsn.0 {
                     return Err(WalDecodeError {
                         msg: "invalid xlog segment header".into(),
@@ -93,15 +113,14 @@ impl WalStreamDecoder {
                 }
                 // TODO: verify the remaining fields in the header
 
-                self.lsn += XLOG_SIZE_OF_XLOG_LONG_PHD as u64;
+                self.lsn += SizeOfXLogLongPHD as u64;
                 continue;
             } else if self.lsn.block_offset() == 0 {
-                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_SHORT_PHD {
+                if self.inputbuf.remaining() < SizeOfXLogShortPHD {
                     return Ok(None);
                 }
 
-                let hdr = XLogPageHeaderData::from_bytes(&mut self.inputbuf);
-
+                let hdr = self.decode_XLogPageHeaderData();
                 if hdr.xlp_pageaddr != self.lsn.0 {
                     return Err(WalDecodeError {
                         msg: "invalid xlog page header".into(),
@@ -110,7 +129,7 @@ impl WalStreamDecoder {
                 }
                 // TODO: verify the remaining fields in the header
 
-                self.lsn += XLOG_SIZE_OF_XLOG_SHORT_PHD as u64;
+                self.lsn += SizeOfXLogShortPHD as u64;
                 continue;
             } else if self.padlen > 0 {
                 if self.inputbuf.remaining() < self.padlen as usize {
@@ -131,7 +150,7 @@ impl WalStreamDecoder {
                 // read xl_tot_len FIXME: assumes little-endian
                 self.startlsn = self.lsn;
                 let xl_tot_len = self.inputbuf.get_u32_le();
-                if (xl_tot_len as usize) < XLOG_SIZE_OF_XLOG_RECORD {
+                if xl_tot_len < SizeOfXLogRecord {
                     return Err(WalDecodeError {
                         msg: format!("invalid xl_tot_len {}", xl_tot_len),
                         lsn: self.lsn,
@@ -190,6 +209,40 @@ impl WalStreamDecoder {
 
         // deal with xlog_switch records
     }
+
+    #[allow(non_snake_case)]
+    fn decode_XLogPageHeaderData(&mut self) -> XLogPageHeaderData {
+        let buf = &mut self.inputbuf;
+
+        // FIXME: Assume little-endian
+
+        let hdr: XLogPageHeaderData = XLogPageHeaderData {
+            xlp_magic: buf.get_u16_le(),
+            xlp_info: buf.get_u16_le(),
+            xlp_tli: buf.get_u32_le(),
+            xlp_pageaddr: buf.get_u64_le(),
+            xlp_rem_len: buf.get_u32_le(),
+        };
+        // 4 bytes of padding, on 64-bit systems
+        buf.advance(4);
+
+        // FIXME: check that hdr.xlp_rem_len matches self.contlen
+        //println!("next xlog page (xlp_rem_len: {})", hdr.xlp_rem_len);
+
+        hdr
+    }
+
+    #[allow(non_snake_case)]
+    fn decode_XLogLongPageHeaderData(&mut self) -> XLogLongPageHeaderData {
+        let hdr: XLogLongPageHeaderData = XLogLongPageHeaderData {
+            std: self.decode_XLogPageHeaderData(),
+            xlp_sysid: self.inputbuf.get_u64_le(),
+            xlp_seg_size: self.inputbuf.get_u32_le(),
+            xlp_xlog_blcksz: self.inputbuf.get_u32_le(),
+        };
+
+        hdr
+    }
 }
 
 #[allow(dead_code)]
@@ -211,7 +264,8 @@ pub struct DecodedBkpBlock {
     /* Information on full-page image, if any */
     has_image: bool,       /* has image, even for consistency checking */
     pub apply_image: bool, /* has image that should be restored */
-    pub will_init: bool,   /* record doesn't need previous page version to apply */
+    pub will_init: bool,   /* record intialize page content */
+    pub will_drop: bool,   /* record drops relation */
     //char	   *bkp_image;
     hole_offset: u16,
     hole_length: u16,
@@ -236,6 +290,7 @@ impl DecodedBkpBlock {
             has_image: false,
             apply_image: false,
             will_init: false,
+            will_drop: false,
             hole_offset: 0,
             hole_length: 0,
             bimg_len: 0,
@@ -247,6 +302,9 @@ impl DecodedBkpBlock {
     }
 }
 
+#[allow(non_upper_case_globals)]
+const SizeOfXLogRecord: u32 = 24;
+
 pub struct DecodedWALRecord {
     pub xl_info: u8,
     pub xl_rmid: u8,
@@ -293,7 +351,7 @@ pub struct XlSmgrTruncate {
 impl XlSmgrTruncate {
     pub fn decode(decoded: &DecodedWALRecord) -> XlSmgrTruncate {
         let mut buf = decoded.record.clone();
-        buf.advance((XLOG_SIZE_OF_XLOG_RECORD + 2) as usize);
+        buf.advance((SizeOfXLogRecord + 2) as usize);
         XlSmgrTruncate {
             blkno: buf.get_u32_le(),
             rnode: RelFileNode {
@@ -318,7 +376,7 @@ pub struct XlCreateDatabase {
 impl XlCreateDatabase {
     pub fn decode(decoded: &DecodedWALRecord) -> XlCreateDatabase {
         let mut buf = decoded.record.clone();
-        buf.advance((XLOG_SIZE_OF_XLOG_RECORD + 2) as usize);
+        buf.advance((SizeOfXLogRecord + 2) as usize);
         XlCreateDatabase {
             db_id: buf.get_u32_le(),
             tablespace_id: buf.get_u32_le(),
@@ -472,7 +530,8 @@ impl XlMultiXactTruncate {
     }
 }
 
-/// Main routine to decode a WAL record and figure out which blocks are modified
+//
+// Routines to decode a WAL record and figure out which blocks are modified
 //
 // See xlogrecord.h for details
 // The overall layout of an XLOG record is:
@@ -509,9 +568,9 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         xlogrec.xl_info
     );
 
-    let remaining: usize = xlogrec.xl_tot_len as usize - XLOG_SIZE_OF_XLOG_RECORD;
+    let remaining = xlogrec.xl_tot_len - SizeOfXLogRecord;
 
-    if buf.remaining() != remaining {
+    if buf.remaining() != remaining as usize {
         //TODO error
     }
 
@@ -747,7 +806,13 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
                     let spcnode = buf.get_u32_le();
                     let dbnode = buf.get_u32_le();
                     let relnode = buf.get_u32_le();
-                    //TODO handle this too?
+                    let mut blk = DecodedBkpBlock::new();
+                    blk.forknum = pg_constants::MAIN_FORKNUM;
+                    blk.rnode_spcnode = spcnode;
+                    blk.rnode_dbnode = dbnode;
+                    blk.rnode_relnode = relnode;
+                    blk.will_drop = true;
+                    blocks.push(blk);
                     trace!(
                         "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
                         spcnode,
@@ -793,7 +858,13 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
                     let spcnode = buf.get_u32_le();
                     let dbnode = buf.get_u32_le();
                     let relnode = buf.get_u32_le();
-                    //TODO handle this too?
+                    let mut blk = DecodedBkpBlock::new();
+                    blk.forknum = pg_constants::MAIN_FORKNUM;
+                    blk.rnode_spcnode = spcnode;
+                    blk.rnode_dbnode = dbnode;
+                    blk.rnode_relnode = relnode;
+                    blk.will_drop = true;
+                    blocks.push(blk);
                     trace!(
                         "XLOG_XACT_ABORT relfilenode {}/{}/{}",
                         spcnode,

pageserver/src/walreceiver.rs

@@ -1,11 +1,12 @@
 //!
-//! WAL receiver connects to the WAL safekeeper service,
-//! streams WAL, decodes records and saves them in page cache.
+//! WAL receiver
+//!
+//! The WAL receiver connects to the WAL safekeeper service, and streams WAL.
+//! For each WAL record, it decodes the record to figure out which data blocks
+//! the record affects, and adds the records to the page cache.
 //!
-//! We keep one WAL receiver active per timeline.
 
 use crate::page_cache;
-use crate::restore_local_repo;
 use crate::waldecoder::*;
 use crate::PageServerConf;
 use crate::ZTimelineId;
@@ -120,10 +121,7 @@ fn walreceiver_main(
 ) -> Result<(), Error> {
     // Connect to the database in replication mode.
     info!("connecting to {:?}", wal_producer_connstr);
-    let connect_cfg = format!(
-        "{} application_name=pageserver replication=true",
-        wal_producer_connstr
-    );
+    let connect_cfg = format!("{} replication=true", wal_producer_connstr);
 
     let mut rclient = Client::connect(&connect_cfg, NoTls)?;
     info!("connected!");
@@ -169,7 +167,7 @@ fn walreceiver_main(
     let mut waldecoder = WalStreamDecoder::new(startpoint);
 
     while let Some(replication_message) = physical_stream.next()? {
-        let status_update = match replication_message {
+        match replication_message {
             ReplicationMessage::XLogData(xlog_data) => {
                 // Pass the WAL data to the decoder, and see if we can decode
                 // more records as a result.
@@ -186,7 +184,11 @@ fn walreceiver_main(
 
                 while let Some((lsn, recdata)) = waldecoder.poll_decode()? {
                     let decoded = decode_wal_record(recdata.clone());
-                    restore_local_repo::save_decoded_record(&*timeline, &decoded, recdata, lsn)?;
+                    timeline.save_decoded_record(decoded, recdata, lsn)?;
+
+                    // Now that this record has been handled, let the page cache know that
+                    // it is up-to-date to this LSN
+                    timeline.advance_last_record_lsn(lsn);
                     last_rec_lsn = lsn;
                 }
 
@@ -229,14 +231,12 @@ fn walreceiver_main(
                     info!("caught up at LSN {}", endlsn);
                     caught_up = true;
                 }
-
-                Some(endlsn)
             }
 
             ReplicationMessage::PrimaryKeepAlive(keepalive) => {
                 let wal_end = keepalive.wal_end();
                 let timestamp = keepalive.timestamp();
-                let reply_requested = keepalive.reply() != 0;
+                let reply_requested: bool = keepalive.reply() != 0;
 
                 trace!(
                     "received PrimaryKeepAlive(wal_end: {}, timestamp: {:?} reply: {})",
@@ -244,27 +244,20 @@ fn walreceiver_main(
                     timestamp,
                     reply_requested,
                 );
-
                 if reply_requested {
-                    Some(timeline.get_last_valid_lsn())
-                } else {
-                    None
+                    // TODO: More thought should go into what values are sent here.
+                    let last_lsn = PgLsn::from(u64::from(timeline.get_last_valid_lsn()));
+                    let write_lsn = last_lsn;
+                    let flush_lsn = last_lsn;
+                    let apply_lsn = PgLsn::from(0);
+                    let ts = SystemTime::now();
+                    const NO_REPLY: u8 = 0u8;
+
+                    physical_stream
+                        .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
                 }
             }
-
-            _ => None,
-        };
-
-        if let Some(last_lsn) = status_update {
-            // TODO: More thought should go into what values are sent here.
-            let last_lsn = PgLsn::from(u64::from(last_lsn));
-            let write_lsn = last_lsn;
-            let flush_lsn = last_lsn;
-            let apply_lsn = PgLsn::from(0);
-            let ts = SystemTime::now();
-            const NO_REPLY: u8 = 0;
-
-            physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
+            _ => (),
         }
     }
     Ok(())

pageserver/src/walredo.rs

@@ -1,10 +1,9 @@
 //!
-//! WAL redo. This service runs PostgreSQL in a special wal_redo mode
-//! to apply given WAL records over an old page image and return new page image.
+//! WAL redo
 //!
 //! We rely on Postgres to perform WAL redo for us. We launch a
 //! postgres process in special "wal redo" mode that's similar to
-//! single-user mode. We then pass the previous page image, if any,
+//! single-user mode. We then pass the the previous page image, if any,
 //! and all the WAL records we want to apply, to the postgres
 //! process. Then we get the page image back. Communication with the
 //! postgres process happens via stdin/stdout
@@ -22,7 +21,7 @@ use std::fs;
 use std::fs::OpenOptions;
 use std::io::prelude::*;
 use std::io::Error;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::process::Stdio;
 use std::sync::mpsc;
 use std::sync::Mutex;
@@ -206,10 +205,15 @@ impl PostgresRedoManagerInternal {
 
         let process: PostgresRedoProcess;
 
+        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
+        // just create one with constant name. That fails if you try to launch more than
+        // one WAL redo manager concurrently.
+        let datadir = self.conf.workdir.join("wal-redo-datadir");
+
         info!("launching WAL redo postgres process");
 
         process = runtime
-            .block_on(PostgresRedoProcess::launch(self.conf))
+            .block_on(PostgresRedoProcess::launch(&datadir))
             .unwrap();
 
         // Loop forever, handling requests as they come.
@@ -255,7 +259,7 @@ impl PostgresRedoManagerInternal {
 
         let result: Result<Bytes, WalRedoError>;
 
-        debug!(
+        trace!(
             "applied {} WAL records in {} ms to reconstruct page image at LSN {}",
             nrecords,
             duration.as_millis(),
@@ -285,12 +289,12 @@ impl PostgresRedoProcess {
     //
     // Start postgres binary in special WAL redo mode.
     //
-    async fn launch(conf: &PageServerConf) -> Result<PostgresRedoProcess, Error> {
-        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
-        // just create one with constant name. That fails if you try to launch more than
-        // one WAL redo manager concurrently.
-        let datadir = conf.workdir.join("wal-redo-datadir");
+    // Tests who run pageserver binary are setting proper PG_BIN_DIR
+    // and PG_LIB_DIR so that WalRedo would start right postgres.
 
+    // do that: We may later
+    // switch to setting same things in pageserver config file.
+    async fn launch(datadir: &Path) -> Result<PostgresRedoProcess, Error> {
         // Create empty data directory for wal-redo postgres, deleting old one first.
         if datadir.exists() {
             info!("directory {:?} exists, removing", &datadir);
@@ -299,12 +303,9 @@ impl PostgresRedoProcess {
             }
         }
         info!("running initdb in {:?}", datadir.display());
-        let initdb = Command::new(conf.pg_bin_dir().join("initdb"))
+        let initdb = Command::new("initdb")
             .args(&["-D", datadir.to_str().unwrap()])
             .arg("-N")
-            .env_clear()
-            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
             .output()
             .await
             .expect("failed to execute initdb");
@@ -326,15 +327,12 @@ impl PostgresRedoProcess {
             config.write_all(b"zenith.wal_redo=on\n")?;
         }
         // Start postgres itself
-        let mut child = Command::new(conf.pg_bin_dir().join("postgres"))
+        let mut child = Command::new("postgres")
             .arg("--wal-redo")
             .stdin(Stdio::piped())
             .stderr(Stdio::piped())
             .stdout(Stdio::piped())
-            .env_clear()
-            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-            .env("PGDATA", &datadir)
+            .env("PGDATA", datadir)
             .spawn()
             .expect("postgres --wal-redo command failed to start");
 

postgres_ffi/Cargo.toml

@@ -17,7 +17,6 @@ crc32c = "0.6.0"
 hex = "0.4.3"
 lazy_static = "1.4"
 log = "0.4.14"
-memoffset = "0.6.2"
 thiserror = "1.0"
 workspace_hack = { path = "../workspace_hack" }
 

postgres_ffi/README

@@ -1,25 +1,3 @@
-This module contains utilities for working with PostgreSQL file
-formats. It's a collection of structs that are auto-generated from the
-PostgreSQL header files using bindgen, and Rust functions to read and
-manipulate them.
+This module contains utility functions for interacting with PostgreSQL
+file formats.
 
-There are also a bunch of constants in `pg_constants.rs` that are copied
-from various PostgreSQL headers, rather than auto-generated. They mostly
-should be auto-generated too, but that's a TODO.
-
-The PostgreSQL on-disk file format is not portable across different
-CPU architectures and operating systems. It is also subject to change
-in each major PostgreSQL version. Currently, this module is based on
-PostgreSQL v14, but in the future we will probably need a separate
-copy for each PostgreSQL version.
-
-To interact with the C structs, there is some unsafe code in this
-module. Do not copy-paste that to the rest of the codebase! Keep the
-amount of unsafe code to a minimum, and limited to this module only,
-and only where it's truly needed.
-
-TODO: Currently, there is also some code that deals with WAL records
-in pageserver/src/waldecoder.rs.  That should be moved into this
-module. The rest of the codebase should not have intimate knowledge of
-PostgreSQL file formats or WAL layout, that knowledge should be
-encapsulated in this module.

postgres_ffi/build.rs

@@ -11,42 +11,27 @@ fn main() {
     // to bindgen, and lets you build up options for
     // the resulting bindings.
     let bindings = bindgen::Builder::default()
-        //
-        // All the needed PostgreSQL headers are included from 'pg_control_ffi.h'
-        //
+        // The input header we would like to generate
+        // bindings for.
         .header("pg_control_ffi.h")
-        //
         // Tell cargo to invalidate the built crate whenever any of the
         // included header files changed.
-        //
         .parse_callbacks(Box::new(bindgen::CargoCallbacks))
-        //
-        // These are the types and constants that we want to generate bindings for
-        //
         .whitelist_type("ControlFileData")
-        .whitelist_type("CheckPoint")
-        .whitelist_type("FullTransactionId")
-        .whitelist_type("XLogRecord")
-        .whitelist_type("XLogPageHeaderData")
-        .whitelist_type("XLogLongPageHeaderData")
-        .whitelist_var("XLOG_PAGE_MAGIC")
         .whitelist_var("PG_CONTROL_FILE_SIZE")
         .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
         .whitelist_type("DBState")
-        //
         // Path the server include dir. It is in tmp_install/include/server, if you did
         // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
         // and used DESTDIR to move it into tmp_install, then it's in
         // tmp_install/include/postgres/server
         // 'pg_config --includedir-server' would perhaps be the more proper way to find it,
         // but this will do for now.
-        //
         .clang_arg("-I../tmp_install/include/server")
         .clang_arg("-I../tmp_install/include/postgresql/server")
-        //
         // Finish the builder and generate the bindings.
-        //
         .generate()
+        // Unwrap the Result and panic on failure.
         .expect("Unable to generate bindings");
 
     // Write the bindings to the $OUT_DIR/bindings.rs file.

postgres_ffi/pg_control_ffi.h

@@ -1,10 +1,4 @@
-/*
- * This header file is the input to bindgen. It includes all the
- * PostgreSQL headers that we need to auto-generate Rust structs
- * from. If you need to expose a new struct to Rust code, add the
- * header here, and whitelist the struct in the build.rs file.
- */
 #include "c.h"
 #include "catalog/pg_control.h"
-#include "access/xlog_internal.h"
 
+const uint32 PG_CONTROLFILEDATA_OFFSETOF_CRC = offsetof(ControlFileData, crc);

postgres_ffi/src/controlfile_utils.rs

@@ -1,124 +0,0 @@
-//!
-//! Utilities for reading and writing the PostgreSQL control file.
-//!
-//! The PostgreSQL control file is one the first things that the PostgreSQL
-//! server reads when it starts up. It indicates whether the server was shut
-//! down cleanly, or if it crashed or was restored from online backup so that
-//! WAL recovery needs to be performed. It also contains a copy of the latest
-//! checkpoint record and its location in the WAL.
-//!
-//! The control file also contains fields for detecting whether the
-//! data directory is compatible with a postgres binary. That includes
-//! a version number, configuration options that can be set at
-//! compilation time like the block size, and the platform's alignment
-//! and endianess information. (The PostgreSQL on-disk file format is
-//! not portable across platforms.)
-//!
-//! The control file is stored in the PostgreSQL data directory, as
-//! `global/pg_control`. The data stored in it is designed to be smaller than
-//! 512 bytes, on the assumption that it can be updated atomically. The actual
-//! file is larger, 8192 bytes, but the rest of it is just filled with zeros.
-//!
-//! See src/include/catalog/pg_control.h in the PostgreSQL sources for more
-//! information. You can use PostgreSQL's pg_controldata utility to view its
-//! contents.
-//!
-use crate::{ControlFileData, PG_CONTROL_FILE_SIZE};
-
-use anyhow::{bail, Result};
-use bytes::{Bytes, BytesMut};
-
-/// Equivalent to sizeof(ControlFileData) in C
-const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
-
-impl ControlFileData {
-    /// Compute the offset of the `crc` field within the `ControlFileData` struct.
-    /// Equivalent to offsetof(ControlFileData, crc) in C.
-    // Someday this can be const when the right compiler features land.
-    fn pg_control_crc_offset() -> usize {
-        memoffset::offset_of!(ControlFileData, crc)
-    }
-
-    ///
-    /// Interpret a slice of bytes as a Postgres control file.
-    ///
-    pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
-        // Check that the slice has the expected size. The control file is
-        // padded with zeros up to a 512 byte sector size, so accept a
-        // larger size too, so that the caller can just the whole file
-        // contents without knowing the exact size of the struct.
-        if buf.len() < SIZEOF_CONTROLDATA {
-            bail!("control file is too short");
-        }
-
-        // Compute the expected CRC of the content.
-        let OFFSETOF_CRC = Self::pg_control_crc_offset();
-        let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);
-
-        // Convert the slice into an array of the right size, and use `transmute` to
-        // reinterpret the raw bytes as a ControlFileData struct.
-        //
-        // NB: Ideally we would use 'zerocopy::FromBytes' for this, but bindgen doesn't
-        // derive FromBytes for us. The safety of this depends on the same constraints
-        // as for FromBytes, namely, all of its fields must implement FromBytes. That
-        // includes the primitive integer types, like `u8`, `u16`, `u32`, `u64` and their
-        // signed variants. But `bool` is not safe, because the contents of the high bits
-        // in a rust bool are undefined. In practice, PostgreSQL uses 1 to represent
-        // true and 0 for false, which is compatible with Rust bool, but let's try not to
-        // depend on it.
-        //
-        // FIXME: ControlFileData does contain 'bool's at the moment.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion on the safety
-        // of this.
-        let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
-        b.copy_from_slice(&buf[0..SIZEOF_CONTROLDATA]);
-        let controlfile: ControlFileData =
-            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
-
-        // Check the CRC
-        if expectedcrc != controlfile.crc {
-            bail!(
-                "invalid CRC in control file: expected {:08X}, was {:08X}",
-                expectedcrc,
-                controlfile.crc
-            );
-        }
-
-        Ok(controlfile)
-    }
-
-    ///
-    /// Convert a struct representing a Postgres control file into raw bytes.
-    ///
-    /// The CRC is recomputed to match the contents of the fields.
-    pub fn encode(&self) -> Bytes {
-        //
-        // Use `transmute` to reinterpret struct as raw bytes.
-        //
-        // FIXME: This triggers undefined behavior, because the contents
-        // of the padding bytes are undefined, and this leaks those
-        // undefined bytes into the resulting array. The Rust code won't
-        // care what's in those bytes, and PostgreSQL doesn't care
-        // either. HOWEVER, it is a potential security issue, because the
-        // bytes can contain arbitrary pieces of memory from the page
-        // server. In the worst case, that could be private keys or
-        // another tenant's data.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion.
-        let b: [u8; SIZEOF_CONTROLDATA] =
-            unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(*self) };
-
-        // Recompute the CRC
-        let OFFSETOF_CRC = Self::pg_control_crc_offset();
-        let newcrc = crc32c::crc32c(&b[0..OFFSETOF_CRC]);
-
-        let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
-        buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
-        buf.extend_from_slice(&newcrc.to_ne_bytes());
-        // Fill the rest of the control file with zeros.
-        buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
-
-        buf.into()
-    }
-}

postgres_ffi/src/lib.rs

@@ -3,7 +3,69 @@
 #![allow(non_snake_case)]
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 
-pub mod controlfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
 pub mod xlog_utils;
+
+use bytes::{Buf, Bytes, BytesMut};
+
+// sizeof(ControlFileData)
+const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
+const OFFSETOF_CRC: usize = PG_CONTROLFILEDATA_OFFSETOF_CRC as usize;
+
+impl ControlFileData {
+    // Initialize an all-zeros ControlFileData struct
+    pub fn new() -> ControlFileData {
+        let controlfile: ControlFileData;
+
+        let b = [0u8; SIZEOF_CONTROLDATA];
+        controlfile =
+            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
+
+        controlfile
+    }
+}
+
+pub fn decode_pg_control(mut buf: Bytes) -> Result<ControlFileData, anyhow::Error> {
+    let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
+    buf.copy_to_slice(&mut b);
+
+    let controlfile: ControlFileData;
+
+    // TODO: verify CRC
+    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
+    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
+    let expectedcrc = crc32c::crc32c(&data_without_crc);
+
+    controlfile = unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
+
+    if expectedcrc != controlfile.crc {
+        anyhow::bail!(
+            "invalid CRC in control file: expected {:08X}, was {:08X}",
+            expectedcrc,
+            controlfile.crc
+        );
+    }
+
+    Ok(controlfile)
+}
+
+pub fn encode_pg_control(controlfile: ControlFileData) -> Bytes {
+    let b: [u8; SIZEOF_CONTROLDATA];
+
+    b = unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(controlfile) };
+
+    // Recompute the CRC
+    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
+    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
+    let newcrc = crc32c::crc32c(&data_without_crc);
+
+    let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
+
+    buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
+    buf.extend_from_slice(&newcrc.to_ne_bytes());
+    // Fill the rest of the control file with zeros.
+    buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
+
+    buf.into()
+}

postgres_ffi/src/pg_constants.rs

@@ -1,11 +1,6 @@
 //!
 //! Misc constants, copied from PostgreSQL headers.
 //!
-//! TODO: These probably should be auto-generated using bindgen,
-//! rather than copied by hand. Although on the other hand, it's nice
-//! to have them all here in one place, and have the ability to add
-//! comments on them.
-//!
 
 //
 // From pg_tablespace_d.h
@@ -131,11 +126,5 @@ pub const BKPIMAGE_HAS_HOLE: u8 = 0x01; /* page image has "hole" */
 pub const BKPIMAGE_IS_COMPRESSED: u8 = 0x02; /* page image is compressed */
 pub const BKPIMAGE_APPLY: u8 = 0x04; /* page image should be restored during replay */
 
-/* From transam.h */
-pub const FIRST_NORMAL_TRANSACTION_ID: u32 = 3;
-pub const INVALID_TRANSACTION_ID: u32 = 0;
-pub const FIRST_BOOTSTRAP_OBJECT_ID: u32 = 12000;
-pub const FIRST_NORMAL_OBJECT_ID: u32 = 16384;
-
 /* FIXME: pageserver should request wal_seg_size from compute node */
 pub const WAL_SEGMENT_SIZE: usize = 16 * 1024 * 1024;

postgres_ffi/src/xlog_utils.rs

@@ -8,13 +8,6 @@
 //
 
 use crate::pg_constants;
-use crate::CheckPoint;
-use crate::FullTransactionId;
-use crate::XLogLongPageHeaderData;
-use crate::XLogPageHeaderData;
-use crate::XLogRecord;
-
-use crate::XLOG_PAGE_MAGIC;
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, Bytes};
 use crc32c::*;
@@ -28,14 +21,14 @@ use std::time::SystemTime;
 pub const XLOG_FNAME_LEN: usize = 24;
 pub const XLOG_BLCKSZ: usize = 8192;
 pub const XLP_FIRST_IS_CONTRECORD: u16 = 0x0001;
+pub const XLOG_PAGE_MAGIC: u16 = 0xD109;
 pub const XLP_REM_LEN_OFFS: usize = 2 + 2 + 4 + 8;
+pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = XLP_REM_LEN_OFFS + 4 + 4;
+pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = XLOG_SIZE_OF_XLOG_SHORT_PHD + 8 + 4 + 4;
 pub const XLOG_RECORD_CRC_OFFS: usize = 4 + 4 + 8 + 1 + 1 + 2;
+pub const XLOG_SIZE_OF_XLOG_RECORD: usize = XLOG_RECORD_CRC_OFFS + 4;
 pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16;
 
-pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = std::mem::size_of::<XLogPageHeaderData>();
-pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = std::mem::size_of::<XLogLongPageHeaderData>();
-pub const XLOG_SIZE_OF_XLOG_RECORD: usize = std::mem::size_of::<XLogRecord>();
-
 pub type XLogRecPtr = u64;
 pub type TimeLineID = u32;
 pub type TimestampTz = u64;
@@ -127,7 +120,7 @@ fn find_end_of_wal_segment(
             let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
             let xlp_info = LittleEndian::read_u16(&buf[2..4]);
             let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
-            if xlp_magic != XLOG_PAGE_MAGIC as u16 {
+            if xlp_magic != XLOG_PAGE_MAGIC {
                 info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
                 break;
             }
@@ -264,6 +257,21 @@ pub fn main() {
     );
 }
 
+//
+// Xlog record parsing routines
+// TODO move here other related code from waldecoder.rs
+//
+#[repr(C)]
+#[derive(Debug)]
+pub struct XLogRecord {
+    pub xl_tot_len: u32,
+    pub xl_xid: u32,
+    pub xl_prev: u64,
+    pub xl_info: u8,
+    pub xl_rmid: u8,
+    pub xl_crc: u32,
+}
+
 impl XLogRecord {
     pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
         XLogRecord {
@@ -279,107 +287,8 @@ impl XLogRecord {
         }
     }
 
-    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_RECORD];
-        b = unsafe { std::mem::transmute::<XLogRecord, [u8; XLOG_SIZE_OF_XLOG_RECORD]>(*self) };
-        Bytes::copy_from_slice(&b[..])
-    }
-
     // Is this record an XLOG_SWITCH record? They need some special processing,
     pub fn is_xlog_switch_record(&self) -> bool {
         self.xl_info == pg_constants::XLOG_SWITCH && self.xl_rmid == pg_constants::RM_XLOG_ID
     }
 }
-
-impl XLogPageHeaderData {
-    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
-        let hdr: XLogPageHeaderData = XLogPageHeaderData {
-            xlp_magic: buf.get_u16_le(),
-            xlp_info: buf.get_u16_le(),
-            xlp_tli: buf.get_u32_le(),
-            xlp_pageaddr: buf.get_u64_le(),
-            xlp_rem_len: buf.get_u32_le(),
-        };
-        buf.get_u32_le(); //padding
-        hdr
-    }
-}
-
-impl XLogLongPageHeaderData {
-    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
-        XLogLongPageHeaderData {
-            std: XLogPageHeaderData::from_bytes(buf),
-            xlp_sysid: buf.get_u64_le(),
-            xlp_seg_size: buf.get_u32_le(),
-            xlp_xlog_blcksz: buf.get_u32_le(),
-        }
-    }
-
-    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_LONG_PHD];
-        b = unsafe {
-            std::mem::transmute::<XLogLongPageHeaderData, [u8; XLOG_SIZE_OF_XLOG_LONG_PHD]>(*self)
-        };
-        Bytes::copy_from_slice(&b[..])
-    }
-}
-
-pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
-
-impl CheckPoint {
-    pub fn new(lsn: u64, timeline: u32) -> CheckPoint {
-        CheckPoint {
-            redo: lsn,
-            ThisTimeLineID: timeline,
-            PrevTimeLineID: timeline,
-            fullPageWrites: true, // TODO: get actual value of full_page_writes
-            nextXid: FullTransactionId {
-                value: pg_constants::FIRST_NORMAL_TRANSACTION_ID as u64,
-            }, // TODO: handle epoch?
-            nextOid: pg_constants::FIRST_BOOTSTRAP_OBJECT_ID,
-            nextMulti: 1,
-            nextMultiOffset: 0,
-            oldestXid: pg_constants::FIRST_NORMAL_TRANSACTION_ID,
-            oldestXidDB: 0,
-            oldestMulti: 1,
-            oldestMultiDB: 0,
-            time: 0,
-            oldestCommitTsXid: 0,
-            newestCommitTsXid: 0,
-            oldestActiveXid: pg_constants::INVALID_TRANSACTION_ID,
-        }
-    }
-
-    pub fn encode(&self) -> Bytes {
-        let b: [u8; SIZEOF_CHECKPOINT];
-        b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(*self) };
-        Bytes::copy_from_slice(&b[..])
-    }
-
-    pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
-        let mut b = [0u8; SIZEOF_CHECKPOINT];
-        b.copy_from_slice(&buf[0..SIZEOF_CHECKPOINT]);
-        let checkpoint: CheckPoint;
-        checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
-        Ok(checkpoint)
-    }
-
-    // Update next XID based on provided new_xid and stored epoch.
-    // Next XID should be greater than new_xid.
-    // Also take in account 32-bit wrap-around.
-    pub fn update_next_xid(&mut self, xid: u32) {
-        let full_xid = self.nextXid.value;
-        let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
-        let old_xid = full_xid as u32;
-        if new_xid.wrapping_sub(old_xid) as i32 > 0 {
-            let mut epoch = full_xid >> 32;
-            if new_xid < old_xid {
-                // wrap-around
-                epoch += 1;
-            }
-            self.nextXid = FullTransactionId {
-                value: (epoch << 32) | new_xid as u64,
-            };
-        }
-    }
-}

run_clippy.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# If you save this in your path under the name "cargo-zclippy" (or whatever
-# name you like), then you can run it as "cargo zclippy" from the shell prompt.
-#
-# If your text editor has rust-analyzer integration, you can also use this new
-# command as a replacement for "cargo check" or "cargo clippy" and see clippy
-# warnings and errors right in the editor.
-# In vscode, this setting is Rust-analyzer>Check On Save:Command
-
-cargo clippy "${@:2}" -- -A clippy::new_without_default -A clippy::manual_range_contains -A clippy::comparison_chain

test_runner/Pipfile

@@ -1,18 +0,0 @@
-[[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-pytest = ">=6.0.0"
-psycopg2 = "*"
-typing-extensions = "*"
-
-[dev-packages]
-yapf = "*"
-flake8 = "*"
-mypy = "*"
-
-[requires]
-# we need at least 3.6, but pipenv doesn't allow to say this directly
-python_version = "3"

test_runner/Pipfile.lock

@@ -1,269 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "4c20c05c20c50cf7e8f78ab461ab23841125345e63e00e2efa7661c165b6b364"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.python.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "attrs": {
-            "hashes": [
-                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
-                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==21.2.0"
-        },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00",
-                "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==4.5.0"
-        },
-        "iniconfig": {
-            "hashes": [
-                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
-                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
-            ],
-            "version": "==1.1.1"
-        },
-        "packaging": {
-            "hashes": [
-                "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
-                "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.9"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
-                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.13.1"
-        },
-        "psycopg2": {
-            "hashes": [
-                "sha256:03a485bf71498870e38b535c0e6e7162d6ac06a91487edddc3b959894d65f79c",
-                "sha256:22102cfeb904898254f287b1a77360bf66c636858e7476593acd5267e5c24ff9",
-                "sha256:8f4c1800e57ad128d20b2e91d222ca238fffd316cef65be781361cdf35e37979",
-                "sha256:b12073fdf2002e828e5921be2c39ff9c6eab361c5c0bd6c529619fc23677accc",
-                "sha256:b6f47af317af8110818d255e693cfa80b7f1e435285be09778db7b66efd95789",
-                "sha256:d549db98fc0e6db41a2aa0d65f7434c4308a9f64012adb209b9e489f26fe87c6",
-                "sha256:e44e39a46af7c30566b7667fb27e701e652ab0a51e05c263a01d3ff0e223b765",
-                "sha256:e84c80be7a238d3c9c099b71f6890eaa35fc881146232cce888a88ab1bfb431e",
-                "sha256:f3d42bd42302293767b84206d9a446abc67ed4a133e4fe04dad8952de06c2091"
-            ],
-            "index": "pypi",
-            "version": "==2.9"
-        },
-        "py": {
-            "hashes": [
-                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
-                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.10.0"
-        },
-        "pyparsing": {
-            "hashes": [
-                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
-                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.4.7"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
-                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
-            ],
-            "index": "pypi",
-            "version": "==6.2.4"
-        },
-        "toml": {
-            "hashes": [
-                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
-                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.10.2"
-        },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
-                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
-                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
-            ],
-            "index": "pypi",
-            "version": "==3.10.0.0"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
-                "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.1"
-        }
-    },
-    "develop": {
-        "flake8": {
-            "hashes": [
-                "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b",
-                "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"
-            ],
-            "index": "pypi",
-            "version": "==3.9.2"
-        },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00",
-                "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==4.5.0"
-        },
-        "mccabe": {
-            "hashes": [
-                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
-                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
-            ],
-            "version": "==0.6.1"
-        },
-        "mypy": {
-            "hashes": [
-                "sha256:0190fb77e93ce971954c9e54ea61de2802065174e5e990c9d4c1d0f54fbeeca2",
-                "sha256:0756529da2dd4d53d26096b7969ce0a47997123261a5432b48cc6848a2cb0bd4",
-                "sha256:2f9fedc1f186697fda191e634ac1d02f03d4c260212ccb018fabbb6d4b03eee8",
-                "sha256:353aac2ce41ddeaf7599f1c73fed2b75750bef3b44b6ad12985a991bc002a0da",
-                "sha256:3f12705eabdd274b98f676e3e5a89f247ea86dc1af48a2d5a2b080abac4e1243",
-                "sha256:4efc67b9b3e2fddbe395700f91d5b8deb5980bfaaccb77b306310bd0b9e002eb",
-                "sha256:517e7528d1be7e187a5db7f0a3e479747307c1b897d9706b1c662014faba3116",
-                "sha256:68a098c104ae2b75e946b107ef69dd8398d54cb52ad57580dfb9fc78f7f997f0",
-                "sha256:746e0b0101b8efec34902810047f26a8c80e1efbb4fc554956d848c05ef85d76",
-                "sha256:8be7bbd091886bde9fcafed8dd089a766fa76eb223135fe5c9e9798f78023a20",
-                "sha256:9236c21194fde5df1b4d8ebc2ef2c1f2a5dc7f18bcbea54274937cae2e20a01c",
-                "sha256:9ef5355eaaf7a23ab157c21a44c614365238a7bdb3552ec3b80c393697d974e1",
-                "sha256:9f1d74eeb3f58c7bd3f3f92b8f63cb1678466a55e2c4612bf36909105d0724ab",
-                "sha256:a26d0e53e90815c765f91966442775cf03b8a7514a4e960de7b5320208b07269",
-                "sha256:ae94c31bb556ddb2310e4f913b706696ccbd43c62d3331cd3511caef466871d2",
-                "sha256:b5ba1f0d5f9087e03bf5958c28d421a03a4c1ad260bf81556195dffeccd979c4",
-                "sha256:b5dfcd22c6bab08dfeded8d5b44bdcb68c6f1ab261861e35c470b89074f78a70",
-                "sha256:cd01c599cf9f897b6b6c6b5d8b182557fb7d99326bcdf5d449a0fbbb4ccee4b9",
-                "sha256:e89880168c67cf4fde4506b80ee42f1537ad66ad366c101d388b3fd7d7ce2afd",
-                "sha256:ebe2bc9cb638475f5d39068d2dbe8ae1d605bb8d8d3ff281c695df1670ab3987",
-                "sha256:f89bfda7f0f66b789792ab64ce0978e4a991a0e4dd6197349d0767b0f1095b21",
-                "sha256:fc4d63da57ef0e8cd4ab45131f3fe5c286ce7dd7f032650d0fbc239c6190e167",
-                "sha256:fd634bc17b1e2d6ce716f0e43446d0d61cdadb1efcad5c56ca211c22b246ebc8"
-            ],
-            "index": "pypi",
-            "version": "==0.902"
-        },
-        "mypy-extensions": {
-            "hashes": [
-                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
-                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
-            ],
-            "version": "==0.4.3"
-        },
-        "pycodestyle": {
-            "hashes": [
-                "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068",
-                "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.7.0"
-        },
-        "pyflakes": {
-            "hashes": [
-                "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3",
-                "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.3.1"
-        },
-        "toml": {
-            "hashes": [
-                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
-                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.10.2"
-        },
-        "typed-ast": {
-            "hashes": [
-                "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace",
-                "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff",
-                "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266",
-                "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528",
-                "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6",
-                "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808",
-                "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4",
-                "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363",
-                "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341",
-                "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04",
-                "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41",
-                "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e",
-                "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3",
-                "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899",
-                "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805",
-                "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c",
-                "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c",
-                "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39",
-                "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a",
-                "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3",
-                "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7",
-                "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f",
-                "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075",
-                "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0",
-                "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40",
-                "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428",
-                "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927",
-                "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3",
-                "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f",
-                "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==1.4.3"
-        },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
-                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
-                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
-            ],
-            "index": "pypi",
-            "version": "==3.10.0.0"
-        },
-        "yapf": {
-            "hashes": [
-                "sha256:408fb9a2b254c302f49db83c59f9aa0b4b0fd0ec25be3a5c51181327922ff63d",
-                "sha256:e3a234ba8455fe201eaa649cdac872d590089a18b661e39bbac7020978dd9c2e"
-            ],
-            "index": "pypi",
-            "version": "==0.31.0"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
-                "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.1"
-        }
-    }
-}

test_runner/README.md

@@ -4,10 +4,13 @@ This directory contains integration tests.
 
 Prerequisites:
 - Python 3.6 or later
-- Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
-  packages are stale, as it commonly happens, so manual installation is not
-  recommended.
-  Run `pipenv shell` to activate the venv.
+- Python packages: pytest, psycopg2
+    - pytest 6.0 is required.
+    - __NOTE: `apt install` on Debian/Ubuntu won't work.__
+      They ship a much older version of pytest (and sometimes rename it to
+      `pytest-3`.)
+    - Install using something like this:
+        - `pip3 install pytest psycopg2` (Debian or Ubuntu)
 - Zenith and Postgres binaries
     - See the root README.md for build directions
     - Tests can be run from the git tree; or see the environment variables
@@ -69,8 +72,7 @@ The tests make heavy use of pytest fixtures. You can read about how they work he
 Essentially, this means that each time you see a fixture named as an input parameter, the function with that name will be run and passed as a parameter to the function.
 
 So this code:
-
-```python
+```
 def test_something(zenith_cli, pg_bin):
     pass
 ```
@@ -78,11 +80,9 @@ def test_something(zenith_cli, pg_bin):
 ... will run the fixtures called `zenith_cli` and `pg_bin` and deliver those results to the test function.
 
 Fixtures can't be imported using the normal python syntax. Instead, use this:
-
-```python
+```
 pytest_plugins = ("fixtures.something")
 ```
-
 That will make all the fixtures in the `fixtures/something.py` file available.
 
 Anything that's likely to be used in multiple tests should be built into a fixture.
@@ -90,15 +90,3 @@ Anything that's likely to be used in multiple tests should be built into a fixtu
 Note that fixtures can clean up after themselves if they use the `yield` syntax.
 Cleanup will happen even if the test fails (raises an unhandled exception).
 Python destructors, e.g. `__del__()` aren't recommended for cleanup.
-
-
-### Code quality
-
-Before submitting a patch, please consider:
-
-* Writing a couple of docstrings to clarify the reasoning behind a new test.
-* Running `flake8` (or a linter of your choice, e.g. `pycodestyle`) and fixing possible defects, if any.
-* Formatting the code with `yapf -r -i .` (TODO: implement an opt-in pre-commit hook for that).
-* (Optional) Typechecking the code with `mypy .`. Currently this mostly affects `fixtures/zenith_fixtures.py`.
-
-The tools can be installed with `pipenv install --dev`.

test_runner/batch_others/test_branch_behind.py

@@ -1,73 +1,67 @@
-pytest_plugins = ("fixtures.zenith_fixtures")
+import pytest
+import getpass
+import psycopg2
 
+pytest_plugins = ("fixtures.zenith_fixtures")
 
 #
 # Create a couple of branches off the main branch, at a historical point in time.
 #
 def test_branch_behind(zenith_cli, pageserver, postgres, pg_bin):
     # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind", "empty"])
+    zenith_cli.run(["branch", "test_branch_behind", "empty"]);
 
     pgmain = postgres.create_start('test_branch_behind')
     print("postgres is running on 'test_branch_behind' branch")
 
-    main_pg_conn = pgmain.connect()
+    main_pg_conn = psycopg2.connect(pgmain.connstr());
+    main_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
     main_cur = main_pg_conn.cursor()
 
     # Create table, and insert the first 100 rows
-    main_cur.execute('CREATE TABLE foo (t text)')
-    main_cur.execute('''
-        INSERT INTO foo
-            SELECT 'long string to consume some space' || g
-            FROM generate_series(1, 100) g
-    ''')
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
+    main_cur.execute('CREATE TABLE foo (t text)');
+    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100) g");
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
     lsn_a = main_cur.fetchone()[0]
     print('LSN after 100 rows: ' + lsn_a)
 
     # Insert some more rows. (This generates enough WAL to fill a few segments.)
-    main_cur.execute('''
-        INSERT INTO foo
-            SELECT 'long string to consume some space' || g
-            FROM generate_series(1, 100000) g
-    ''')
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
+    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
     lsn_b = main_cur.fetchone()[0]
     print('LSN after 100100 rows: ' + lsn_b)
 
     # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@" + lsn_a])
+    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@"+lsn_a]);
 
     # Insert many more rows. This generates enough WAL to fill a few segments.
-    main_cur.execute('''
-        INSERT INTO foo
-            SELECT 'long string to consume some space' || g
-            FROM generate_series(1, 100000) g
-    ''')
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
+    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
 
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
     lsn_c = main_cur.fetchone()[0]
     print('LSN after 200100 rows: ' + lsn_c)
 
     # Branch at the point where only 200 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@" + lsn_b])
+    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@"+lsn_b]);
 
     pg_hundred = postgres.create_start("test_branch_behind_hundred")
     pg_more = postgres.create_start("test_branch_behind_more")
 
     # On the 'hundred' branch, we should see only 100 rows
-    hundred_pg_conn = pg_hundred.connect()
+    hundred_pg_conn = psycopg2.connect(pg_hundred.connstr())
+    hundred_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
     hundred_cur = hundred_pg_conn.cursor()
-    hundred_cur.execute('SELECT count(*) FROM foo')
-    assert hundred_cur.fetchone() == (100, )
+    hundred_cur.execute('SELECT count(*) FROM foo');
+    assert(hundred_cur.fetchone()[0] == 100);
 
     # On the 'more' branch, we should see 100200 rows
-    more_pg_conn = pg_more.connect()
+    more_pg_conn = psycopg2.connect(pg_more.connstr())
+    more_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
     more_cur = more_pg_conn.cursor()
-    more_cur.execute('SELECT count(*) FROM foo')
-    assert more_cur.fetchone() == (100100, )
+    more_cur.execute('SELECT count(*) FROM foo');
+    assert(more_cur.fetchone()[0] == 100100);
 
     # All the rows are visible on the main branch
-    main_cur.execute('SELECT count(*) FROM foo')
-    assert main_cur.fetchone() == (200100, )
+    main_cur.execute('SELECT count(*) FROM foo');
+    assert(main_cur.fetchone()[0] == 200100);

test_runner/batch_others/test_config.py

@@ -1,4 +1,7 @@
-from contextlib import closing
+import pytest
+import os
+import getpass
+import psycopg2
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
@@ -8,22 +11,20 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #
 def test_config(zenith_cli, pageserver, postgres, pg_bin):
     # Create a branch for us
-    zenith_cli.run(["branch", "test_config", "empty"])
+    zenith_cli.run(["branch", "test_config", "empty"]);
 
     # change config
-    pg = postgres.create_start('test_config', config_lines=['log_min_messages=debug1'])
+    pg = postgres.create_start('test_config', ['log_min_messages=debug1'])
     print('postgres is running on test_config branch')
 
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            cur.execute('''
-                SELECT setting
-                FROM pg_settings
-                WHERE
-                    source != 'default'
-                    AND source != 'override'
-                    AND name = 'log_min_messages'
-            ''')
+    pg_conn = psycopg2.connect(pg.connstr())
+    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = pg_conn.cursor()
 
-            # check that config change was applied
-            assert cur.fetchone() == ('debug1', )
+    #check that config change was applied
+    cur.execute('SELECT name, setting from pg_settings WHERE source!=%s and source!=%s', ("default","override",))
+    for record in cur:
+        if record[0] == 'log_min_messages':
+            assert(record[1] == 'debug1')
+
+    pg_conn.close()

test_runner/batch_others/test_createdb.py

@@ -1,32 +1,37 @@
-from contextlib import closing
+import pytest
+import getpass
+import psycopg2
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
-
 #
 # Test CREATE DATABASE when there have been relmapper changes
 #
 def test_createdb(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_createdb", "empty"])
+    zenith_cli.run(["branch", "test_createdb", "empty"]);
 
     pg = postgres.create_start('test_createdb')
     print("postgres is running on 'test_createdb' branch")
 
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # Cause a 'relmapper' change in the original branch
-            cur.execute('VACUUM FULL pg_class')
+    conn = psycopg2.connect(pg.connstr());
+    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = conn.cursor()
 
-            cur.execute('CREATE DATABASE foodb')
+    # Cause a 'relmapper' change in the original branch
+    cur.execute('VACUUM FULL pg_class');
 
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn = cur.fetchone()[0]
+    cur.execute('CREATE DATABASE foodb');
+
+    cur.execute('SELECT pg_current_wal_insert_lsn()');
+    lsn = cur.fetchone()[0]
+
+    conn.close();
 
     # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@"+lsn]);
 
     pg2 = postgres.create_start('test_createdb2')
 
     # Test that you can connect to the new database on both branches
-    for db in (pg, pg2):
-        db.connect(dbname='foodb').close()
+    conn = psycopg2.connect(pg.connstr('foodb'));
+    conn2 = psycopg2.connect(pg2.connstr('foodb'));

test_runner/batch_others/test_createuser.py

@@ -1,31 +0,0 @@
-from contextlib import closing
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-#
-# Test CREATE USER to check shared catalog restore
-#
-def test_createuser(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_createuser", "empty"])
-
-    pg = postgres.create_start('test_createuser')
-    print("postgres is running on 'test_createuser' branch")
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # Cause a 'relmapper' change in the original branch
-            cur.execute('CREATE USER testuser with password %s', ('testpwd', ))
-
-            cur.execute('CHECKPOINT')
-
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn = cur.fetchone()[0]
-
-    # Create a branch
-    zenith_cli.run(["branch", "test_createuser2", "test_createuser@" + lsn])
-
-    pg2 = postgres.create_start('test_createuser2')
-
-    # Test that you can connect to new branch as a new user
-    assert pg2.safe_psql('select current_user', username='testuser') == [('testuser', )]

test_runner/batch_others/test_multixact.py

@@ -1,5 +1,9 @@
-pytest_plugins = ("fixtures.zenith_fixtures")
+import pytest
+import os
+import psycopg2
+import multiprocessing
 
+pytest_plugins = ("fixtures.zenith_fixtures")
 
 #
 # Test multixact state after branching
@@ -7,57 +11,61 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 # it only checks next_multixact_id field in restored pg_control,
 # since we don't have functions to check multixact internals.
 #
+
+def runQuery(connstr):
+    con = psycopg2.connect(connstr) 
+    con.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = con.cursor()
+    cur.execute('select * from t1 for key share;') 
+
+
 def test_multixact(pageserver, postgres, pg_bin, zenith_cli, base_dir):
+
     # Create a branch for us
     zenith_cli.run(["branch", "test_multixact", "empty"])
     pg = postgres.create_start('test_multixact')
 
     print("postgres is running on 'test_multixact' branch")
-    pg_conn = pg.connect()
+    pg_conn = psycopg2.connect(pg.connstr())
+    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
     cur = pg_conn.cursor()
 
-    cur.execute('''
-        CREATE TABLE t1(i int primary key);
-        INSERT INTO t1 select * from generate_series(1, 100);
-    ''')
+    cur.execute('CREATE TABLE t1(i int primary key);'
+    'INSERT INTO t1 select * from generate_series(1,100);')
 
-    cur.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
+    cur.execute('SELECT next_multixact_id FROM pg_control_checkpoint();')
     next_multixact_id_old = cur.fetchone()[0]
 
     # Lock entries in parallel connections to set multixact
     nclients = 3
-    connections = []
-    for i in range(nclients):
-        # Do not turn on autocommit. We want to hold the key-share locks.
-        conn = pg.connect(autocommit=False)
-        conn.cursor().execute('select * from t1 for key share')
-        connections.append(conn)
-
-    # We should have a multixact now. We can close the connections.
-    for c in connections:
-        c.close()
+    pool = multiprocessing.Pool(nclients)
+    args = [pg.connstr()] * nclients
+    pool.map(runQuery, args)
+    pool.close()
+    pool.join()
 
     # force wal flush
     cur.execute('checkpoint')
 
-    cur.execute('SELECT next_multixact_id, pg_current_wal_flush_lsn() FROM pg_control_checkpoint()')
+    cur.execute('SELECT next_multixact_id, pg_current_wal_flush_lsn() FROM pg_control_checkpoint();')
     res = cur.fetchone()
     next_multixact_id = res[0]
     lsn = res[1]
 
     # Ensure that we did lock some tuples
-    assert int(next_multixact_id) > int(next_multixact_id_old)
+    assert(int(next_multixact_id) > int(next_multixact_id_old))
 
     # Branch at this point
-    zenith_cli.run(["branch", "test_multixact_new", "test_multixact@" + lsn])
+    zenith_cli.run(["branch", "test_multixact_new", "test_multixact@"+lsn]);
     pg_new = postgres.create_start('test_multixact_new')
 
     print("postgres is running on 'test_multixact_new' branch")
-    pg_new_conn = pg_new.connect()
+    pg_new_conn = psycopg2.connect(pg_new.connstr())
+    pg_new_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
     cur_new = pg_new_conn.cursor()
 
-    cur_new.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
+    cur_new.execute('SELECT next_multixact_id FROM pg_control_checkpoint();')
     next_multixact_id_new = cur_new.fetchone()[0]
 
     # Check that we restored pg_controlfile correctly
-    assert next_multixact_id_new == next_multixact_id
+    assert(next_multixact_id_new == next_multixact_id)

test_runner/batch_others/test_pageserver_api.py

@@ -1,23 +1,29 @@
+import pytest
+import psycopg2
+import getpass
 import json
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
-
 def test_status(pageserver):
-    assert pageserver.safe_psql('status') == [
-        ('hello world', ),
-    ]
-
+    pg_conn = psycopg2.connect(pageserver.connstr())
+    pg_conn.autocommit = True
+    cur = pg_conn.cursor()
+    cur.execute('status;')
+    assert cur.fetchone() == ('hello world',)
+    pg_conn.close()
 
 def test_branch_list(pageserver, zenith_cli):
+
     # Create a branch for us
-    zenith_cli.run(["branch", "test_branch_list_main", "empty"])
+    zenith_cli.run(["branch", "test_branch_list_main", "empty"]);
 
-    conn = pageserver.connect()
-    cur = conn.cursor()
+    page_server_conn = psycopg2.connect(pageserver.connstr())
+    page_server_conn.autocommit = True
+    page_server_cur = page_server_conn.cursor()
 
-    cur.execute('branch_list')
-    branches = json.loads(cur.fetchone()[0])
+    page_server_cur.execute('branch_list;')
+    branches = json.loads(page_server_cur.fetchone()[0])
     # Filter out branches created by other tests
     branches = [x for x in branches if x['name'].startswith('test_branch_list')]
 
@@ -32,8 +38,8 @@ def test_branch_list(pageserver, zenith_cli):
     zenith_cli.run(['branch', 'test_branch_list_experimental', 'test_branch_list_main'])
     zenith_cli.run(['pg', 'create', 'test_branch_list_experimental'])
 
-    cur.execute('branch_list')
-    new_branches = json.loads(cur.fetchone()[0])
+    page_server_cur.execute('branch_list;')
+    new_branches = json.loads(page_server_cur.fetchone()[0])
     # Filter out branches created by other tests
     new_branches = [x for x in new_branches if x['name'].startswith('test_branch_list')]
     assert len(new_branches) == 2
@@ -45,4 +51,4 @@ def test_branch_list(pageserver, zenith_cli):
     # TODO: do the LSNs have to match here?
     assert new_branches[1] == branches[0]
 
-    conn.close()
+    page_server_conn.close()

test_runner/batch_others/test_pgbench.py

@@ -1,15 +1,17 @@
+import pytest
+
 pytest_plugins = ("fixtures.zenith_fixtures")
 
 
 def test_pgbench(pageserver, postgres, pg_bin, zenith_cli):
 
     # Create a branch for us
-    zenith_cli.run(["branch", "test_pgbench", "empty"])
+    zenith_cli.run(["branch", "test_pgbench", "empty"]);
 
     pg = postgres.create_start('test_pgbench')
     print("postgres is running on 'test_pgbench' branch")
 
-    connstr = pg.connstr()
+    connstr = pg.connstr();
 
     pg_bin.run_capture(['pgbench', '-i', connstr])
     pg_bin.run_capture(['pgbench'] + '-c 10 -T 5 -P 1 -M prepared'.split() + [connstr])

test_runner/batch_others/test_restart_compute.py

@@ -1,42 +0,0 @@
-from contextlib import closing
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-#
-# Test restarting and recreating a postgres instance
-#
-def test_restart_compute(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_restart_compute", "empty"])
-
-    pg = postgres.create_start('test_restart_compute')
-    print("postgres is running on 'test_restart_compute' branch")
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # Create table, and insert a row
-            cur.execute('CREATE TABLE foo (t text)')
-            cur.execute("INSERT INTO foo VALUES ('bar')")
-
-    # Stop and restart the Postgres instance
-    pg.stop().start()
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # We can still see the row
-            cur.execute('SELECT count(*) FROM foo')
-            assert cur.fetchone() == (1, )
-
-            # Insert another row
-            cur.execute("INSERT INTO foo VALUES ('bar2')")
-            cur.execute('SELECT count(*) FROM foo')
-            assert cur.fetchone() == (2, )
-
-    # Stop, and destroy the Postgres instance. Then recreate and restart it.
-    pg.stop_and_destroy().create_start('test_restart_compute')
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # We can still see the rows
-            cur.execute('SELECT count(*) FROM foo')
-            assert cur.fetchone() == (2, )

test_runner/batch_others/test_twophase.py

@@ -1,46 +1,50 @@
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
 #
 # Test branching, when a transaction is in prepared state
 #
-def test_twophase(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_twophase", "empty"])
+import pytest
+import getpass
+import psycopg2
 
-    pg = postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5'])
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+def test_twophase(zenith_cli, pageserver, postgres, pg_bin):
+    zenith_cli.run(["branch", "test_twophase", "empty"]);
+
+    pg = postgres.create_start('test_twophase', ['max_prepared_transactions=5'])
     print("postgres is running on 'test_twophase' branch")
 
-    conn = pg.connect()
+    conn = psycopg2.connect(pg.connstr());
+    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
     cur = conn.cursor()
 
-    cur.execute('CREATE TABLE foo (t text)')
+    cur.execute('CREATE TABLE foo (t text)');
 
     # Prepare a transaction that will insert a row
-    cur.execute('BEGIN')
-    cur.execute("INSERT INTO foo VALUES ('one')")
-    cur.execute("PREPARE TRANSACTION 'insert_one'")
+    cur.execute('BEGIN');
+    cur.execute("INSERT INTO foo VALUES ('one')");
+    cur.execute("PREPARE TRANSACTION 'insert_one'");
 
     # Prepare another transaction that will insert a row
-    cur.execute('BEGIN')
-    cur.execute("INSERT INTO foo VALUES ('two')")
-    cur.execute("PREPARE TRANSACTION 'insert_two'")
+    cur.execute('BEGIN');
+    cur.execute("INSERT INTO foo VALUES ('two')");
+    cur.execute("PREPARE TRANSACTION 'insert_two'");
 
     # Create a branch with the transaction in prepared state
-    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"])
+    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"]);
 
-    pg2 = postgres.create_start('test_twophase_prepared',
-                                config_lines=['max_prepared_transactions=5'])
-    conn2 = pg2.connect()
+    pg2 = postgres.create_start('test_twophase_prepared', ['max_prepared_transactions=5'])
+    conn2 = psycopg2.connect(pg2.connstr());
+    conn2.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
     cur2 = conn2.cursor()
 
     # On the new branch, commit one of the prepared transactions, abort the other one.
-    cur2.execute("COMMIT PREPARED 'insert_one'")
-    cur2.execute("ROLLBACK PREPARED 'insert_two'")
+    cur2.execute("COMMIT PREPARED 'insert_one'");
+    cur2.execute("ROLLBACK PREPARED 'insert_two'");
 
-    cur2.execute('SELECT * FROM foo')
-    assert cur2.fetchall() == [('one', )]
+    cur2.execute('SELECT * FROM foo');
+    assert(cur2.fetchall() == [('one',)]);
 
     # Neither insert is visible on the original branch, the transactions are still
     # in prepared state there.
-    cur.execute('SELECT * FROM foo')
-    assert cur.fetchall() == []
+    cur.execute('SELECT * FROM foo');
+    assert(cur.fetchall() == []);

test_runner/batch_others/test_wal_acceptor.py

@@ -1,219 +0,0 @@
-import pytest
-import random
-import time
-import json
-
-from contextlib import closing
-from multiprocessing import Process, Value
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-# basic test, write something in setup with wal acceptors, ensure that commits
-# succeed and data is written
-def test_normal_work(zenith_cli, pageserver, postgres, wa_factory):
-    zenith_cli.run(["branch", "test_wal_acceptors_normal_work", "empty"])
-    wa_factory.start_n_new(3)
-    pg = postgres.create_start('test_wal_acceptors_normal_work',
-                               wal_acceptors=wa_factory.get_connstrs())
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # we rely upon autocommit after each statement
-            # as waiting for acceptors happens there
-            cur.execute('CREATE TABLE t(key int primary key, value text)')
-            cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
-            cur.execute('SELECT sum(key) FROM t')
-            assert cur.fetchone() == (5000050000, )
-
-
-# Run page server and multiple acceptors, and multiple compute nodes running
-# against different timelines.
-def test_many_timelines(zenith_cli, pageserver, postgres, wa_factory):
-    n_timelines = 2
-
-    wa_factory.start_n_new(3)
-
-    branches = ["test_wal_acceptors_many_timelines_{}".format(tlin) for tlin in range(n_timelines)]
-
-    # start postgres on each timeline
-    pgs = []
-    for branch in branches:
-        zenith_cli.run(["branch", branch, "empty"])
-        pgs.append(postgres.create_start(branch, wal_acceptors=wa_factory.get_connstrs()))
-
-    # Do everything in different loops to have actions on different timelines
-    # interleaved.
-    # create schema
-    for pg in pgs:
-        pg.safe_psql("CREATE TABLE t(key int primary key, value text)")
-
-    # Populate data
-    for pg in pgs:
-        pg.safe_psql("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
-
-    # Check data
-    for pg in pgs:
-        res = pg.safe_psql("SELECT sum(key) FROM t")
-        assert res[0] == (5000050000, )
-
-
-# Check that dead minority doesn't prevent the commits: execute insert n_inserts
-# times, with fault_probability chance of getting a wal acceptor down or up
-# along the way. 2 of 3 are always alive, so the work keeps going.
-def test_restarts(zenith_cli, pageserver, postgres, wa_factory):
-    fault_probability = 0.01
-    n_inserts = 1000
-    n_acceptors = 3
-
-    wa_factory.start_n_new(n_acceptors)
-
-    zenith_cli.run(["branch", "test_wal_acceptors_restarts", "empty"])
-    pg = postgres.create_start('test_wal_acceptors_restarts',
-                               wal_acceptors=wa_factory.get_connstrs())
-
-    # we rely upon autocommit after each statement
-    # as waiting for acceptors happens there
-    pg_conn = pg.connect()
-    cur = pg_conn.cursor()
-
-    failed_node = None
-    cur.execute('CREATE TABLE t(key int primary key, value text)')
-    for i in range(n_inserts):
-        cur.execute("INSERT INTO t values (%s, 'payload');", (i + 1, ))
-
-        if random.random() <= fault_probability:
-            if failed_node is None:
-                failed_node = wa_factory.instances[random.randrange(0, n_acceptors)]
-                failed_node.stop()
-            else:
-                failed_node.start()
-                failed_node = None
-    cur.execute('SELECT sum(key) FROM t')
-    assert cur.fetchone() == (500500, )
-
-
-start_delay_sec = 2
-
-
-def delayed_wal_acceptor_start(wa):
-    time.sleep(start_delay_sec)
-    wa.start()
-
-
-# When majority of acceptors is offline, commits are expected to be frozen
-def test_unavailability(zenith_cli, pageserver, postgres, wa_factory):
-    wa_factory.start_n_new(2)
-
-    zenith_cli.run(["branch", "test_wal_acceptors_unavailability", "empty"])
-    pg = postgres.create_start('test_wal_acceptors_unavailability',
-                               wal_acceptors=wa_factory.get_connstrs())
-
-    # we rely upon autocommit after each statement
-    # as waiting for acceptors happens there
-    pg_conn = pg.connect()
-    cur = pg_conn.cursor()
-
-    # check basic work with table
-    cur.execute('CREATE TABLE t(key int primary key, value text)')
-    cur.execute("INSERT INTO t values (1, 'payload')")
-
-    # shutdown one of two acceptors, that is, majority
-    wa_factory.instances[0].stop()
-
-    proc = Process(target=delayed_wal_acceptor_start, args=(wa_factory.instances[0], ))
-    proc.start()
-
-    start = time.time()
-    cur.execute("INSERT INTO t values (2, 'payload')")
-    # ensure that the query above was hanging while acceptor was down
-    assert (time.time() - start) >= start_delay_sec
-    proc.join()
-
-    # for the world's balance, do the same with second acceptor
-    wa_factory.instances[1].stop()
-
-    proc = Process(target=delayed_wal_acceptor_start, args=(wa_factory.instances[1], ))
-    proc.start()
-
-    start = time.time()
-    cur.execute("INSERT INTO t values (3, 'payload')")
-    # ensure that the query above was hanging while acceptor was down
-    assert (time.time() - start) >= start_delay_sec
-    proc.join()
-
-    cur.execute("INSERT INTO t values (4, 'payload')")
-
-    cur.execute('SELECT sum(key) FROM t')
-    assert cur.fetchone() == (10, )
-
-
-# shut down random subset of acceptors, sleep, wake them up, rinse, repeat
-def xmas_garland(acceptors, stop):
-    while not bool(stop.value):
-        victims = []
-        for wa in acceptors:
-            if random.random() >= 0.5:
-                victims.append(wa)
-        for v in victims:
-            v.stop()
-        time.sleep(1)
-        for v in victims:
-            v.start()
-        time.sleep(1)
-
-
-# value which gets unset on exit
-@pytest.fixture
-def stop_value():
-    stop = Value('i', 0)
-    yield stop
-    stop.value = 1
-
-
-# do inserts while concurrently getting up/down subsets of acceptors
-def test_race_conditions(zenith_cli, pageserver, postgres, wa_factory, stop_value):
-
-    wa_factory.start_n_new(3)
-
-    zenith_cli.run(["branch", "test_wal_acceptors_race_conditions", "empty"])
-    pg = postgres.create_start('test_wal_acceptors_race_conditions',
-                               wal_acceptors=wa_factory.get_connstrs())
-
-    # we rely upon autocommit after each statement
-    # as waiting for acceptors happens there
-    pg_conn = pg.connect()
-    cur = pg_conn.cursor()
-
-    cur.execute('CREATE TABLE t(key int primary key, value text)')
-
-    proc = Process(target=xmas_garland, args=(wa_factory.instances, stop_value))
-    proc.start()
-
-    for i in range(1000):
-        cur.execute("INSERT INTO t values (%s, 'payload');", (i + 1, ))
-
-    cur.execute('SELECT sum(key) FROM t')
-    assert cur.fetchone() == (500500, )
-
-    stop_value.value = 1
-    proc.join()
-
-def test_state(zenith_cli, pageserver, postgres, wa_factory):
-    wa_factory.start_n_new(1)
-
-    zenith_cli.run(["branch", "test_wal_acceptors_state", "empty"])
-    pg = postgres.create_start('test_wal_acceptors_state',
-                               wal_acceptors=wa_factory.get_connstrs())
-
-    # learn zenith timeline from compute
-    ztli = pg.safe_psql("show zenith.zenith_timeline")[0][0]
-
-    pg.safe_psql("create table t(i int)")
-
-    pg.stop().start()
-    pg.safe_psql("insert into t values(10)")
-
-    wa = wa_factory.instances[0]
-    state = wa.safe_psql("state", options="'-c ztimelineid={}'".format(ztli), sslmode='disable')[0][0]
-    print(state)

test_runner/batch_others/test_zenith_cli.py

@@ -1,48 +1,49 @@
+import pytest
+import psycopg2
 import json
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
-
 def helper_compare_branch_list(page_server_cur, zenith_cli):
     """
     Compare branches list returned by CLI and directly via API.
     Filters out branches created by other tests.
     """
 
-    page_server_cur.execute('branch_list')
+    page_server_cur.execute('branch_list;')
     branches_api = sorted(map(lambda b: b['name'], json.loads(page_server_cur.fetchone()[0])))
     branches_api = [b for b in branches_api if b.startswith('test_cli_') or b in ('empty', 'main')]
 
-    res = zenith_cli.run(["branch"])
-    assert res.stderr == ''
+    res = zenith_cli.run(["branch"]);
+    assert(res.stderr == '')
     branches_cli = sorted(map(lambda b: b.split(':')[-1].strip(), res.stdout.strip().split("\n")))
     branches_cli = [b for b in branches_cli if b.startswith('test_cli_') or b in ('empty', 'main')]
 
-    assert branches_api == branches_cli
-
+    assert(branches_api == branches_cli)
 
 def test_cli_branch_list(pageserver, zenith_cli):
 
-    page_server_conn = pageserver.connect()
+    page_server_conn = psycopg2.connect(pageserver.connstr())
+    page_server_conn.autocommit = True
     page_server_cur = page_server_conn.cursor()
 
     # Initial sanity check
     helper_compare_branch_list(page_server_cur, zenith_cli)
 
     # Create a branch for us
-    res = zenith_cli.run(["branch", "test_cli_branch_list_main", "main"])
-    assert res.stderr == ''
+    res = zenith_cli.run(["branch", "test_cli_branch_list_main", "main"]);
+    assert(res.stderr == '')
     helper_compare_branch_list(page_server_cur, zenith_cli)
 
     # Create a nested branch
-    res = zenith_cli.run(["branch", "test_cli_branch_list_nested", "test_cli_branch_list_main"])
-    assert res.stderr == ''
+    res = zenith_cli.run(["branch", "test_cli_branch_list_nested", "test_cli_branch_list_main"]);
+    assert(res.stderr == '')
     helper_compare_branch_list(page_server_cur, zenith_cli)
 
     # Check that all new branches are visible via CLI
-    res = zenith_cli.run(["branch"])
-    assert res.stderr == ''
+    res = zenith_cli.run(["branch"]);
+    assert(res.stderr == '')
     branches_cli = sorted(map(lambda b: b.split(':')[-1].strip(), res.stdout.strip().split("\n")))
 
-    assert 'test_cli_branch_list_main' in branches_cli
-    assert 'test_cli_branch_list_nested' in branches_cli
+    assert('test_cli_branch_list_main' in branches_cli)
+    assert('test_cli_branch_list_nested' in branches_cli)

test_runner/batch_pg_regress/test_isolation.py

@@ -1,50 +0,0 @@
-import os
-
-from fixtures.utils import mkdir_if_needed
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-def test_isolation(pageserver, postgres, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir,
-                   base_dir, capsys):
-
-    # Create a branch for us
-    zenith_cli.run(["branch", "test_isolation", "empty"])
-
-    # Connect to postgres and create a database called "regression".
-    # isolation tests use prepared transactions, so enable them
-    pg = postgres.create_start('test_isolation', config_lines=['max_prepared_transactions=100'])
-    pg.safe_psql('CREATE DATABASE isolation_regression')
-
-    # Create some local directories for pg_isolation_regress to run in.
-    runpath = os.path.join(test_output_dir, 'regress')
-    mkdir_if_needed(runpath)
-    mkdir_if_needed(os.path.join(runpath, 'testtablespace'))
-
-    # Compute all the file locations that pg_isolation_regress will need.
-    build_path = os.path.join(pg_distrib_dir, 'build/src/test/isolation')
-    src_path = os.path.join(base_dir, 'vendor/postgres/src/test/isolation')
-    bindir = os.path.join(pg_distrib_dir, 'bin')
-    schedule = os.path.join(src_path, 'isolation_schedule')
-    pg_isolation_regress = os.path.join(build_path, 'pg_isolation_regress')
-
-    pg_isolation_regress_command = [
-        pg_isolation_regress,
-        '--use-existing',
-        '--bindir={}'.format(bindir),
-        '--dlpath={}'.format(build_path),
-        '--inputdir={}'.format(src_path),
-        '--schedule={}'.format(schedule),
-    ]
-
-    env = {
-        'PGPORT': str(pg.port),
-        'PGUSER': pg.username,
-        'PGHOST': pg.host,
-    }
-
-    # Run the command.
-    # We don't capture the output. It's not too chatty, and it always
-    # logs the exact same data to `regression.out` anyway.
-    with capsys.disabled():
-        pg_bin.run(pg_isolation_regress_command, env=env, cwd=runpath)

test_runner/batch_pg_regress/test_pg_regress.py

@@ -1,19 +1,28 @@
-import os
-
+import pytest
 from fixtures.utils import mkdir_if_needed
+import getpass
+import os
+import psycopg2
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
+# FIXME: put host + port in a fixture
+HOST = 'localhost'
+PORT = 55432
 
-def test_pg_regress(pageserver, postgres, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir,
-                    base_dir, capsys):
+
+def test_pg_regress(pageserver, postgres, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir, base_dir, capsys):
 
     # Create a branch for us
-    zenith_cli.run(["branch", "test_pg_regress", "empty"])
+    zenith_cli.run(["branch", "test_pg_regress", "empty"]);
 
     # Connect to postgres and create a database called "regression".
     pg = postgres.create_start('test_pg_regress')
-    pg.safe_psql('CREATE DATABASE regression')
+    pg_conn = psycopg2.connect(pg.connstr())
+    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = pg_conn.cursor()
+    cur.execute('CREATE DATABASE regression')
+    pg_conn.close()
 
     # Create some local directories for pg_regress to run in.
     runpath = os.path.join(test_output_dir, 'regress')
@@ -21,8 +30,10 @@ def test_pg_regress(pageserver, postgres, pg_bin, zenith_cli, test_output_dir, p
     mkdir_if_needed(os.path.join(runpath, 'testtablespace'))
 
     # Compute all the file locations that pg_regress will need.
-    build_path = os.path.join(pg_distrib_dir, 'build/src/test/regress')
-    src_path = os.path.join(base_dir, 'vendor/postgres/src/test/regress')
+    build_path = os.path.join(
+        pg_distrib_dir, 'build/src/test/regress')
+    src_path = os.path.join(
+        base_dir, 'vendor/postgres/src/test/regress')
     bindir = os.path.join(pg_distrib_dir, 'bin')
     schedule = os.path.join(src_path, 'parallel_schedule')
     pg_regress = os.path.join(build_path, 'pg_regress')

test_runner/batch_pg_regress/test_zehith_regress.py

@@ -1,19 +1,28 @@
-import os
-
+import pytest
 from fixtures.utils import mkdir_if_needed
+import getpass
+import os
+import psycopg2
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
+# FIXME: put host + port in a fixture
+HOST = 'localhost'
+PORT = 55432
 
-def test_zenith_regress(pageserver, postgres, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir,
-                        base_dir, capsys):
+
+def test_zenith_regress(pageserver, postgres, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir, base_dir, capsys):
 
     # Create a branch for us
-    zenith_cli.run(["branch", "test_zenith_regress", "empty"])
+    zenith_cli.run(["branch", "test_zenith_regress", "empty"]);
 
     # Connect to postgres and create a database called "regression".
     pg = postgres.create_start('test_zenith_regress')
-    pg.safe_psql('CREATE DATABASE regression')
+    pg_conn = psycopg2.connect(pg.connstr())
+    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = pg_conn.cursor()
+    cur.execute('CREATE DATABASE regression')
+    pg_conn.close()
 
     # Create some local directories for pg_regress to run in.
     runpath = os.path.join(test_output_dir, 'regress')
@@ -22,8 +31,10 @@ def test_zenith_regress(pageserver, postgres, pg_bin, zenith_cli, test_output_di
 
     # Compute all the file locations that pg_regress will need.
     # This test runs zenith specific tests
-    build_path = os.path.join(pg_distrib_dir, 'build/src/test/regress')
-    src_path = os.path.join(base_dir, 'test_runner/zenith_regress')
+    build_path = os.path.join(
+        pg_distrib_dir, 'build/src/test/regress')
+    src_path = os.path.join(
+        base_dir, 'test_runner/zenith_regress')
     bindir = os.path.join(pg_distrib_dir, 'bin')
     schedule = os.path.join(src_path, 'parallel_schedule')
     pg_regress = os.path.join(build_path, 'pg_regress')

test_runner/fixtures/utils.py

@@ -1,15 +1,13 @@
+
 import os
 import subprocess
 
-from typing import Any, List
-
-
-def get_self_dir() -> str:
+def get_self_dir():
     """ Get the path to the directory where this script lives. """
     return os.path.dirname(os.path.abspath(__file__))
 
 
-def mkdir_if_needed(path: str) -> None:
+def mkdir_if_needed(path):
     """ Create a directory if it doesn't already exist
 
     Note this won't try to create intermediate directories.
@@ -20,7 +18,7 @@ def mkdir_if_needed(path: str) -> None:
     os.mkdir(path)
 
 
-def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> None:
+def subprocess_capture(capture_dir, cmd, **kwargs):
     """ Run a process and capture its output
 
     Output will go to files named "cmd_NNN.stdout" and "cmd_NNN.stderr"
@@ -44,7 +42,7 @@ def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> None:
 _global_counter = 0
 
 
-def global_counter() -> int:
+def global_counter():
     """ A really dumb global counter.
 
     This is useful for giving output files a unique number, so if we run the

test_runner/fixtures/zenith_fixtures.py

@@ -3,18 +3,11 @@ import os
 import psycopg2
 import pytest
 import shutil
-import signal
 import subprocess
+import sys
+from .utils import (get_self_dir, mkdir_if_needed,
+                    subprocess_capture, global_counter)
 
-from contextlib import closing
-from pathlib import Path
-
-# Type-related stuff
-from psycopg2.extensions import connection as PgConnection
-from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast
-from typing_extensions import Literal
-
-from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
 """
 This file contains pytest fixtures. A fixture is a test resource that can be
 summoned by placing its name in the test's arguments.
@@ -27,28 +20,22 @@ ZENITH_BIN, POSTGRES_DISTRIB_DIR, etc. See README.md for more information.
 
 To use fixtures in a test file, add this line of code:
 
->>> pytest_plugins = ("fixtures.zenith_fixtures")
+    pytest_plugins = ("fixtures.zenith_fixtures")
 
 Don't import functions from this file, or pytest will emit warnings. Instead
 put directly-importable functions into utils.py or another separate file.
 """
 
-Env = Dict[str, str]
-Fn = TypeVar('Fn', bound=Callable[..., Any])
-
 DEFAULT_OUTPUT_DIR = 'test_output'
 DEFAULT_POSTGRES_DIR = 'tmp_install'
 
-DEFAULT_PAGESERVER_PORT = 64000
 
-
-def determine_scope(fixture_name: str, config: Any) -> str:
+def determine_scope(fixture_name, config):
     return 'session'
 
 
-def zenfixture(func: Fn) -> Fn:
-    """
-    This is a python decorator for fixtures with a flexible scope.
+def zenfixture(func):
+    """ This is a python decorator for fixtures with a flexible scope.
 
     By default every test function will set up and tear down a new
     database. In pytest, this is called fixtures "function" scope.
@@ -56,18 +43,18 @@ def zenfixture(func: Fn) -> Fn:
     If the environment variable TEST_SHARED_FIXTURES is set, then all
     tests will share the same database. State, logs, etc. will be
     stored in a directory called "shared".
+
     """
-
-    scope: Literal['session', 'function'] = \
-        'function' if os.environ.get('TEST_SHARED_FIXTURES') is None else 'session'
-
+    if os.environ.get('TEST_SHARED_FIXTURES') is None:
+        scope = 'function'
+    else:
+        scope = 'session'
     return pytest.fixture(func, scope=scope)
 
 
 @pytest.fixture(autouse=True, scope='session')
-def safety_check() -> None:
+def safety_check():
     """ Ensure that no unwanted daemons are running before we start testing. """
-
     # does not use -c as it is not supported on macOS
     cmd = ['pgrep', 'pageserver|postgres|wal_acceptor']
     result = subprocess.run(cmd, stdout=subprocess.DEVNULL)
@@ -78,60 +65,14 @@ def safety_check() -> None:
         raise Exception('found interfering processes running')
 
 
-class PgProtocol:
-    """ Reusable connection logic """
-    def __init__(self, host: str, port: int, username: Optional[str] = None):
-        self.host = host
-        self.port = port
-        self.username = username or getpass.getuser()
-
-    def connstr(self, *, dbname: str = 'postgres', username: Optional[str] = None, **kwargs) -> str:
-        """
-        Build a libpq connection string for the Postgres instance.
-        """
-
-        username = username or self.username
-        connstr = f'host={self.host} port={self.port} user={username} dbname={dbname}'
-        for k, v in kwargs.items():
-            connstr += " {}={}".format(k, v)
-        print("connstr is {}".format(connstr))
-        return connstr
-
-    # autocommit=True here by default because that's what we need most of the time
-    def connect(self, *, autocommit=True, **kwargs: Any) -> PgConnection:
-        """
-        Connect to the node.
-        Returns psycopg2's connection object.
-        This method passes all extra params to connstr.
-        """
-
-        conn = psycopg2.connect(self.connstr(**kwargs))
-        # WARNING: this setting affects *all* tests!
-        conn.autocommit = autocommit
-        return conn
-
-    def safe_psql(self, query: str, **kwargs: Any) -> List[Any]:
-        """
-        Execute query against the node and return all rows.
-        This method passes all extra params to connstr.
-        """
-
-        with closing(self.connect(**kwargs)) as conn:
-            with conn.cursor() as cur:
-                cur.execute(query)
-                if cur.description is None:
-                    return []  # query didn't return data
-                return cast(List[Any], cur.fetchall())
-
-
 class ZenithCli:
-    """
-    An object representing the CLI binary named "zenith".
+    """ An object representing the CLI binary named "zenith".
 
     We also store an environment that will tell the CLI to operate
     on a particular ZENITH_REPO_DIR.
     """
-    def __init__(self, binpath: str, repo_dir: str, pg_distrib_dir: str):
+
+    def __init__(self, binpath, repo_dir, pg_distrib_dir):
         assert os.path.isdir(binpath)
         self.binpath = binpath
         self.bin_zenith = os.path.join(binpath, 'zenith')
@@ -139,259 +80,148 @@ class ZenithCli:
         self.env['ZENITH_REPO_DIR'] = repo_dir
         self.env['POSTGRES_DISTRIB_DIR'] = pg_distrib_dir
 
-    def run(self, arguments: List[str]) -> Any:
-        """
-        Run "zenith" with the specified arguments.
+    def run(self, arguments):
+        """ Run "zenith" with the specified arguments.
 
-        Arguments must be in list form, e.g. ['pg', 'create']
+        arguments must be in list form, e.g. ['pg', 'create']
 
         Return both stdout and stderr, which can be accessed as
 
-        >>> result = zenith_cli.run(...)
-        >>> assert result.stderr == ""
-        >>> print(result.stdout)
+        result = zenith_cli.run(...)
+        assert(result.stderr == "")
+        print(result.stdout)
+
         """
-
         assert type(arguments) == list
-
         args = [self.bin_zenith] + arguments
         print('Running command "{}"'.format(' '.join(args)))
-        return subprocess.run(args,
-                              env=self.env,
-                              check=True,
-                              universal_newlines=True,
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE)
+        return subprocess.run(args, env=self.env, check=True, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
 
 @zenfixture
-def zenith_cli(zenith_binpath: str, repo_dir: str, pg_distrib_dir: str) -> ZenithCli:
+def zenith_cli(zenith_binpath, repo_dir, pg_distrib_dir):
     return ZenithCli(zenith_binpath, repo_dir, pg_distrib_dir)
 
 
-class ZenithPageserver(PgProtocol):
+class ZenithPageserver:
     """ An object representing a running pageserver. """
-    def __init__(self, zenith_cli: ZenithCli):
-        super().__init__(host='localhost', port=DEFAULT_PAGESERVER_PORT)
 
+    def __init__(self, zenith_cli):
         self.zenith_cli = zenith_cli
         self.running = False
 
-    def init(self) -> 'ZenithPageserver':
-        """
-        Initialize the repository, i.e. run "zenith init".
-        Returns self.
-        """
-
+    # Initialize the repository, i.e. run "zenith init"
+    def init(self):
         self.zenith_cli.run(['init'])
-        return self
-
-    def start(self) -> 'ZenithPageserver':
-        """
-        Start the page server.
-        Returns self.
-        """
 
+    # Start the page server
+    def start(self):
         self.zenith_cli.run(['start'])
         self.running = True
-        return self
 
-    def stop(self) -> 'ZenithPageserver':
-        """
-        Stop the page server.
-        Returns self.
-        """
-
-        if self.running:
-            self.zenith_cli.run(['stop'])
-            self.running = False
-
-        return self
+    # Stop the page server
+    def stop(self):
+        self.zenith_cli.run(['stop'])
+        self.running = True
 
+    # The page server speaks the Postgres FE/BE protocol, so you can connect
+    # to it with any Postgres client, and run special commands. This function
+    # returns a libpq connection string for connecting to it.
+    def connstr(self):
+        username = getpass.getuser()
+        conn_str = 'host={} port={} dbname=postgres user={}'.format(
+            'localhost', 64000, username)
+        return conn_str
 
+# The 'pageserver' fixture provides a Page Server that's up and running.
+#
+# If TEST_SHARED_FIXTURES is set, the Page Server instance is shared by all
+# the tests. To avoid clashing with other tests, don't use the 'main' branch in
+# the tests directly. Instead, create a branch off the 'empty' branch and use
+# that.
+#
+# By convention, the test branches are named after the tests. For example,
+# test called 'test_foo' would create and use branches with the 'test_foo' prefix.
 @zenfixture
-def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
-    """
-    The 'pageserver' fixture provides a Page Server that's up and running.
-
-    If TEST_SHARED_FIXTURES is set, the Page Server instance is shared by all
-    the tests. To avoid clashing with other tests, don't use the 'main' branch in
-    the tests directly. Instead, create a branch off the 'empty' branch and use
-    that.
-
-    By convention, the test branches are named after the tests. For example,
-    test called 'test_foo' would create and use branches with the 'test_foo' prefix.
-    """
-
-    ps = ZenithPageserver(zenith_cli).init().start()
+def pageserver(zenith_cli):
+    ps = ZenithPageserver(zenith_cli)
+    ps.init()
+    ps.start()
     # For convenience in tests, create a branch from the freshly-initialized cluster.
-    zenith_cli.run(["branch", "empty", "main"])
-
+    zenith_cli.run(["branch", "empty", "main"]);
     yield ps
-
     # After the yield comes any cleanup code we need.
     print('Starting pageserver cleanup')
     ps.stop()
 
-
-class Postgres(PgProtocol):
+class Postgres:
     """ An object representing a running postgres daemon. """
-    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, instance_num: int):
-        super().__init__(host='localhost', port=55431 + instance_num)
 
+    def __init__(self, zenith_cli, repo_dir, instance_num):
         self.zenith_cli = zenith_cli
         self.instance_num = instance_num
         self.running = False
+        self.username = getpass.getuser()
+        self.host = 'localhost'
+        self.port = 55431 + instance_num
         self.repo_dir = repo_dir
-        self.branch: Optional[str] = None  # dubious, see asserts below
+        self.branch = None
         # path to conf is <repo_dir>/pgdatadirs/<branch_name>/postgresql.conf
 
-    def create(self,
-               branch: str,
-               wal_acceptors: Optional[str] = None,
-               config_lines: Optional[List[str]] = None) -> 'Postgres':
-        """
-        Create the pg data directory.
-        If wal_acceptors is not None, node will use wal acceptors; config is
-        adjusted accordingly.
-        Returns self.
-        """
-
-        if not config_lines:
-            config_lines = []
-
+    def create_start(self, branch, config_lines=None):
+        """ create the pg data directory, and start the server """
         self.zenith_cli.run(['pg', 'create', branch])
         self.branch = branch
-        if wal_acceptors is not None:
-            self.adjust_for_wal_acceptors(wal_acceptors)
         if config_lines is None:
             config_lines = []
         self.config(config_lines)
-
-        return self
-
-    def start(self) -> 'Postgres':
-        """
-        Start the Postgres instance.
-        Returns self.
-        """
-
-        assert self.branch is not None
-        self.zenith_cli.run(['pg', 'start', self.branch])
+        self.zenith_cli.run(['pg', 'start', branch])
         self.running = True
+        return
 
-        return self
-
-    def config_file_path(self) -> str:
-        """ Path to postgresql.conf """
-        filename = f'pgdatadirs/{self.branch}/postgresql.conf'
-        return os.path.join(self.repo_dir, filename)
-
-    def adjust_for_wal_acceptors(self, wal_acceptors: str) -> 'Postgres':
-        """
-        Adjust instance config for working with wal acceptors instead of
-        pageserver (pre-configured by CLI) directly.
-        """
-
-        # TODO: reuse config()
-        with open(self.config_file_path(), "r") as f:
-            cfg_lines = f.readlines()
-        with open(self.config_file_path(), "w") as f:
-            for cfg_line in cfg_lines:
-                # walproposer uses different application_name
-                if ("synchronous_standby_names" in cfg_line or
-                        # don't ask pageserver to fetch WAL from compute
-                        "callmemaybe_connstring" in cfg_line):
-                    continue
-                f.write(cfg_line)
-            f.write("synchronous_standby_names = 'walproposer'\n")
-            f.write("wal_acceptors = '{}'\n".format(wal_acceptors))
-        return self
-
-    def config(self, lines: List[str]) -> 'Postgres':
-        """
-        Add lines to postgresql.conf.
-        Lines should be an array of valid postgresql.conf rows.
-        Returns self.
-        """
-
-        with open(self.config_file_path(), 'a') as conf:
+    #lines should be an array of valid postgresql.conf rows
+    def config(self, lines):
+        filename = 'pgdatadirs/{}/postgresql.conf'.format(self.branch)
+        config_name = os.path.join(self.repo_dir, filename)
+        with open(config_name, 'a') as conf:
             for line in lines:
                 conf.write(line)
                 conf.write('\n')
 
-        return self
-
-    def stop(self) -> 'Postgres':
-        """
-        Stop the Postgres instance if it's running.
-        Returns self.
-        """
-
+    def stop(self):
         if self.running:
-            assert self.branch is not None
             self.zenith_cli.run(['pg', 'stop', self.branch])
-            self.running = False
-
-        return self
-
-    def stop_and_destroy(self) -> 'Postgres':
-        """
-        Stop the Postgres instance, then destroy it.
-        Returns self.
-        """
-
-        assert self.branch is not None
-        self.zenith_cli.run(['pg', 'stop', '--destroy', self.branch])
-
-        return self
-
-    def create_start(self,
-                     branch: str,
-                     wal_acceptors: Optional[str] = None,
-                     config_lines: Optional[List[str]] = None) -> 'Postgres':
-        """
-        Create a Postgres instance, then start it.
-        Returns self.
-        """
-
-        self.create(branch, wal_acceptors, config_lines).start()
-
-        return self
 
+    # Return a libpq connection string to connect to the Postgres instance
+    def connstr(self, dbname='postgres'):
+        conn_str = 'host={} port={} dbname={} user={}'.format(
+            self.host, self.port, dbname, self.username)
+        return conn_str
 
 class PostgresFactory:
     """ An object representing multiple running postgres daemons. """
-    def __init__(self, zenith_cli: ZenithCli, repo_dir: str):
+    def __init__(self, zenith_cli, repo_dir):
         self.zenith_cli = zenith_cli
+        self.host = 'localhost'
         self.repo_dir = repo_dir
         self.num_instances = 0
-        self.instances: List[Postgres] = []
-
-    def create_start(self,
-                     branch: str = "main",
-                     wal_acceptors: Optional[str] = None,
-                     config_lines: Optional[List[str]] = None) -> Postgres:
+        self.instances = []
 
+    def create_start(self, branch="main", config_lines=None):
         pg = Postgres(self.zenith_cli, self.repo_dir, self.num_instances + 1)
         self.num_instances += 1
         self.instances.append(pg)
+        pg.create_start(branch, config_lines)
+        return pg
 
-        return pg.create_start(branch, wal_acceptors, config_lines)
-
-    def stop_all(self) -> 'PostgresFactory':
+    def stop_all(self):
         for pg in self.instances:
             pg.stop()
 
-        return self
-
-
 @zenfixture
-def postgres(zenith_cli: ZenithCli, repo_dir: str) -> Iterator[PostgresFactory]:
+def postgres(zenith_cli, repo_dir):
     pgfactory = PostgresFactory(zenith_cli, repo_dir)
-
     yield pgfactory
-
     # After the yield comes any cleanup code we need.
     print('Starting postgres cleanup')
     pgfactory.stop_all()
@@ -399,27 +229,27 @@ def postgres(zenith_cli: ZenithCli, repo_dir: str) -> Iterator[PostgresFactory]:
 
 class PgBin:
     """ A helper class for executing postgres binaries """
-    def __init__(self, log_dir: str, pg_distrib_dir: str):
+
+    def __init__(self, log_dir, pg_distrib_dir):
         self.log_dir = log_dir
         self.pg_install_path = pg_distrib_dir
         self.pg_bin_path = os.path.join(self.pg_install_path, 'bin')
         self.env = os.environ.copy()
         self.env['LD_LIBRARY_PATH'] = os.path.join(self.pg_install_path, 'lib')
 
-    def _fixpath(self, command: List[str]) -> None:
-        if '/' not in command[0]:
+    def _fixpath(self, command):
+        if not '/' in command[0]:
             command[0] = os.path.join(self.pg_bin_path, command[0])
 
-    def _build_env(self, env_add: Optional[Env]) -> Env:
+    def _build_env(self, env_add):
         if env_add is None:
             return self.env
         env = self.env.copy()
         env.update(env_add)
         return env
 
-    def run(self, command: List[str], env: Optional[Env] = None, cwd: Optional[str] = None) -> None:
-        """
-        Run one of the postgres binaries.
+    def run(self, command, env=None, cwd=None):
+        """ Run one of the postgres binaries.
 
         The command should be in list form, e.g. ['pgbench', '-p', '55432']
 
@@ -429,23 +259,18 @@ class PgBin:
         characters present), then it will be edited to include the correct path.
 
         If you want stdout/stderr captured to files, use `run_capture` instead.
-        """
 
+        """
         self._fixpath(command)
         print('Running command "{}"'.format(' '.join(command)))
         env = self._build_env(env)
         subprocess.run(command, env=env, cwd=cwd, check=True)
 
-    def run_capture(self,
-                    command: List[str],
-                    env: Optional[Env] = None,
-                    cwd: Optional[str] = None) -> None:
-        """
-        Run one of the postgres binaries, with stderr and stdout redirected to a file.
+    def run_capture(self, command, env=None, cwd=None):
+        """ Run one of the postgres binaries, with stderr and stdout redirected to a file.
 
         This is just like `run`, but for chatty programs.
         """
-
         self._fixpath(command)
         print('Running command "{}"'.format(' '.join(command)))
         env = self._build_env(env)
@@ -453,127 +278,21 @@ class PgBin:
 
 
 @zenfixture
-def pg_bin(test_output_dir: str, pg_distrib_dir: str) -> PgBin:
+def pg_bin(test_output_dir, pg_distrib_dir):
     return PgBin(test_output_dir, pg_distrib_dir)
 
 
-def read_pid(path):
-    """ Read content of file into number """
-    return int(Path(path).read_text())
-
-
-class WalAcceptor(PgProtocol):
-    """ An object representing a running wal acceptor daemon. """
-    def __init__(self, wa_binpath, data_dir, port, num):
-        super().__init__(host='127.0.0.1', port=port)
-        self.wa_binpath = wa_binpath
-        self.data_dir = data_dir
-        self.port = port
-        self.num = num  # identifier for logging
-
-    def start(self) -> 'WalAcceptor':
-        # create data directory if not exists
-        Path(self.data_dir).mkdir(parents=True, exist_ok=True)
-
-        cmd = [self.wa_binpath]
-        cmd.extend(["-D", self.data_dir])
-        cmd.extend(["-l", "127.0.0.1:{}".format(self.port)])
-        cmd.append("--daemonize")
-        cmd.append("--no-sync")
-        # Tell page server it can receive WAL from this WAL safekeeper
-        cmd.extend(["--pageserver", "127.0.0.1:{}".format(DEFAULT_PAGESERVER_PORT)])
-        cmd.extend(["--recall", "1 second"])
-        print('Running command "{}"'.format(' '.join(cmd)))
-        subprocess.run(cmd, check=True)
-
-        return self
-
-    def stop(self) -> 'WalAcceptor':
-        print('Stopping wal acceptor {}'.format(self.num))
-        pidfile_path = os.path.join(self.data_dir, "wal_acceptor.pid")
-        try:
-            pid = read_pid(pidfile_path)
-            try:
-                os.kill(pid, signal.SIGTERM)
-            except Exception:
-                pass  # pidfile might be obsolete
-            # TODO: cleanup pid file on exit in wal acceptor
-            return self
-            # for _ in range(5):
-            # print('waiting wal acceptor {} (pid {}) to stop...', self.num, pid)
-            # try:
-            # read_pid(pidfile_path)
-            # except FileNotFoundError:
-            # return  # done
-            # time.sleep(1)
-            # raise Exception('Failed to wait for wal acceptor {} shutdown'.format(self.num))
-        except FileNotFoundError:
-            print("Wal acceptor {} is not running".format(self.num))
-            return self
-
-
-class WalAcceptorFactory:
-    """ An object representing multiple running wal acceptors. """
-    def __init__(self, zenith_binpath, data_dir):
-        self.wa_binpath = os.path.join(zenith_binpath, 'wal_acceptor')
-        self.data_dir = data_dir
-        self.instances = []
-        self.initial_port = 54321
-
-    def start_new(self) -> WalAcceptor:
-        """
-        Start new wal acceptor.
-        """
-
-        wa_num = len(self.instances)
-        wa = WalAcceptor(self.wa_binpath,
-                         os.path.join(self.data_dir, "wal_acceptor_{}".format(wa_num)),
-                         self.initial_port + wa_num, wa_num)
-        wa.start()
-        self.instances.append(wa)
-        return wa
-
-    def start_n_new(self, n: int) -> None:
-        """
-        Start n new wal acceptors.
-        """
-
-        for _ in range(n):
-            self.start_new()
-
-    def stop_all(self) -> 'WalAcceptorFactory':
-        for wa in self.instances:
-            wa.stop()
-        return self
-
-    def get_connstrs(self) -> str:
-        """ Get list of wal acceptor endpoints suitable for wal_acceptors GUC  """
-        return ','.join(["127.0.0.1:{}".format(wa.port) for wa in self.instances])
-
-
 @zenfixture
-def wa_factory(zenith_binpath: str, repo_dir: str) -> Iterator[WalAcceptorFactory]:
-    """ Gives WalAcceptorFactory providing wal acceptors. """
-    wafactory = WalAcceptorFactory(zenith_binpath, os.path.join(repo_dir, "wal_acceptors"))
-    yield wafactory
-    # After the yield comes any cleanup code we need.
-    print('Starting wal acceptors cleanup')
-    wafactory.stop_all()
-
-
-@zenfixture
-def base_dir() -> str:
+def base_dir():
     """ find the base directory (currently this is the git root) """
-
     base_dir = os.path.normpath(os.path.join(get_self_dir(), '../..'))
-    print('\nbase_dir is', base_dir)
+    print('base_dir is', base_dir)
     return base_dir
 
 
 @zenfixture
-def top_output_dir(base_dir: str) -> str:
+def top_output_dir(base_dir):
     """ Compute the top-level directory for all tests. """
-
     env_test_output = os.environ.get('TEST_OUTPUT')
     if env_test_output is not None:
         output_dir = env_test_output
@@ -584,9 +303,8 @@ def top_output_dir(base_dir: str) -> str:
 
 
 @zenfixture
-def test_output_dir(request: Any, top_output_dir: str) -> str:
+def test_output_dir(request, top_output_dir):
     """ Compute the working directory for an individual test. """
-
     if os.environ.get('TEST_SHARED_FIXTURES') is None:
         # one directory per test
         test_name = request.node.name
@@ -602,22 +320,19 @@ def test_output_dir(request: Any, top_output_dir: str) -> str:
 
 
 @zenfixture
-def repo_dir(request: Any, test_output_dir: str) -> str:
-    """
-    Compute the test repo_dir.
+def repo_dir(request, test_output_dir):
+    """ Compute the test repo_dir
 
     "repo_dir" is the place where all of the pageserver files will go.
     It doesn't have anything to do with the git repo.
     """
-
     repo_dir = os.path.join(test_output_dir, 'repo')
     return repo_dir
 
 
 @zenfixture
-def zenith_binpath(base_dir: str) -> str:
-    """ Find the zenith binaries. """
-
+def zenith_binpath(base_dir):
+    """ find the zenith binaries """
     env_zenith_bin = os.environ.get('ZENITH_BIN')
     if env_zenith_bin:
         zenith_dir = env_zenith_bin
@@ -629,9 +344,8 @@ def zenith_binpath(base_dir: str) -> str:
 
 
 @zenfixture
-def pg_distrib_dir(base_dir: str) -> str:
-    """ Find the postgres install. """
-
+def pg_distrib_dir(base_dir):
+    """ find the postgress install """
     env_postgres_bin = os.environ.get('POSTGRES_DISTRIB_DIR')
     if env_postgres_bin:
         pg_dir = env_postgres_bin

test_runner/setup.cfg

@@ -1,28 +0,0 @@
-# Just trying to gather linter settings in one file.
-# I wonder if there's a way to de-duplicate them...
-
-[flake8]
-max-line-length = 100
-
-[pycodestyle]
-max-line-length = 100
-
-[yapf]
-based_on_style = pep8
-column_limit = 100
-
-[mypy]
-# some tests don't typecheck when this flag is set
-check_untyped_defs = false
-
-disallow_incomplete_defs = false
-disallow_untyped_calls = false
-disallow_untyped_decorators = false
-disallow_untyped_defs = false
-strict = true
-
-[mypy-psycopg2.*]
-ignore_missing_imports = true
-
-[mypy-pytest.*]
-ignore_missing_imports = true

test_runner/test_broken.py

@@ -2,7 +2,9 @@ import pytest
 import os
 
 pytest_plugins = ("fixtures.zenith_fixtures")
+
 """
+
 Use this test to see what happens when tests fail.
 
 We should be able to clean up after ourselves, including stopping any
@@ -10,18 +12,21 @@ postgres or pageserver processes.
 
 Set the environment variable RUN_BROKEN to see this test run (and fail,
 and hopefully not leave any server processes behind).
+
 """
 
-run_broken = pytest.mark.skipif(os.environ.get('RUN_BROKEN') is None,
-                                reason="only used for testing the fixtures")
 
+run_broken = pytest.mark.skipif(
+    os.environ.get('RUN_BROKEN') == None,
+    reason="only used for testing the fixtures"
+)
 
 @run_broken
 def test_broken(zenith_cli, pageserver, postgres, pg_bin):
     # Create a branch for us
-    zenith_cli.run(["branch", "test_broken", "empty"])
+    zenith_cli.run(["branch", "test_broken", "empty"]);
 
-    postgres.create_start("test_broken")
+    pg = postgres.create_start("test_broken")
     print('postgres is running')
 
     print('THIS NEXT COMMAND WILL FAIL:')

test_runner/zenith_regress/README.md

@@ -1,8 +1,11 @@
 To add a new SQL test
 
 - add sql script to run to zenith_regress/sql/testname.sql
-- add expected output to zenith_regress/expected/testname.out
-- add testname to parallel_schedule
+- add expected output to zenith/regress/expected/testname.out
+- add testname to both parallel_schedule and serial_schedule files*
 
 That's it.
 For more complex tests see PostgreSQL regression tests. These works basically the same.
+
+*it was changed recently in PostgreSQL upstream - no more separate serial_schedule.
+Someday we'll catch up with these changes.

test_runner/zenith_regress/expected/zenith-rel-truncate.out

@@ -17,29 +17,3 @@ SELECT * FROM truncatetest;
 (0 rows)
 
 DROP TABLE truncatetest;
---
--- Test that the FSM is truncated along with the table.
---
--- Create a test table and delete and vacuum away most of the rows.
--- This leaves the FSM full of pages with plenty of space
-create table tt(i int);
-insert into tt select g from generate_series(1, 100000) g;
-delete from tt where i%100 != 0 and i > 10000;
-vacuum freeze tt;
--- Delete the rest of the rows, and vacuum again. This truncates the
--- heap to 0 blocks, and should also truncate the FSM.
-delete from tt;
-vacuum tt;
--- This can be used to look at the FSM directly, if the 'pg_freespace' contrib module
--- is installed
---SELECT blkno, avail from generate_series(1, 450) blkno, pg_freespace('tt'::regclass, blkno) AS avail;
--- Insert a row again. It should go on block #0. If the FSM was not truncated,
--- the insertion would find a higher-numbered block in the FSM and use that instead.
-insert into tt values (0);
-select ctid, * from tt;
- ctid  | i 
--------+---
- (0,1) | 0
-(1 row)
-
-drop table tt;

test_runner/zenith_regress/expected/zenith-truncate.out

@@ -0,0 +1,9 @@
+create table tt(x integer);
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+drop table tt;

test_runner/zenith_regress/parallel_schedule

@@ -1,10 +1,12 @@
 # ----------
-# Like in PostgreSQL src/test/regress/parallel_schedule, we put no
-# more than twenty tests in any one parallel group; this limits the
-# number of connections needed to run the tests.
+# src/test/regress/parallel_schedule
+#
+# By convention, we put no more than twenty tests in any one parallel group;
+# this limits the number of connections needed to run the tests.
 # ----------
 
 test: zenith-cid
 test: zenith-rel-truncate
 test: zenith-clog
 test: zenith-vacuum-full
+test: zenith-truncate

test_runner/zenith_regress/serial_schedule

@@ -0,0 +1,7 @@
+# src/test/regress/serial_schedule
+# This should probably be in an order similar to parallel_schedule.
+test: zenith-cid
+test: zenith-rel-truncate
+test: zenith-clog
+test: zenith-vacuum-full
+test: zenith-truncate

test_runner/zenith_regress/sql/zenith-rel-truncate.sql

@@ -16,31 +16,3 @@ VACUUM truncatetest;
 SELECT * FROM truncatetest;
 
 DROP TABLE truncatetest;
-
-
---
--- Test that the FSM is truncated along with the table.
---
-
--- Create a test table and delete and vacuum away most of the rows.
--- This leaves the FSM full of pages with plenty of space
-create table tt(i int);
-insert into tt select g from generate_series(1, 100000) g;
-delete from tt where i%100 != 0 and i > 10000;
-vacuum freeze tt;
-
--- Delete the rest of the rows, and vacuum again. This truncates the
--- heap to 0 blocks, and should also truncate the FSM.
-delete from tt;
-vacuum tt;
-
--- This can be used to look at the FSM directly, if the 'pg_freespace' contrib module
--- is installed
---SELECT blkno, avail from generate_series(1, 450) blkno, pg_freespace('tt'::regclass, blkno) AS avail;
-
--- Insert a row again. It should go on block #0. If the FSM was not truncated,
--- the insertion would find a higher-numbered block in the FSM and use that instead.
-insert into tt values (0);
-select ctid, * from tt;
-
-drop table tt;

test_runner/zenith_regress/sql/zenith-truncate.sql

@@ -0,0 +1,9 @@
+create table tt(x integer);
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+drop table tt;

walkeeper/Cargo.toml

@@ -30,7 +30,6 @@ crc32c = "0.6.0"
 parse_duration = "2.1.1"
 walkdir = "2"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
 hex = "0.4.3"
 
 # FIXME: 'pageserver' is needed for ZTimelineId. Refactor

walkeeper/README

@@ -12,13 +12,13 @@ safekeeper yet.
 The primary connects to the WAL safekeeper, so it works in a "push"
 fashion.  That's different from how streaming replication usually
 works, where the replica initiates the connection. To do that, there
-is a component called the "WAL proposer". The WAL proposer is a
-background worker that runs in the primary Postgres server. It
-connects to the WAL safekeeper, and
-sends all the WAL. (PostgreSQL's archive_commands works in the
+is a component called "safekeeper_proxy". The safekeeper_proxy runs on
+the same host as the primary Postgres server and connects to it to do
+streaming replication.  It also connects to the WAL safekeeper, and
+forwards all the WAL. (PostgreSQL's archive_commands works in the
 "push" style, but it operates on a WAL segment granularity. If
-PostgreSQL had a push style API for streaming, WAL propose could be
-implemented using it.)
+PostgreSQL had a push style API for streaming, we wouldn't need the
+proxy).
 
 The Page Server connects to the WAL safekeeper, using the same
 streaming replication protocol that's used between Postgres primary
@@ -33,4 +33,6 @@ safekeepers. The Paxos and crash recovery algorithm ensures that only
 one primary node can be actively streaming WAL to the quorum of
 safekeepers.
 
-See README.md for a more detailed desription of the consensus protocol.
+
+See vendor/postgres/src/bin/safekeeper/README.md for a more detailed
+desription of the consensus protocol. (TODO: move the text here?)

walkeeper/README.md

@@ -1,15 +1,15 @@
-# WAL proposer-safekeeper communication consensus protocol.
+# Proxy-safekeeper communication consensus protocol.
 
 ## General requirements and architecture
 
 There is single stateless master and several safekeepers. Number of safekeepers is determined by redundancy level.
 To minimize number of changes in Postgres core, we are using standard streaming replication from master (through WAL sender).
-This replication stream is initiated by the WAL proposer process that runs in the PostgreSQL server, which broadcasts the WAL generated by PostgreSQL to safekeepers.
+This replication stream is initiated by `safekeeper_proxy` which receives data from the master and broadcasts it to safekeepers.
 To provide durability we use synchronous replication at master (response to the commit statement is sent to the client
-only when acknowledged by WAL receiver). WAL proposer sends this acknowledgment only when LSN of commit record is confirmed by quorum of safekeepers.
+only when acknowledged by WAL receiver). `safekeeper_proxy` sends this acknowledgment only when LSN of commit record is confirmed by quorum of safekeepers.
 
-WAL proposer tries to establish connections with safekeepers.
-At any moment of time each safekeeper can serve exactly once proposer, but it can accept new connections.
+`Safekeeper_proxy` tries to establish connections with safekeepers.
+At any moment of time each safekeeper can serve exactly once proxy, but it can accept new connections.
 
 Any of safekeepers can be used as WAL server, producing replication stream. So both `Pagers` and `Replicas`
 (read-only computation nodes) can connect to safekeeper to receive WAL stream. Safekeepers is streaming WAL until
@@ -30,13 +30,13 @@ then accepts proposed nodeId and persists this choice in the local control file.
 5. If quorum of safekeepers approve proposed nodeId, then server assumes that handshake is successfully completed and switch to recovery stage.
 
 ## Recovery
-Proposer computes max(`restartLSN`) and max(`flushLSN`) from quorum of attached safekeepers.
+Proxy computes max(`restartLSN`) and max(`flushLSN`) from quorum of attached safekeepers.
 `RestartLSN` - is position in WAL which is known to be delivered to all safekeepers.
 In other words: `restartLSN` can be also considered as cut-off horizon (all preceding WAL segments can be removed).
 `FlushLSN` is position flushed by safekeeper to the local persistent storage.
 
 If max(`restartLSN`) != max(`flushLSN`), then recovery has to be performed.
-Proposer creates replication channel with most advanced safekeeper (safekeeper with the largest `flushLSN`).
+Proxy creates replication channel with most advanced safekeeper (safekeeper with the largest `flushLSN`).
 Then it downloads all WAL messages between max(`restartLSN`)..max(`flushLSN`).
 Messages are inserted in L1-list (ordered by LSN). Then we locate position of each safekeeper in this list according
 to their `flushLSN`s. Safekeepers that are not yet connected (out of quorum) should start from the beginning of the list
@@ -49,11 +49,11 @@ to avoid loose of committed data.
 Calculated max(`flushLSN`) is called `VCL` (Volume Complete LSN). As far as it is chosen among quorum, there may be some other offline safekeeper with larger
 `VCL`. Once it becomes online, we need to overwrite its WAL beyond `VCL`. To support it, each safekeeper maintains
 `epoch` number. `Epoch` plays almost the same role as `term`, but algorithm of `epoch` bumping is different.
-`VCL` and new epoch are received by safekeeper from proposer during voting.
+`VCL` and new epoch are received by safekeeper from proxy during voting.
 But safekeeper doesn't switch to new epoch immediately after voting.
 Instead of it, safekeepers waits record with LSN > Max(`flushLSN`,`VCL`) is received.
 It means that we restore all records from old generation and switch to new generation.
-When proposer calculates max(`FlushLSN`), it first compares `Epoch`. So actually we compare (`Epoch`,`FlushLSN`) pairs.
+When proxy calculates max(`FlushLSN`), it first compares `Epoch`. So actually we compare (`Epoch`,`FlushLSN`) pairs.
 
 Let's looks at the examples. Consider that we have three safekeepers: S1, S2, S3. Si(N) means that i-th safekeeper has epoch=N.
 Ri(x) - WAL record for resource X with LSN=i. Assume that we have the following state:
@@ -64,7 +64,7 @@ S2(1): R1(a),R2(b)
 S3(1): R1(a),R2(b),R3(c),R4(d)  - offline
 ```
 
-Proposer choose quorum (S1,S2). VCL for them is 2. We download S2 to proposer and schedule its write to S1.
+Proxy choose quorum (S1,S2). VCL for them is 2. We download S2 to proxy and schedule its write to S1.
 After receiving record R5 the picture can be:
 
 ```
@@ -119,26 +119,29 @@ S3(1): R1(a),R2(b),R3(c),R4(d)
 ```
 
 ## Main loop
-Once recovery is completed, proposer switches to normal processing loop: it receives WAL stream from Postgres and appends WAL
+Once recovery is completed, proxy switches to normal processing loop: it receives WAL stream from master and appends WAL
 messages to the list. At the same time it tries to push messages to safekeepers. Each safekeeper is associated
 with some element in message list and once it acknowledged receiving of the message, position is moved forward.
 Each queue element contains acknowledgment mask, which bits corresponds to safekeepers.
 Once all safekeepers acknowledged receiving of this message (by setting correspondent bit),
 then element can be removed from queue and `restartLSN` is advanced forward.
 
-Proposer maintains `restartLSN` and `commitLSN` based on the responses received by safekeepers.
+Proxy maintains `restartLSN` and `commitLSN` based on the responses received by safekeepers.
 `RestartLSN` equals to the LSN of head message in the list. `CommitLSN` is `flushLSN[nSafekeepers-Quorum]` element
 in ordered array with `flushLSN`s of safekeepers. `CommitLSN` and `RestartLSN` are included in requests
-sent from proposer to safekeepers and stored in safekeepers control file.
+sent from proxy to safekeepers and stored in safekeepers control file.
 To avoid overhead of extra fsync, this control file is not fsynced on each request. Flushing this file is performed
 periodically, which means that `restartLSN`/`commitLSN` stored by safekeeper may be slightly deteriorated.
 It is not critical because may only cause redundant processing of some WAL record.
 And `FlushLSN` is recalculated after node restart by scanning local WAL files.
 
 ## Fault tolerance
-If the WAL proposer process looses connection to safekeeper it tries to reestablish this connection using the same nodeId.
+Once `safekeeper_proxy` looses connection to safekeeper it tries to reestablish this connection using the same nodeId.
+If `safekeeper_proxy` looses connection with master, it is terminated. Right now safekeeper is standalone process,
+which can be launched at any node, but it can be also spawned as master's background worker, so that it is automatically
+restarted in case of Postgres instance restart.
 
-Restart of PostgreSQL initiates new round of voting and switching new epoch.
+Restart of `safekeeper_proxy` initiates new round of voting and switching new epoch.
 
 ## Limitations
 Right now message queue is maintained in main memory and is not spilled to the disk.
@@ -158,7 +161,7 @@ It is assumed that in case of loosing local data by some safekeepers, it should
 ## Algorithm
 
 ```python
-process WalProposer(safekeepers,server,curr_epoch,restart_lsn=0,message_queue={},feedbacks={})
+process SafekeeperProxy(safekeepers,server,curr_epoch,restart_lsn=0,message_queue={},feedbacks={})
     function do_recovery(epoch,restart_lsn,VCL)
         leader = i:safekeepers[i].state.epoch=epoch and safekeepers[i].state.flushLsn=VCL
         wal_stream = safekeepers[leader].start_replication(restart_lsn,VCL)
@@ -238,31 +241,31 @@ end process
 
 process safekeeper(gateway,state)
     function handshake()
-        proposer = gateway.accept()
-        server_info = proposer.read()
-        proposer.write(state)
-        proposal = proposer.read()
+        proxy = gateway.accept()
+        server_info = proxy.read()
+        proxy.write(state)
+        proposal = proxy.read()
         if proposal.nodeId < state.nodeId
-            proposer.write(rejected)
+            proxy.write(rejected)
             return null
         else
             state.nodeId = proposal.nodeId
             state.proposed_epoch = proposal.epoch
             state.VCL = proposal.VCL
             write_control_file(state)
-            proposer.write(accepted)
-            return proposer
+            proxy.write(accepted)
+            return proxy
     end function
 
     state = read_control_file()
     state.flushLsn = locate_end_of_wal()
 
     for ever
-        proposer = handshake()
-        if not proposer
+        proxy = handshake()
+        if not proxy
             continue
         for ever
-            req = proposer.read()
+            req = proxy.read()
             if req.nodeId != state.nodeId
                 break
             save_wal_file(req.data)
@@ -273,7 +276,7 @@ process safekeeper(gateway,state)
                 state.flushLsn = req.endPos
             save_control_file(state)
             resp = Response(state.epoch,req.endPos)
-            proposer.write(resp)
+            proxy.write(resp)
             notify_wal_sender(Min(req.commitLsn,req.endPos))
 end process
 ```

walkeeper/src/bin/wal_acceptor.rs

@@ -8,7 +8,6 @@ use log::*;
 use parse_duration::parse;
 use slog::Drain;
 use std::io;
-use std::net::ToSocketAddrs;
 use std::path::{Path, PathBuf};
 use std::thread;
 use std::time::Duration;
@@ -28,6 +27,13 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Path to the WAL acceptor data directory"),
         )
+        .arg(
+            Arg::with_name("systemid")
+                .long("systemid")
+                .takes_value(true)
+                .required(true)
+                .help("PostgreSQL system id, from pg_control"),
+        )
         .arg(
             Arg::with_name("listen")
                 .short("l")
@@ -47,12 +53,6 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("interval for keeping WAL as walkeeper node, after which them will be uploaded to S3 and removed locally"),
         )
-        .arg(
-            Arg::with_name("recall-period")
-                .long("recall")
-                .takes_value(true)
-                .help("Period for requestion pageserver to call for replication"),
-        )
         .arg(
             Arg::with_name("daemonize")
                 .short("d")
@@ -69,14 +69,17 @@ fn main() -> Result<()> {
         )
         .get_matches();
 
+    let systemid_str = arg_matches.value_of("systemid").unwrap();
+    let systemid: u64 = systemid_str.parse()?;
+
     let mut conf = WalAcceptorConf {
         data_dir: PathBuf::from("./"),
+        systemid,
         daemonize: false,
         no_sync: false,
         pageserver_addr: None,
         listen_addr: "127.0.0.1:5454".parse()?,
         ttl: None,
-        recall_period: None,
     };
 
     if let Some(dir) = arg_matches.value_of("datadir") {
@@ -95,27 +98,17 @@ fn main() -> Result<()> {
     }
 
     if let Some(addr) = arg_matches.value_of("listen") {
-        // TODO: keep addr vector in config and listen them all
-        // XXX: with our callmemaybe approach we need to set 'advertised address'
-        // as it is not always possible to listen it. Another reason to ditch callmemaybe.
-        let addrs: Vec<_> = addr.to_socket_addrs().unwrap().collect();
-        conf.listen_addr = addrs[0];
+        conf.listen_addr = addr.parse().unwrap();
     }
 
     if let Some(addr) = arg_matches.value_of("pageserver") {
-        // TODO: keep addr vector in config and check them all while connecting
-        let addrs: Vec<_> = addr.to_socket_addrs().unwrap().collect();
-        conf.pageserver_addr = Some(addrs[0]);
+        conf.pageserver_addr = Some(addr.parse().unwrap());
     }
 
     if let Some(ttl) = arg_matches.value_of("ttl") {
         conf.ttl = Some::<Duration>(parse(ttl)?);
     }
 
-    if let Some(recall) = arg_matches.value_of("recall") {
-        conf.recall_period = Some::<Duration>(parse(recall)?);
-    }
-
     start_wal_acceptor(conf)
 }
 

walkeeper/src/lib.rs

@@ -3,6 +3,7 @@ use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::time::Duration;
 
+pub mod pq_protocol;
 pub mod receive_wal;
 pub mod replication;
 pub mod s3_offload;
@@ -10,13 +11,15 @@ pub mod send_wal;
 pub mod timeline;
 pub mod wal_service;
 
+use crate::pq_protocol::SystemId;
+
 #[derive(Debug, Clone)]
 pub struct WalAcceptorConf {
     pub data_dir: PathBuf,
+    pub systemid: SystemId,
     pub daemonize: bool,
     pub no_sync: bool,
     pub listen_addr: SocketAddr,
     pub pageserver_addr: Option<SocketAddr>,
     pub ttl: Option<Duration>,
-    pub recall_period: Option<Duration>,
 }

walkeeper/src/pq_protocol.rs

@@ -0,0 +1,224 @@
+use byteorder::{BigEndian, ReadBytesExt};
+use bytes::{BufMut, Bytes, BytesMut};
+use pageserver::ZTimelineId;
+use std::io::{self, Read};
+use std::str;
+use std::str::FromStr;
+
+pub type Oid = u32;
+pub type SystemId = u64;
+
+#[derive(Debug)]
+pub enum FeMessage {
+    Query(FeQueryMessage),
+    Terminate,
+    CopyData(FeCopyData),
+}
+
+#[derive(Debug)]
+pub struct RowDescriptor {
+    pub typoid: Oid,
+    pub typlen: i16,
+    pub name: &'static [u8],
+}
+
+#[derive(Debug)]
+pub enum BeMessage<'a> {
+    AuthenticationOk,
+    ReadyForQuery,
+    RowDescription(&'a [RowDescriptor]),
+    DataRow(&'a [Option<&'a [u8]>]),
+    CommandComplete(&'a [u8]),
+    Negotiate,
+    Copy,
+}
+
+#[derive(Debug)]
+pub struct FeStartupMessage {
+    pub version: u32,
+    pub kind: StartupRequestCode,
+    pub timelineid: ZTimelineId,
+    pub appname: Option<String>,
+}
+
+#[derive(Debug)]
+pub enum StartupRequestCode {
+    Cancel,
+    NegotiateSsl,
+    NegotiateGss,
+    Normal,
+}
+
+impl FeStartupMessage {
+    pub fn read_from(reader: &mut impl Read) -> io::Result<Self> {
+        const MAX_STARTUP_PACKET_LENGTH: usize = 10000;
+        const CANCEL_REQUEST_CODE: u32 = (1234 << 16) | 5678;
+        const NEGOTIATE_SSL_CODE: u32 = (1234 << 16) | 5679;
+        const NEGOTIATE_GSS_CODE: u32 = (1234 << 16) | 5680;
+
+        let len = reader.read_u32::<BigEndian>()? as usize;
+
+        if len < 4 || len > MAX_STARTUP_PACKET_LENGTH {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "FeStartupMessage: invalid message length",
+            ));
+        }
+
+        let version = reader.read_u32::<BigEndian>()?;
+
+        let kind = match version {
+            CANCEL_REQUEST_CODE => StartupRequestCode::Cancel,
+            NEGOTIATE_SSL_CODE => StartupRequestCode::NegotiateSsl,
+            NEGOTIATE_GSS_CODE => StartupRequestCode::NegotiateGss,
+            _ => StartupRequestCode::Normal,
+        };
+
+        let params_len = len - 8;
+        let mut params_bytes = vec![0u8; params_len];
+        reader.read_exact(params_bytes.as_mut())?;
+
+        let params_str = str::from_utf8(&params_bytes).unwrap();
+        let params = params_str.split('\0');
+        let mut options = false;
+        let mut timelineid: Option<ZTimelineId> = None;
+        let mut appname: Option<String> = None;
+        for p in params {
+            if p == "options" {
+                options = true;
+            } else if options {
+                for opt in p.split(' ') {
+                    if let Some(ztimelineid_str) = opt.strip_prefix("ztimelineid=") {
+                        // FIXME: rethrow parsing error, don't unwrap
+                        timelineid = Some(ZTimelineId::from_str(ztimelineid_str).unwrap());
+                    } else if let Some(val) = opt.strip_prefix("application_name=") {
+                        appname = Some(val.to_string());
+                    }
+                }
+                break;
+            }
+        }
+        if timelineid.is_none() {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "timelineid is required",
+            ));
+        }
+
+        Ok(FeStartupMessage {
+            version,
+            kind,
+            appname,
+            timelineid: timelineid.unwrap(),
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct FeQueryMessage {
+    pub body: Bytes,
+}
+
+#[derive(Debug)]
+pub struct FeCopyData {
+    pub body: Bytes,
+}
+
+impl<'a> BeMessage<'a> {
+    pub fn write(buf: &mut BytesMut, message: &BeMessage) {
+        match message {
+            BeMessage::AuthenticationOk => {
+                buf.put_u8(b'R');
+                buf.put_i32(4 + 4);
+                buf.put_i32(0);
+            }
+
+            BeMessage::ReadyForQuery => {
+                buf.put_u8(b'Z');
+                buf.put_i32(4 + 1);
+                buf.put_u8(b'I');
+            }
+
+            BeMessage::Negotiate => {
+                buf.put_u8(b'N');
+            }
+
+            BeMessage::Copy => {
+                buf.put_u8(b'W');
+                buf.put_i32(7);
+                buf.put_u8(b'\0');
+                buf.put_u8(b'\0');
+                buf.put_u8(b'\0');
+            }
+
+            BeMessage::RowDescription(rows) => {
+                buf.put_u8(b'T');
+
+                let mut body = BytesMut::new();
+                body.put_i16(rows.len() as i16); // # of fields
+                for row in rows.iter() {
+                    body.put_slice(row.name);
+                    body.put_i32(0); /* table oid */
+                    body.put_i16(0); /* attnum */
+                    body.put_u32(row.typoid);
+                    body.put_i16(row.typlen);
+                    body.put_i32(-1); /* typmod */
+                    body.put_i16(0); /* format code */
+                }
+                buf.put_i32((4 + body.len()) as i32); // # of bytes, including len field itself
+                buf.put(body);
+            }
+
+            BeMessage::DataRow(vals) => {
+                buf.put_u8(b'D');
+                let total_len: usize = vals
+                    .iter()
+                    .fold(0, |acc, row| acc + 4 + row.map_or(0, |s| s.len()));
+                buf.put_u32(4 + 2 + total_len as u32);
+                buf.put_u16(vals.len() as u16);
+                for val_opt in vals.iter() {
+                    if let Some(val) = val_opt {
+                        buf.put_u32(val.len() as u32);
+                        buf.put_slice(val);
+                    } else {
+                        buf.put_i32(-1);
+                    }
+                }
+            }
+
+            BeMessage::CommandComplete(cmd) => {
+                buf.put_u8(b'C');
+                buf.put_i32(4 + cmd.len() as i32);
+                buf.put_slice(cmd);
+            }
+        }
+    }
+}
+
+impl FeMessage {
+    pub fn read_from(reader: &mut impl Read) -> io::Result<FeMessage> {
+        let tag = reader.read_u8()?;
+        let len = reader.read_u32::<BigEndian>()?;
+
+        if len < 4 {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "FeMessage: invalid message length",
+            ));
+        }
+
+        let body_len = (len - 4) as usize;
+        let mut body = vec![0u8; body_len];
+        reader.read_exact(&mut body)?;
+
+        match tag {
+            b'Q' => Ok(FeMessage::Query(FeQueryMessage { body: body.into() })),
+            b'd' => Ok(FeMessage::CopyData(FeCopyData { body: body.into() })),
+            b'X' => Ok(FeMessage::Terminate),
+            tag => Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                format!("unknown message tag: {},'{:?}'", tag, body),
+            )),
+        }
+    }
+}

walkeeper/src/receive_wal.rs

@@ -12,17 +12,15 @@ use std::io::{BufReader, Read, Seek, SeekFrom, Write};
 use std::net::{SocketAddr, TcpStream};
 use std::str;
 use std::sync::Arc;
-use std::thread;
-use std::thread::sleep;
 use zenith_utils::bin_ser::LeSer;
 use zenith_utils::lsn::Lsn;
 
+use crate::pq_protocol::*;
 use crate::replication::HotStandbyFeedback;
 use crate::timeline::{Timeline, TimelineTools};
 use crate::WalAcceptorConf;
 use pageserver::ZTimelineId;
 use postgres_ffi::xlog_utils::{TimeLineID, XLogFileName, MAX_SEND_SIZE, XLOG_BLCKSZ};
-use zenith_utils::pq_proto::SystemId;
 
 pub const SK_MAGIC: u32 = 0xcafeceefu32;
 pub const SK_FORMAT_VERSION: u32 = 1;
@@ -38,9 +36,9 @@ pub struct NodeId {
     uuid: [u8; 16],
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
 pub struct ServerInfo {
-    /// proposer-safekeeper protocol version
+    /// proxy-safekeeper protocol version
     pub protocol_version: u32,
     /// Postgres server version
     pub pg_version: u32,
@@ -53,7 +51,7 @@ pub struct ServerInfo {
     pub wal_seg_size: u32,
 }
 
-/// Vote request sent from proposer to safekeepers
+/// Vote request sent from proxy to safekeepers
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 struct RequestVote {
     node_id: NodeId,
@@ -89,7 +87,7 @@ impl SafeKeeperInfo {
             format_version: SK_FORMAT_VERSION,
             epoch: 0,
             server: ServerInfo {
-                protocol_version: SK_PROTOCOL_VERSION, /* proposer-safekeeper protocol version */
+                protocol_version: SK_PROTOCOL_VERSION, /* proxy-safekeeper protocol version */
                 pg_version: UNKNOWN_SERVER_VERSION,    /* Postgres server version */
                 node_id: NodeId {
                     term: 0,
@@ -108,7 +106,7 @@ impl SafeKeeperInfo {
     }
 }
 
-/// Request with WAL message sent from proposer to safekeeper.
+/// Request with WAL message sent from proxy to safekeeper.
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 struct SafeKeeperRequest {
     /// Sender's node identifier (looks like we do not need it for TCP streaming connection)
@@ -117,13 +115,13 @@ struct SafeKeeperRequest {
     begin_lsn: Lsn,
     /// end position of message in WAL
     end_lsn: Lsn,
-    /// restart LSN position  (minimal LSN which may be needed by proposer to perform recovery)
+    /// restart LSN position  (minimal LSN which may be needed by proxy to perform recovery)
     restart_lsn: Lsn,
     /// LSN committed by quorum of safekeepers
     commit_lsn: Lsn,
 }
 
-/// Report safekeeper state to proposer
+/// Report safekeeper state to proxy
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 struct SafeKeeperResponse {
     epoch: u64,
@@ -144,48 +142,6 @@ pub struct ReceiveWalConn {
     pub conf: WalAcceptorConf,
 }
 
-///
-/// Periodically request pageserver to call back.
-/// If pageserver already has replication channel, it will just ignore this request
-///
-fn request_callback(conf: WalAcceptorConf, timelineid: ZTimelineId) {
-    let addr = conf.pageserver_addr.unwrap();
-    let ps_connstr = format!(
-        "host={} port={} dbname={} user={}",
-        addr.ip(),
-        addr.port(),
-        "no_db",
-        "no_user",
-    );
-    let callme = format!(
-        "callmemaybe {} host={} port={} options='-c ztimelineid={}'",
-        timelineid,
-        conf.listen_addr.ip(),
-        conf.listen_addr.port(),
-        timelineid
-    );
-    loop {
-        info!(
-            "requesting page server to connect to us: start {} {}",
-            ps_connstr, callme
-        );
-        match Client::connect(&ps_connstr, NoTls) {
-            Ok(mut client) => {
-                if let Err(e) = client.simple_query(&callme) {
-                    error!("Failed to send callme request to pageserver: {}", e);
-                }
-            }
-            Err(e) => error!("Failed to connect to pageserver {}: {}", &ps_connstr, e),
-        }
-
-        if let Some(period) = conf.recall_period {
-            sleep(period);
-        } else {
-            break;
-        }
-    }
-}
-
 impl ReceiveWalConn {
     pub fn new(socket: TcpStream, conf: WalAcceptorConf) -> Result<ReceiveWalConn> {
         let peer_addr = socket.peer_addr()?;
@@ -204,6 +160,32 @@ impl ReceiveWalConn {
         Ok(T::des_from(&mut self.stream_in)?)
     }
 
+    fn request_callback(&self) -> std::result::Result<(), postgres::error::Error> {
+        if let Some(addr) = self.conf.pageserver_addr {
+            let ps_connstr = format!(
+                "host={} port={} dbname={} user={}",
+                addr.ip(),
+                addr.port(),
+                "no_db",
+                "no_user",
+            );
+            let callme = format!(
+                "callmemaybe {} host={} port={} options='-c ztimelineid={}'",
+                self.timeline.get().timelineid,
+                self.conf.listen_addr.ip(),
+                self.conf.listen_addr.port(),
+                self.timeline.get().timelineid
+            );
+            info!(
+                "requesting page server to connect to us: start {} {}",
+                ps_connstr, callme
+            );
+            let mut client = Client::connect(&ps_connstr, NoTls)?;
+            client.simple_query(&callme)?;
+        }
+        Ok(())
+    }
+
     /// Receive WAL from wal_proposer
     pub fn run(&mut self) -> Result<()> {
         // Receive information about server
@@ -213,8 +195,8 @@ impl ReceiveWalConn {
             self.peer_addr, server_info.system_id, server_info.timeline_id,
         );
         // FIXME: also check that the system identifier matches
-        self.timeline
-            .set(&self.conf, server_info.timeline_id, true)?;
+        self.timeline.set(server_info.timeline_id)?;
+        self.timeline.get().load_control_file(&self.conf)?;
 
         let mut my_info = self.timeline.get().get_info();
 
@@ -238,7 +220,7 @@ impl ReceiveWalConn {
 
         /* Update information about server, but preserve locally stored node_id */
         let node_id = my_info.server.node_id;
-        my_info.server = server_info.clone();
+        my_info.server = server_info;
         my_info.server.node_id = node_id;
 
         /* Calculate WAL end based on local data */
@@ -246,14 +228,14 @@ impl ReceiveWalConn {
         my_info.flush_lsn = flush_lsn;
         my_info.server.timeline = timeline;
 
-        /* Report my identifier to proposer */
+        /* Report my identifier to proxy */
         my_info.ser_into(&mut self.stream_out)?;
 
         /* Wait for vote request */
         let prop = self.read_req::<RequestVote>()?;
         /* This is Paxos check which should ensure that only one master can perform commits */
         if prop.node_id < my_info.server.node_id {
-            /* Send my node-id to inform proposer that it's candidate was rejected */
+            /* Send my node-id to inform proxy that it's candidate was rejected */
             my_info.server.node_id.ser_into(&mut self.stream_out)?;
             bail!(
                 "Reject connection attempt with term {} because my term is {}",
@@ -269,17 +251,15 @@ impl ReceiveWalConn {
         let mut flushed_restart_lsn = Lsn(0);
         let wal_seg_size = server_info.wal_seg_size as usize;
 
-        /* Acknowledge the proposed candidate by returning it to the proposer */
+        /* Acknowledge the proposed candidate by returning it to the proxy */
         prop.node_id.ser_into(&mut self.stream_out)?;
 
-        if self.conf.pageserver_addr.is_some() {
-            // Need to establish replication channel with page server.
-            // Add far as replication in postgres is initiated by receiver, we should use callme mechanism
-            let conf = self.conf.clone();
-            let timelineid = self.timeline.get().timelineid;
-            thread::spawn(move || {
-                request_callback(conf, timelineid);
-            });
+        // Need to establish replication channel with page server.
+        // Add far as replication in postgres is initiated by receiver, we should use callme mechanism
+        if let Err(e) = self.request_callback() {
+            // Do not treate it as fatal error and continue work
+            // FIXME: we should retry after a while...
+            error!("Failed to send callme request to pageserver: {}", e);
         }
 
         info!(
@@ -323,7 +303,7 @@ impl ReceiveWalConn {
             /*
              * Epoch switch happen when written WAL record cross the boundary.
              * The boundary is maximum of last WAL position at this node (FlushLSN) and global
-             * maximum (vcl) determined by WAL proposer during handshake.
+             * maximum (vcl) determined by safekeeper_proxy during handshake.
              * Switching epoch means that node completes recovery and start writing in the WAL new data.
              */
             if my_info.epoch < prop.epoch && end_pos > max(my_info.flush_lsn, prop.vcl) {

walkeeper/src/replication.rs

@@ -1,28 +1,29 @@
-//! This module implements the streaming side of replication protocol, starting
-//! with the "START REPLICATION" message.
+//! This module implements the replication protocol, starting with the
+//! "START REPLICATION" message.
 
-use crate::send_wal::SendWalHandler;
+use crate::pq_protocol::{BeMessage, FeMessage};
+use crate::send_wal::SendWalConn;
 use crate::timeline::{Timeline, TimelineTools};
-use anyhow::{anyhow, Result};
-use bytes::Bytes;
+use crate::WalAcceptorConf;
+use anyhow::{anyhow, bail, Result};
+use bytes::{BufMut, Bytes, BytesMut};
 use log::*;
 use postgres_ffi::xlog_utils::{get_current_timestamp, TimestampTz, XLogFileName, MAX_SEND_SIZE};
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 use std::cmp::min;
 use std::fs::File;
-use std::io::{BufReader, Read, Seek, SeekFrom};
+use std::io::{BufReader, Read, Seek, SeekFrom, Write};
 use std::net::TcpStream;
 use std::path::Path;
 use std::sync::Arc;
-use std::thread::sleep;
-use std::time::Duration;
 use std::{str, thread};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
-use zenith_utils::postgres_backend::PostgresBackend;
-use zenith_utils::pq_proto::{BeMessage, FeMessage, XLogDataBody};
 
+const XLOG_HDR_SIZE: usize = 1 + 8 * 3; /* 'w' + startPos + walEnd + timestamp */
+const LIBPQ_HDR_SIZE: usize = 5; /* 1 byte with message type + 4 bytes length */
+const LIBPQ_MSG_SIZE_OFFS: usize = 1;
 pub const END_REPLICATION_MARKER: Lsn = Lsn::MAX;
 
 type FullTransactionId = u64;
@@ -37,16 +38,29 @@ pub struct HotStandbyFeedback {
 
 /// A network connection that's speaking the replication protocol.
 pub struct ReplicationConn {
+    timeline: Option<Arc<Timeline>>,
+    /// Postgres connection, buffered input
+    ///
     /// This is an `Option` because we will spawn a background thread that will
     /// `take` it from us.
     stream_in: Option<BufReader<TcpStream>>,
+    /// Postgres connection, output
+    stream_out: TcpStream,
+    /// wal acceptor configuration
+    conf: WalAcceptorConf,
+    /// assigned application name
+    appname: Option<String>,
 }
 
 impl ReplicationConn {
-    /// Create a new `ReplicationConn`
-    pub fn new(pgb: &mut PostgresBackend) -> Self {
+    /// Create a new `SendWal`, consuming the `Connection`.
+    pub fn new(conn: SendWalConn) -> Self {
         Self {
-            stream_in: pgb.take_stream_in(),
+            timeline: conn.timeline,
+            stream_in: Some(conn.stream_in),
+            stream_out: conn.stream_out,
+            conf: conn.conf,
+            appname: None,
         }
     }
 
@@ -58,9 +72,9 @@ impl ReplicationConn {
         // Wait for replica's feedback.
         // We only handle `CopyData` messages. Anything else is ignored.
         loop {
-            match FeMessage::read(&mut stream_in)? {
-                Some(FeMessage::CopyData(m)) => {
-                    let feedback = HotStandbyFeedback::des(&m)?;
+            match FeMessage::read_from(&mut stream_in)? {
+                FeMessage::CopyData(m) => {
+                    let feedback = HotStandbyFeedback::des(&m.body)?;
                     timeline.add_hs_feedback(feedback)
                 }
                 msg => {
@@ -104,14 +118,9 @@ impl ReplicationConn {
     ///
     /// Handle START_REPLICATION replication command
     ///
-    pub fn run(
-        &mut self,
-        swh: &mut SendWalHandler,
-        pgb: &mut PostgresBackend,
-        cmd: &Bytes,
-    ) -> Result<()> {
+    pub fn run(&mut self, cmd: &Bytes) -> Result<()> {
         // spawn the background thread which receives HotStandbyFeedback messages.
-        let bg_timeline = Arc::clone(swh.timeline.get());
+        let bg_timeline = Arc::clone(self.timeline.get());
         let bg_stream_in = self.stream_in.take().unwrap();
 
         thread::spawn(move || {
@@ -122,27 +131,23 @@ impl ReplicationConn {
 
         let (mut start_pos, mut stop_pos) = Self::parse_start_stop(&cmd)?;
 
-        let mut wal_seg_size: usize;
-        loop {
-            wal_seg_size = swh.timeline.get().get_info().server.wal_seg_size as usize;
-            if wal_seg_size == 0 {
-                error!("Can not start replication before connecting to wal_proposer");
-                sleep(Duration::from_secs(1));
-            } else {
-                break;
-            }
+        let wal_seg_size = self.timeline.get().get_info().server.wal_seg_size as usize;
+        if wal_seg_size == 0 {
+            bail!("Can not start replication before connecting to wal_proposer");
         }
-        let (wal_end, timeline) = swh.timeline.find_end_of_wal(&swh.conf.data_dir, false);
+        let (wal_end, timeline) = self.timeline.find_end_of_wal(&self.conf.data_dir, false);
         if start_pos == Lsn(0) {
             start_pos = wal_end;
         }
-        if stop_pos == Lsn(0) && swh.appname == Some("wal_proposer_recovery".to_string()) {
+        if stop_pos == Lsn(0) && self.appname == Some("wal_proposer_recovery".to_string()) {
             stop_pos = wal_end;
         }
         info!("Start replication from {} till {}", start_pos, stop_pos);
 
-        // switch to copy
-        pgb.write_message(&BeMessage::CopyBothResponse)?;
+        let mut outbuf = BytesMut::new();
+        BeMessage::write(&mut outbuf, &BeMessage::Copy);
+        self.send(&outbuf)?;
+        outbuf.clear();
 
         let mut end_pos: Lsn;
         let mut wal_file: Option<File> = None;
@@ -158,7 +163,7 @@ impl ReplicationConn {
                 end_pos = stop_pos;
             } else {
                 /* normal mode */
-                let timeline = swh.timeline.get();
+                let timeline = self.timeline.get();
                 end_pos = timeline.wait_for_lsn(start_pos);
             }
             if end_pos == END_REPLICATION_MARKER {
@@ -172,8 +177,8 @@ impl ReplicationConn {
                     // Open a new file.
                     let segno = start_pos.segment_number(wal_seg_size);
                     let wal_file_name = XLogFileName(timeline, segno, wal_seg_size);
-                    let timeline_id = swh.timeline.get().timelineid.to_string();
-                    let wal_file_path = swh.conf.data_dir.join(timeline_id).join(wal_file_name);
+                    let timeline_id = self.timeline.get().timelineid.to_string();
+                    let wal_file_path = self.conf.data_dir.join(timeline_id).join(wal_file_name);
                     Self::open_wal_file(&wal_file_path)?
                 }
             };
@@ -186,19 +191,32 @@ impl ReplicationConn {
             let send_size = min(send_size, wal_seg_size - xlogoff);
             let send_size = min(send_size, MAX_SEND_SIZE);
 
+            let msg_size = LIBPQ_HDR_SIZE + XLOG_HDR_SIZE + send_size;
+
             // Read some data from the file.
             let mut file_buf = vec![0u8; send_size];
             file.seek(SeekFrom::Start(xlogoff as u64))?;
             file.read_exact(&mut file_buf)?;
 
             // Write some data to the network socket.
-            pgb.write_message(&BeMessage::XLogData(XLogDataBody {
-                wal_start: start_pos.0,
-                wal_end: end_pos.0,
-                timestamp: get_current_timestamp(),
-                data: &file_buf,
-            }))?;
+            // FIXME: turn these into structs.
+            // 'd' is CopyData;
+            // 'w' is "WAL records"
+            // https://www.postgresql.org/docs/9.1/protocol-message-formats.html
+            // src/backend/replication/walreceiver.c
+            outbuf.clear();
+            outbuf.put_u8(b'd');
+            outbuf.put_u32((msg_size - LIBPQ_MSG_SIZE_OFFS) as u32);
+            outbuf.put_u8(b'w');
+            outbuf.put_u64(start_pos.0);
+            outbuf.put_u64(end_pos.0);
+            outbuf.put_u64(get_current_timestamp());
 
+            assert!(outbuf.len() + file_buf.len() == msg_size);
+            // This thread has exclusive access to the TcpStream, so it's fine
+            // to do this as two separate calls.
+            self.send(&outbuf)?;
+            self.send(&file_buf)?;
             start_pos += send_size as u64;
 
             debug!("Sent WAL to page server up to {}", end_pos);
@@ -211,4 +229,10 @@ impl ReplicationConn {
         }
         Ok(())
     }
+
+    /// Send messages on the network.
+    fn send(&mut self, buf: &[u8]) -> Result<()> {
+        self.stream_out.write_all(buf.as_ref())?;
+        Ok(())
+    }
 }

walkeeper/src/send_wal.rs

@@ -1,78 +1,111 @@
-//! Part of WAL acceptor pretending to be Postgres, streaming xlog to
-//! pageserver/any other consumer and answering to some utility queries.
+//! This implements the libpq replication protocol between wal_acceptor
+//! and replicas/pagers
 //!
 
+use crate::pq_protocol::{
+    BeMessage, FeMessage, FeStartupMessage, RowDescriptor, StartupRequestCode,
+};
 use crate::replication::ReplicationConn;
 use crate::timeline::{Timeline, TimelineTools};
 use crate::WalAcceptorConf;
 use anyhow::{bail, Result};
-use bytes::Bytes;
-use pageserver::ZTimelineId;
-use std::str::FromStr;
+use bytes::BytesMut;
+use log::*;
+use std::io::{BufReader, Write};
+use std::net::{SocketAddr, TcpStream};
 use std::sync::Arc;
-use zenith_utils::postgres_backend;
-use zenith_utils::postgres_backend::PostgresBackend;
-use zenith_utils::pq_proto::{BeMessage, FeStartupMessage, RowDescriptor, JSON_OID, TEXT_OID};
 
-/// Handler for streaming WAL from acceptor
-pub struct SendWalHandler {
+/// A network connection that's speaking the libpq replication protocol.
+pub struct SendWalConn {
+    pub timeline: Option<Arc<Timeline>>,
+    /// Postgres connection, buffered input
+    pub stream_in: BufReader<TcpStream>,
+    /// Postgres connection, output
+    pub stream_out: TcpStream,
+    /// The cached result of socket.peer_addr()
+    pub peer_addr: SocketAddr,
     /// wal acceptor configuration
     pub conf: WalAcceptorConf,
     /// assigned application name
-    pub appname: Option<String>,
-    pub timeline: Option<Arc<Timeline>>,
+    appname: Option<String>,
 }
 
-impl postgres_backend::Handler for SendWalHandler {
-    fn startup(&mut self, _pgb: &mut PostgresBackend, sm: &FeStartupMessage) -> Result<()> {
-        match sm.params.get("ztimelineid") {
-            Some(ref ztimelineid) => match ZTimelineId::from_str(ztimelineid) {
-                Ok(ztlid) => {
-                    self.timeline.set(&self.conf, ztlid, false)?;
-                }
-                Err(e) => {
-                    bail!("failed to parse ztimelineid: {}", e)
-                }
-            },
-            _ => bail!("timelineid is required"),
-        }
-        if let Some(app_name) = sm.params.get("application_name") {
-            self.appname = Some(app_name.clone());
-        }
-        Ok(())
-    }
-
-    fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: Bytes) -> Result<()> {
-        if query_string.starts_with(b"IDENTIFY_SYSTEM") {
-            self.handle_identify_system(pgb)?;
-        } else if query_string.starts_with(b"START_REPLICATION") {
-            ReplicationConn::new(pgb).run(self, pgb, &query_string)?;
-        } else if query_string.starts_with(b"state") {
-            self.handle_state(pgb)?;
-        } else if query_string.to_ascii_lowercase().starts_with(b"set ") {
-            // have it because psycopg2 executes "SET datestyle TO 'ISO'"
-            // on connect
-            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else {
-            bail!("Unexpected command {:?}", query_string);
-        }
-        Ok(())
-    }
-}
-
-impl SendWalHandler {
-    pub fn new(conf: WalAcceptorConf) -> Self {
-        SendWalHandler {
+impl SendWalConn {
+    /// Create a new `SendWal`, consuming the `Connection`.
+    pub fn new(socket: TcpStream, conf: WalAcceptorConf) -> Result<Self> {
+        let peer_addr = socket.peer_addr()?;
+        let conn = SendWalConn {
+            timeline: None,
+            stream_in: BufReader::new(socket.try_clone()?),
+            stream_out: socket,
+            peer_addr,
             conf,
             appname: None,
-            timeline: None,
+        };
+        Ok(conn)
+    }
+
+    ///
+    /// Send WAL to replica or WAL receiver using standard libpq replication protocol
+    ///
+    pub fn run(mut self) -> Result<()> {
+        let peer_addr = self.peer_addr;
+        info!("WAL sender to {:?} is started", peer_addr);
+
+        // Handle the startup message first.
+
+        let m = FeStartupMessage::read_from(&mut self.stream_in)?;
+        trace!("got startup message {:?}", m);
+        match m.kind {
+            StartupRequestCode::NegotiateGss | StartupRequestCode::NegotiateSsl => {
+                let mut buf = BytesMut::new();
+                BeMessage::write(&mut buf, &BeMessage::Negotiate);
+                info!("SSL requested");
+                self.stream_out.write_all(&buf)?;
+            }
+            StartupRequestCode::Normal => {
+                let mut buf = BytesMut::new();
+                BeMessage::write(&mut buf, &BeMessage::AuthenticationOk);
+                BeMessage::write(&mut buf, &BeMessage::ReadyForQuery);
+                self.stream_out.write_all(&buf)?;
+                self.timeline.set(m.timelineid)?;
+                self.appname = m.appname;
+            }
+            StartupRequestCode::Cancel => return Ok(()),
         }
+
+        loop {
+            let msg = FeMessage::read_from(&mut self.stream_in)?;
+            match msg {
+                FeMessage::Query(q) => {
+                    trace!("got query {:?}", q.body);
+
+                    if q.body.starts_with(b"IDENTIFY_SYSTEM") {
+                        self.handle_identify_system()?;
+                    } else if q.body.starts_with(b"START_REPLICATION") {
+                        // Create a new replication object, consuming `self`.
+                        ReplicationConn::new(self).run(&q.body)?;
+                        break;
+                    } else {
+                        bail!("Unexpected command {:?}", q.body);
+                    }
+                }
+                FeMessage::Terminate => {
+                    break;
+                }
+                _ => {
+                    bail!("unexpected message");
+                }
+            }
+        }
+        info!("WAL sender to {:?} is finished", peer_addr);
+        Ok(())
     }
 
     ///
     /// Handle IDENTIFY_SYSTEM replication command
     ///
-    fn handle_identify_system(&mut self, pgb: &mut PostgresBackend) -> Result<()> {
+    fn handle_identify_system(&mut self) -> Result<()> {
         let (start_pos, timeline) = self.timeline.find_end_of_wal(&self.conf.data_dir, false);
         let lsn = start_pos.to_string();
         let tli = timeline.to_string();
@@ -81,56 +114,42 @@ impl SendWalHandler {
         let tli_bytes = tli.as_bytes();
         let sysid_bytes = sysid.as_bytes();
 
-        pgb.write_message_noflush(&BeMessage::RowDescription(&[
-            RowDescriptor {
-                name: b"systemid",
-                typoid: TEXT_OID,
-                typlen: -1,
-                ..Default::default()
-            },
-            RowDescriptor {
-                name: b"timeline",
-                typoid: 23,
-                typlen: 4,
-                ..Default::default()
-            },
-            RowDescriptor {
-                name: b"xlogpos",
-                typoid: TEXT_OID,
-                typlen: -1,
-                ..Default::default()
-            },
-            RowDescriptor {
-                name: b"dbname",
-                typoid: TEXT_OID,
-                typlen: -1,
-                ..Default::default()
-            },
-        ]))?
-        .write_message_noflush(&BeMessage::DataRow(&[
-            Some(sysid_bytes),
-            Some(tli_bytes),
-            Some(lsn_bytes),
-            None,
-        ]))?
-        .write_message(&BeMessage::CommandComplete(b"IDENTIFY_SYSTEM"))?;
-        Ok(())
-    }
-
-    /// Send current state of safekeeper
-    fn handle_state(&mut self, pgb: &mut PostgresBackend) -> Result<()> {
-        let my_info = self.timeline.get().get_info();
-        pgb.write_message_noflush(&BeMessage::RowDescription(&[RowDescriptor {
-            name: b"state",
-            typoid: JSON_OID,
-            typlen: -1,
-            ..Default::default()
-        }]))?
-        .write_message_noflush(&BeMessage::DataRow(&[Some(
-            serde_json::to_string(&my_info).unwrap().as_bytes(),
-        )]))?
-        .write_message(&BeMessage::CommandComplete(b"IDENTIFY_SYSTEM"))?;
-
+        let mut outbuf = BytesMut::new();
+        BeMessage::write(
+            &mut outbuf,
+            &BeMessage::RowDescription(&[
+                RowDescriptor {
+                    name: b"systemid\0",
+                    typoid: 25,
+                    typlen: -1,
+                },
+                RowDescriptor {
+                    name: b"timeline\0",
+                    typoid: 23,
+                    typlen: 4,
+                },
+                RowDescriptor {
+                    name: b"xlogpos\0",
+                    typoid: 25,
+                    typlen: -1,
+                },
+                RowDescriptor {
+                    name: b"dbname\0",
+                    typoid: 25,
+                    typlen: -1,
+                },
+            ]),
+        );
+        BeMessage::write(
+            &mut outbuf,
+            &BeMessage::DataRow(&[Some(sysid_bytes), Some(tli_bytes), Some(lsn_bytes), None]),
+        );
+        BeMessage::write(
+            &mut outbuf,
+            &BeMessage::CommandComplete(b"IDENTIFY_SYSTEM\0"),
+        );
+        BeMessage::write(&mut outbuf, &BeMessage::ReadyForQuery);
+        self.stream_out.write_all(&outbuf)?;
         Ok(())
     }
 }

walkeeper/src/timeline.rs

@@ -55,12 +55,10 @@ impl SharedState {
     }
 
     /// Load and lock control file (prevent running more than one instance of safekeeper)
-    /// If create=false and file doesn't exist, bails out.
     pub fn load_control_file(
         &mut self,
         conf: &WalAcceptorConf,
         timelineid: ZTimelineId,
-        create: bool,
     ) -> Result<()> {
         if self.control_file.is_some() {
             info!("control file for timeline {} is already open", timelineid);
@@ -71,17 +69,13 @@ impl SharedState {
             .data_dir
             .join(timelineid.to_string())
             .join(CONTROL_FILE_NAME);
-        info!(
-            "loading control file {}, create={}",
-            control_file_path.display(),
-            create
-        );
-        let mut opts = OpenOptions::new();
-        opts.read(true).write(true);
-        if create {
-            opts.create(true);
-        }
-        match opts.open(&control_file_path) {
+        info!("loading control file {}", control_file_path.display());
+        match OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .open(&control_file_path)
+        {
             Ok(file) => {
                 // Lock file to prevent two or more active wal_acceptors
                 match file.try_lock_exclusive() {
@@ -97,36 +91,29 @@ impl SharedState {
                 self.control_file = Some(file);
 
                 let cfile_ref = self.control_file.as_mut().unwrap();
-                if cfile_ref.metadata().unwrap().len() == 0 {
-                    if !create {
-                        bail!("control file is empty");
+                match SafeKeeperInfo::des_from(cfile_ref) {
+                    Err(e) => {
+                        warn!("read from {:?} failed: {}", control_file_path, e);
                     }
-                } else {
-                    match SafeKeeperInfo::des_from(cfile_ref) {
-                        Err(e) => {
-                            bail!("failed to read control file {:?}: {}", control_file_path, e);
+                    Ok(info) => {
+                        if info.magic != SK_MAGIC {
+                            bail!("Invalid control file magic: {}", info.magic);
                         }
-                        Ok(info) => {
-                            if info.magic != SK_MAGIC {
-                                bail!("Invalid control file magic: {}", info.magic);
-                            }
-                            if info.format_version != SK_FORMAT_VERSION {
-                                bail!(
-                                    "Incompatible format version: {} vs. {}",
-                                    info.format_version,
-                                    SK_FORMAT_VERSION
-                                );
-                            }
-                            self.info = info;
+                        if info.format_version != SK_FORMAT_VERSION {
+                            bail!(
+                                "Incompatible format version: {} vs. {}",
+                                info.format_version,
+                                SK_FORMAT_VERSION
+                            );
                         }
+                        self.info = info;
                     }
                 }
             }
             Err(e) => {
-                bail!(
+                panic!(
                     "Failed to open control file {:?}: {}",
-                    &control_file_path,
-                    e
+                    &control_file_path, e
                 );
             }
         }
@@ -211,9 +198,9 @@ impl Timeline {
         shared_state.hs_feedback.clone()
     }
 
-    pub fn load_control_file(&self, conf: &WalAcceptorConf, create: bool) -> Result<()> {
+    pub fn load_control_file(&self, conf: &WalAcceptorConf) -> Result<()> {
         let mut shared_state = self.mutex.lock().unwrap();
-        shared_state.load_control_file(conf, self.timelineid, create)
+        shared_state.load_control_file(conf, self.timelineid)
     }
 
     pub fn save_control_file(&self, sync: bool) -> Result<()> {
@@ -224,23 +211,17 @@ impl Timeline {
 
 // Utilities needed by various Connection-like objects
 pub trait TimelineTools {
-    fn set(&mut self, conf: &WalAcceptorConf, timeline_id: ZTimelineId, create: bool)
-        -> Result<()>;
+    fn set(&mut self, timeline_id: ZTimelineId) -> Result<()>;
     fn get(&self) -> &Arc<Timeline>;
     fn find_end_of_wal(&self, data_dir: &Path, precise: bool) -> (Lsn, TimeLineID);
 }
 
 impl TimelineTools for Option<Arc<Timeline>> {
-    fn set(
-        &mut self,
-        conf: &WalAcceptorConf,
-        timeline_id: ZTimelineId,
-        create: bool,
-    ) -> Result<()> {
+    fn set(&mut self, timeline_id: ZTimelineId) -> Result<()> {
         // We will only set the timeline once. If it were to ever change,
         // anyone who cloned the Arc would be out of date.
         assert!(self.is_none());
-        *self = Some(GlobalTimelines::get(conf, timeline_id, create)?);
+        *self = Some(GlobalTimelines::store(timeline_id)?);
         Ok(())
     }
 
@@ -262,16 +243,11 @@ lazy_static! {
 }
 
 /// A zero-sized struct used to manage access to the global timelines map.
-pub struct GlobalTimelines;
+struct GlobalTimelines;
 
 impl GlobalTimelines {
-    /// Get a timeline with control file loaded from the global TIMELINES map.
-    /// If control file doesn't exist and create=false, bails out.
-    pub fn get(
-        conf: &WalAcceptorConf,
-        timeline_id: ZTimelineId,
-        create: bool,
-    ) -> Result<Arc<Timeline>> {
+    /// Store a new timeline into the global TIMELINES map.
+    fn store(timeline_id: ZTimelineId) -> Result<Arc<Timeline>> {
         let mut timelines = TIMELINES.lock().unwrap();
 
         match timelines.get(&timeline_id) {
@@ -282,10 +258,9 @@ impl GlobalTimelines {
 
                 let shared_state = SharedState::new();
 
-                let new_tli = Arc::new(Timeline::new(timeline_id, shared_state));
-                new_tli.load_control_file(conf, create)?;
-                timelines.insert(timeline_id, Arc::clone(&new_tli));
-                Ok(new_tli)
+                let new_tid = Arc::new(Timeline::new(timeline_id, shared_state));
+                timelines.insert(timeline_id, Arc::clone(&new_tid));
+                Ok(new_tid)
             }
         }
     }

walkeeper/src/wal_service.rs

@@ -9,9 +9,8 @@ use std::net::{TcpListener, TcpStream};
 use std::thread;
 
 use crate::receive_wal::ReceiveWalConn;
-use crate::send_wal::SendWalHandler;
+use crate::send_wal::SendWalConn;
 use crate::WalAcceptorConf;
-use zenith_utils::postgres_backend::PostgresBackend;
 
 /// Accept incoming TCP connections and spawn them into a background thread.
 pub fn thread_main(conf: WalAcceptorConf) -> Result<()> {
@@ -49,10 +48,7 @@ fn handle_socket(mut socket: TcpStream, conf: WalAcceptorConf) -> Result<()> {
         socket.read_exact(&mut [0u8; 4])?;
         ReceiveWalConn::new(socket, conf)?.run()?; // internal protocol between wal_proposer and wal_acceptor
     } else {
-        let mut conn_handler = SendWalHandler::new(conf);
-        let mut pgbackend = PostgresBackend::new(socket)?;
-        // libpq replication protocol between wal_acceptor and replicas/pagers
-        pgbackend.run(&mut conn_handler)?;
+        SendWalConn::new(socket, conf)?.run()?; // libpq replication protocol between wal_acceptor and replicas/pagers
     }
     Ok(())
 }

zenith/src/main.rs

@@ -91,12 +91,6 @@ fn main() -> Result<()> {
                         ),
                 ),
         )
-        .subcommand(
-            SubCommand::with_name("push")
-                .about("Push timeline to remote pageserver")
-                .arg(Arg::with_name("timeline").required(true))
-                .arg(Arg::with_name("remote").required(true)),
-        )
         .get_matches();
 
     // Create config file
@@ -177,14 +171,6 @@ fn main() -> Result<()> {
                 exit(1);
             }
         }
-
-        ("push", Some(push_match)) => {
-            if let Err(e) = handle_push(push_match, &env) {
-                eprintln!("push operation failed: {}", e);
-                exit(1);
-            }
-        }
-
         _ => {}
     };
 
@@ -427,20 +413,3 @@ fn handle_remote(remote_match: &ArgMatches, local_env: &LocalEnv) -> Result<()>
 
     Ok(())
 }
-
-fn handle_push(push_match: &ArgMatches, local_env: &LocalEnv) -> Result<()> {
-    let timeline_id_str = push_match.value_of("timeline").unwrap();
-    ZTimelineId::from_str(timeline_id_str)?;
-
-    let remote_name = push_match.value_of("remote").unwrap();
-    let remote = local_env
-        .remotes
-        .get(remote_name)
-        .ok_or_else(|| anyhow!("remote {} not found", remote_name))?;
-
-    let page_server = PageServerNode::from_env(local_env);
-    let mut client = page_server.page_server_psql_client()?;
-    client.simple_query(&format!("request_push {} {}", timeline_id_str, remote))?;
-
-    Ok(())
-}

zenith_utils/Cargo.toml

@@ -5,10 +5,6 @@ authors = ["Eric Seppanen <eric@zenith.tech>"]
 edition = "2018"
 
 [dependencies]
-anyhow = "1.0"
-bytes = "1.0.1"
-byteorder = "1.4.3"
-log = "0.4.14"
 serde = { version = "1.0", features = ["derive"] }
 bincode = "1.3"
 thiserror = "1.0"

zenith_utils/src/lib.rs

@@ -10,6 +10,3 @@ pub mod seqwait;
 // pub mod seqwait_async;
 
 pub mod bin_ser;
-
-pub mod postgres_backend;
-pub mod pq_proto;

zenith_utils/src/postgres_backend.rs

@@ -1,186 +0,0 @@
-//! Server-side synchronous Postgres connection, as limited as we need.
-//! To use, create PostgresBackend and run() it, passing the Handler
-//! implementation determining how to process the queries. Currently its API
-//! is rather narrow, but we can extend it once required.
-
-use crate::pq_proto::{BeMessage, FeMessage, FeStartupMessage, StartupRequestCode};
-use anyhow::bail;
-use anyhow::Result;
-use bytes::{Bytes, BytesMut};
-use log::*;
-use std::io;
-use std::io::{BufReader, Write};
-use std::net::{Shutdown, TcpStream};
-
-pub trait Handler {
-    /// Handle single query.
-    /// postgres_backend will issue ReadyForQuery after calling this (this
-    /// might be not what we want after CopyData streaming, but currently we don't
-    /// care).
-    fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: Bytes) -> Result<()>;
-    /// Called on startup packet receival, allows to process params.
-    fn startup(&mut self, _pgb: &mut PostgresBackend, _sm: &FeStartupMessage) -> Result<()> {
-        Ok(())
-    }
-}
-
-pub struct PostgresBackend {
-    // replication.rs wants to handle reading on its own in separate thread, so
-    // wrap in Option to be able to take and transfer the BufReader. Ugly, but I
-    // have no better ideas.
-    stream_in: Option<BufReader<TcpStream>>,
-    stream_out: TcpStream,
-    // Output buffer. c.f. BeMessage::write why we are using BytesMut here.
-    buf_out: BytesMut,
-    init_done: bool,
-}
-
-// In replication.rs a separate thread is reading keepalives from the
-// socket. When main one finishes, tell it to get down by shutdowning the
-// socket.
-impl Drop for PostgresBackend {
-    fn drop(&mut self) {
-        let _res = self.stream_out.shutdown(Shutdown::Both);
-    }
-}
-
-impl PostgresBackend {
-    pub fn new(socket: TcpStream) -> Result<Self, std::io::Error> {
-        let mut pb = PostgresBackend {
-            stream_in: None,
-            stream_out: socket,
-            buf_out: BytesMut::with_capacity(10 * 1024),
-            init_done: false,
-        };
-        // if socket cloning fails, report the error and bail out
-        pb.stream_in = match pb.stream_out.try_clone() {
-            Ok(read_sock) => Some(BufReader::new(read_sock)),
-            Err(error) => {
-                let errmsg = format!("{}", error);
-                let _res = pb.write_message_noflush(&BeMessage::ErrorResponse(errmsg));
-                return Err(error);
-            }
-        };
-        Ok(pb)
-    }
-
-    /// Get direct reference (into the Option) to the read stream.
-    fn get_stream_in(&mut self) -> Result<&mut BufReader<TcpStream>> {
-        match self.stream_in {
-            Some(ref mut stream_in) => Ok(stream_in),
-            None => bail!("stream_in was taken"),
-        }
-    }
-
-    pub fn take_stream_in(&mut self) -> Option<BufReader<TcpStream>> {
-        self.stream_in.take()
-    }
-
-    /// Read full message or return None if connection is closed.
-    pub fn read_message(&mut self) -> Result<Option<FeMessage>> {
-        if !self.init_done {
-            FeStartupMessage::read(self.get_stream_in()?)
-        } else {
-            FeMessage::read(self.get_stream_in()?)
-        }
-    }
-
-    /// Write message into internal output buffer.
-    pub fn write_message_noflush(&mut self, message: &BeMessage) -> io::Result<&mut Self> {
-        BeMessage::write(&mut self.buf_out, message)?;
-        Ok(self)
-    }
-
-    /// Flush output buffer into the socket.
-    pub fn flush(&mut self) -> io::Result<&mut Self> {
-        self.stream_out.write_all(&self.buf_out)?;
-        self.buf_out.clear();
-        Ok(self)
-    }
-
-    /// Write message into internal buffer and flush it.
-    pub fn write_message(&mut self, message: &BeMessage) -> io::Result<&mut Self> {
-        self.write_message_noflush(message)?;
-        self.flush()
-    }
-
-    pub fn run(&mut self, handler: &mut impl Handler) -> Result<()> {
-        let peer_addr = self.stream_out.peer_addr()?;
-        info!("postgres backend to {:?} started", peer_addr);
-        let mut unnamed_query_string = Bytes::new();
-
-        loop {
-            let msg = self.read_message()?;
-            trace!("got message {:?}", msg);
-            match msg {
-                Some(FeMessage::StartupMessage(m)) => {
-                    trace!("got startup message {:?}", m);
-
-                    if let Err(e) = handler.startup(self, &m) {
-                        // try to send error to the client
-                        let errmsg = format!("{}", e);
-                        self.write_message(&BeMessage::ErrorResponse(errmsg))?;
-                        return Err(e);
-                    }
-
-                    match m.kind {
-                        StartupRequestCode::NegotiateGss | StartupRequestCode::NegotiateSsl => {
-                            info!("SSL requested");
-                            self.write_message(&BeMessage::Negotiate)?;
-                        }
-                        StartupRequestCode::Normal => {
-                            self.write_message_noflush(&BeMessage::AuthenticationOk)?;
-                            // psycopg2 will not connect if client_encoding is not
-                            // specified by the server
-                            self.write_message_noflush(&BeMessage::ParameterStatus)?;
-                            self.write_message(&BeMessage::ReadyForQuery)?;
-                            self.init_done = true;
-                        }
-                        StartupRequestCode::Cancel => break,
-                    }
-                }
-                Some(FeMessage::Query(m)) => {
-                    trace!("got query {:?}", m.body);
-                    // xxx distinguish fatal and recoverable errors?
-                    if let Err(e) = handler.process_query(self, m.body) {
-                        let errmsg = format!("{}", e);
-                        self.write_message_noflush(&BeMessage::ErrorResponse(errmsg))?;
-                    }
-                    self.write_message(&BeMessage::ReadyForQuery)?;
-                }
-                Some(FeMessage::Parse(m)) => {
-                    unnamed_query_string = m.query_string;
-                    self.write_message(&BeMessage::ParseComplete)?;
-                }
-                Some(FeMessage::Describe(_)) => {
-                    self.write_message_noflush(&BeMessage::ParameterDescription)?
-                        .write_message(&BeMessage::NoData)?;
-                }
-                Some(FeMessage::Bind(_)) => {
-                    self.write_message(&BeMessage::BindComplete)?;
-                }
-                Some(FeMessage::Close(_)) => {
-                    self.write_message(&BeMessage::CloseComplete)?;
-                }
-                Some(FeMessage::Execute(_)) => {
-                    handler.process_query(self, unnamed_query_string.clone())?;
-                }
-                Some(FeMessage::Sync) => {
-                    self.write_message(&BeMessage::ReadyForQuery)?;
-                }
-                Some(FeMessage::Terminate) => {
-                    break;
-                }
-                None => {
-                    info!("connection closed");
-                    break;
-                }
-                x => {
-                    bail!("unexpected message type : {:?}", x);
-                }
-            }
-        }
-        info!("postgres backend to {:?} exited", peer_addr);
-        Ok(())
-    }
-}

zenith_utils/src/pq_proto.rs

@@ -1,639 +0,0 @@
-//! Postgres protocol messages serialization-deserialization. See
-//! https://www.postgresql.org/docs/devel/protocol-message-formats.html
-//! on message formats.
-
-use anyhow::{anyhow, bail, Result};
-use byteorder::{BigEndian, ByteOrder};
-use byteorder::{ReadBytesExt, BE};
-use bytes::{Buf, BufMut, Bytes, BytesMut};
-// use postgres_ffi::xlog_utils::TimestampTz;
-use std::collections::HashMap;
-use std::io;
-use std::io::Read;
-use std::str;
-
-pub type Oid = u32;
-pub type SystemId = u64;
-
-#[derive(Debug)]
-pub enum FeMessage {
-    StartupMessage(FeStartupMessage),
-    Query(FeQueryMessage), // Simple query
-    Parse(FeParseMessage), // Extended query protocol
-    Describe(FeDescribeMessage),
-    Bind(FeBindMessage),
-    Execute(FeExecuteMessage),
-    Close(FeCloseMessage),
-    Sync,
-    Terminate,
-    CopyData(Bytes),
-    CopyDone,
-}
-
-#[derive(Debug)]
-pub struct FeStartupMessage {
-    pub version: u32,
-    pub kind: StartupRequestCode,
-    // optional params arriving in startup packet
-    pub params: HashMap<String, String>,
-}
-
-#[derive(Debug)]
-pub enum StartupRequestCode {
-    Cancel,
-    NegotiateSsl,
-    NegotiateGss,
-    Normal,
-}
-
-#[derive(Debug)]
-pub struct FeQueryMessage {
-    pub body: Bytes,
-}
-
-// We only support the simple case of Parse on unnamed prepared statement and
-// no params
-#[derive(Debug)]
-pub struct FeParseMessage {
-    pub query_string: Bytes,
-}
-
-#[derive(Debug)]
-pub struct FeDescribeMessage {
-    pub kind: u8, // 'S' to describe a prepared statement; or 'P' to describe a portal.
-                  // we only support unnamed prepared stmt or portal
-}
-
-// we only support unnamed prepared stmt and portal
-#[derive(Debug)]
-pub struct FeBindMessage {}
-
-// we only support unnamed prepared stmt or portal
-#[derive(Debug)]
-pub struct FeExecuteMessage {
-    /// max # of rows
-    pub maxrows: i32,
-}
-
-// we only support unnamed prepared stmt and portal
-#[derive(Debug)]
-pub struct FeCloseMessage {}
-
-impl FeMessage {
-    /// Read one message from the stream.
-    pub fn read(stream: &mut impl Read) -> anyhow::Result<Option<FeMessage>> {
-        // Each libpq message begins with a message type byte, followed by message length
-        // If the client closes the connection, return None. But if the client closes the
-        // connection in the middle of a message, we will return an error.
-        let tag = match stream.read_u8() {
-            Ok(b) => b,
-            Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
-            Err(e) => return Err(e.into()),
-        };
-        let len = stream.read_u32::<BE>()?;
-
-        // The message length includes itself, so it better be at least 4
-        let bodylen = len
-            .checked_sub(4)
-            .ok_or_else(|| anyhow!("invalid message length: parsing u32"))?;
-
-        // Read message body
-        let mut body_buf: Vec<u8> = vec![0; bodylen as usize];
-        stream.read_exact(&mut body_buf)?;
-
-        let body = Bytes::from(body_buf);
-
-        // Parse it
-        match tag {
-            b'Q' => Ok(Some(FeMessage::Query(FeQueryMessage { body }))),
-            b'P' => Ok(Some(FeParseMessage::parse(body)?)),
-            b'D' => Ok(Some(FeDescribeMessage::parse(body)?)),
-            b'E' => Ok(Some(FeExecuteMessage::parse(body)?)),
-            b'B' => Ok(Some(FeBindMessage::parse(body)?)),
-            b'C' => Ok(Some(FeCloseMessage::parse(body)?)),
-            b'S' => Ok(Some(FeMessage::Sync)),
-            b'X' => Ok(Some(FeMessage::Terminate)),
-            b'd' => Ok(Some(FeMessage::CopyData(body))),
-            b'c' => Ok(Some(FeMessage::CopyDone)),
-            tag => Err(anyhow!("unknown message tag: {},'{:?}'", tag, body)),
-        }
-    }
-}
-
-impl FeStartupMessage {
-    /// Read startup message from the stream.
-    pub fn read(stream: &mut impl std::io::Read) -> anyhow::Result<Option<FeMessage>> {
-        const MAX_STARTUP_PACKET_LENGTH: usize = 10000;
-        const CANCEL_REQUEST_CODE: u32 = (1234 << 16) | 5678;
-        const NEGOTIATE_SSL_CODE: u32 = (1234 << 16) | 5679;
-        const NEGOTIATE_GSS_CODE: u32 = (1234 << 16) | 5680;
-
-        // Read length. If the connection is closed before reading anything (or before
-        // reading 4 bytes, to be precise), return None to indicate that the connection
-        // was closed. This matches the PostgreSQL server's behavior, which avoids noise
-        // in the log if the client opens connection but closes it immediately.
-        let len = match stream.read_u32::<BE>() {
-            Ok(len) => len as usize,
-            Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
-            Err(e) => return Err(e.into()),
-        };
-
-        if len < 4 || len > MAX_STARTUP_PACKET_LENGTH {
-            bail!("invalid message length");
-        }
-
-        let version = stream.read_u32::<BE>()?;
-        let kind = match version {
-            CANCEL_REQUEST_CODE => StartupRequestCode::Cancel,
-            NEGOTIATE_SSL_CODE => StartupRequestCode::NegotiateSsl,
-            NEGOTIATE_GSS_CODE => StartupRequestCode::NegotiateGss,
-            _ => StartupRequestCode::Normal,
-        };
-
-        // the rest of startup packet are params
-        let params_len = len - 8;
-        let mut params_bytes = vec![0u8; params_len];
-        stream.read_exact(params_bytes.as_mut())?;
-
-        // Then null-terminated (String) pairs of param name / param value go.
-        let params_str = str::from_utf8(&params_bytes).unwrap();
-        let params = params_str.split('\0');
-        let mut params_hash: HashMap<String, String> = HashMap::new();
-        for pair in params.collect::<Vec<_>>().chunks_exact(2) {
-            let name = pair[0];
-            let value = pair[1];
-            if name == "options" {
-                // deprecated way of passing params as cmd line args
-                for cmdopt in value.split(' ') {
-                    let nameval: Vec<&str> = cmdopt.split('=').collect();
-                    if nameval.len() == 2 {
-                        params_hash.insert(nameval[0].to_string(), nameval[1].to_string());
-                    }
-                }
-            } else {
-                params_hash.insert(name.to_string(), value.to_string());
-            }
-        }
-
-        Ok(Some(FeMessage::StartupMessage(FeStartupMessage {
-            version,
-            kind,
-            params: params_hash,
-        })))
-    }
-}
-
-impl FeParseMessage {
-    pub fn parse(mut buf: Bytes) -> anyhow::Result<FeMessage> {
-        let _pstmt_name = read_null_terminated(&mut buf)?;
-        let query_string = read_null_terminated(&mut buf)?;
-        let nparams = buf.get_i16();
-
-        // FIXME: the rust-postgres driver uses a named prepared statement
-        // for copy_out(). We're not prepared to handle that correctly. For
-        // now, just ignore the statement name, assuming that the client never
-        // uses more than one prepared statement at a time.
-        /*
-        if !pstmt_name.is_empty() {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "named prepared statements not implemented in Parse",
-            ));
-        }
-         */
-
-        if nparams != 0 {
-            bail!("query params not implemented");
-        }
-
-        Ok(FeMessage::Parse(FeParseMessage { query_string }))
-    }
-}
-
-impl FeDescribeMessage {
-    pub fn parse(mut buf: Bytes) -> anyhow::Result<FeMessage> {
-        let kind = buf.get_u8();
-        let _pstmt_name = read_null_terminated(&mut buf)?;
-
-        // FIXME: see FeParseMessage::parse
-        /*
-        if !pstmt_name.is_empty() {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "named prepared statements not implemented in Describe",
-            ));
-        }
-        */
-
-        if kind != b'S' {
-            bail!("only prepared statmement Describe is implemented");
-        }
-
-        Ok(FeMessage::Describe(FeDescribeMessage { kind }))
-    }
-}
-
-impl FeExecuteMessage {
-    pub fn parse(mut buf: Bytes) -> anyhow::Result<FeMessage> {
-        let portal_name = read_null_terminated(&mut buf)?;
-        let maxrows = buf.get_i32();
-
-        if !portal_name.is_empty() {
-            bail!("named portals not implemented");
-        }
-
-        if maxrows != 0 {
-            bail!("row limit in Execute message not supported");
-        }
-
-        Ok(FeMessage::Execute(FeExecuteMessage { maxrows }))
-    }
-}
-
-impl FeBindMessage {
-    pub fn parse(mut buf: Bytes) -> anyhow::Result<FeMessage> {
-        let portal_name = read_null_terminated(&mut buf)?;
-        let _pstmt_name = read_null_terminated(&mut buf)?;
-
-        if !portal_name.is_empty() {
-            bail!("named portals not implemented");
-        }
-
-        // FIXME: see FeParseMessage::parse
-        /*
-        if !pstmt_name.is_empty() {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "named prepared statements not implemented",
-            ));
-        }
-        */
-
-        Ok(FeMessage::Bind(FeBindMessage {}))
-    }
-}
-
-impl FeCloseMessage {
-    pub fn parse(mut buf: Bytes) -> anyhow::Result<FeMessage> {
-        let _kind = buf.get_u8();
-        let _pstmt_or_portal_name = read_null_terminated(&mut buf)?;
-
-        // FIXME: we do nothing with Close
-
-        Ok(FeMessage::Close(FeCloseMessage {}))
-    }
-}
-
-fn read_null_terminated(buf: &mut Bytes) -> anyhow::Result<Bytes> {
-    let mut result = BytesMut::new();
-
-    loop {
-        if !buf.has_remaining() {
-            bail!("no null-terminator in string");
-        }
-
-        let byte = buf.get_u8();
-
-        if byte == 0 {
-            break;
-        }
-        result.put_u8(byte);
-    }
-    Ok(result.freeze())
-}
-
-// Backend
-
-#[derive(Debug)]
-pub enum BeMessage<'a> {
-    AuthenticationOk,
-    BindComplete,
-    CommandComplete(&'a [u8]),
-    ControlFile,
-    CopyData(&'a [u8]),
-    CopyDone,
-    CopyInResponse,
-    CopyOutResponse,
-    CopyBothResponse,
-    CloseComplete,
-    // None means column is NULL
-    DataRow(&'a [Option<&'a [u8]>]),
-    ErrorResponse(String),
-    // see https://www.postgresql.org/docs/devel/protocol-flow.html#id-1.10.5.7.11
-    Negotiate,
-    NoData,
-    ParameterDescription,
-    ParameterStatus,
-    ParseComplete,
-    ReadyForQuery,
-    RowDescription(&'a [RowDescriptor<'a>]),
-    XLogData(XLogDataBody<'a>),
-}
-
-// One row desciption in RowDescription packet.
-#[derive(Debug)]
-pub struct RowDescriptor<'a> {
-    pub name: &'a [u8],
-    pub tableoid: Oid,
-    pub attnum: i16,
-    pub typoid: Oid,
-    pub typlen: i16,
-    pub typmod: i32,
-    pub formatcode: i16,
-}
-
-impl Default for RowDescriptor<'_> {
-    fn default() -> RowDescriptor<'static> {
-        RowDescriptor {
-            name: b"",
-            tableoid: 0,
-            attnum: 0,
-            typoid: 0,
-            typlen: 0,
-            typmod: 0,
-            formatcode: 0,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct XLogDataBody<'a> {
-    pub wal_start: u64,
-    pub wal_end: u64,
-    pub timestamp: u64,
-    pub data: &'a [u8],
-}
-
-pub static HELLO_WORLD_ROW: BeMessage = BeMessage::DataRow(&[Some(b"hello world")]);
-pub const TEXT_OID: Oid = 25;
-pub const JSON_OID: Oid = 114;
-// single text column
-pub static SINGLE_COL_ROWDESC: BeMessage = BeMessage::RowDescription(&[RowDescriptor {
-    name: b"data",
-    tableoid: 0,
-    attnum: 0,
-    typoid: TEXT_OID,
-    typlen: -1,
-    typmod: 0,
-    formatcode: 0,
-}]);
-
-// Safe usize -> i32|i16 conversion, from rust-postgres
-trait FromUsize: Sized {
-    fn from_usize(x: usize) -> Result<Self, io::Error>;
-}
-
-macro_rules! from_usize {
-    ($t:ty) => {
-        impl FromUsize for $t {
-            #[inline]
-            fn from_usize(x: usize) -> io::Result<$t> {
-                if x > <$t>::max_value() as usize {
-                    Err(io::Error::new(
-                        io::ErrorKind::InvalidInput,
-                        "value too large to transmit",
-                    ))
-                } else {
-                    Ok(x as $t)
-                }
-            }
-        }
-    };
-}
-
-from_usize!(i32);
-
-/// Call f() to write body of the message and prepend it with 4-byte len as
-/// prescribed by the protocol.
-fn write_body<F>(buf: &mut BytesMut, f: F) -> io::Result<()>
-where
-    F: FnOnce(&mut BytesMut) -> io::Result<()>,
-{
-    let base = buf.len();
-    buf.extend_from_slice(&[0; 4]);
-
-    f(buf)?;
-
-    let size = i32::from_usize(buf.len() - base)?;
-    BigEndian::write_i32(&mut buf[base..], size);
-    Ok(())
-}
-
-/// Safe write of s into buf as cstring (String in the protocol).
-fn write_cstr(s: &[u8], buf: &mut BytesMut) -> Result<(), io::Error> {
-    if s.contains(&0) {
-        return Err(io::Error::new(
-            io::ErrorKind::InvalidInput,
-            "string contains embedded null",
-        ));
-    }
-    buf.put_slice(s);
-    buf.put_u8(0);
-    Ok(())
-}
-
-impl<'a> BeMessage<'a> {
-    /// Write message to the given buf.
-    // Unlike the reading side, we use BytesMut
-    // here as msg len preceeds its body and it is handy to write it down first
-    // and then fill the length. With Write we would have to either calc it
-    // manually or have one more buffer.
-    pub fn write(buf: &mut BytesMut, message: &BeMessage) -> io::Result<()> {
-        match message {
-            BeMessage::AuthenticationOk => {
-                buf.put_u8(b'R');
-                write_body(buf, |buf| {
-                    buf.put_i32(0);
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap(); // write into BytesMut can't fail
-            }
-
-            BeMessage::BindComplete => {
-                buf.put_u8(b'2');
-                write_body(buf, |_| Ok::<(), io::Error>(())).unwrap();
-            }
-
-            BeMessage::CloseComplete => {
-                buf.put_u8(b'3');
-                write_body(buf, |_| Ok::<(), io::Error>(())).unwrap();
-            }
-
-            BeMessage::CommandComplete(cmd) => {
-                buf.put_u8(b'C');
-                write_body(buf, |buf| {
-                    write_cstr(cmd, buf)?;
-                    Ok::<_, io::Error>(())
-                })?;
-            }
-
-            BeMessage::ControlFile => {
-                // TODO pass checkpoint and xid info in this message
-                BeMessage::write(buf, &BeMessage::DataRow(&[Some(b"hello pg_control")]))?;
-            }
-
-            BeMessage::CopyData(data) => {
-                buf.put_u8(b'd');
-                write_body(buf, |buf| {
-                    buf.put_slice(data);
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::CopyDone => {
-                buf.put_u8(b'c');
-                write_body(buf, |_| Ok::<(), io::Error>(())).unwrap();
-            }
-
-            BeMessage::CopyInResponse => {
-                buf.put_u8(b'G');
-                write_body(buf, |buf| {
-                    buf.put_u8(1); /* copy_is_binary */
-                    buf.put_i16(0); /* numAttributes */
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::CopyOutResponse => {
-                buf.put_u8(b'H');
-                write_body(buf, |buf| {
-                    buf.put_u8(0); /* copy_is_binary */
-                    buf.put_i16(0); /* numAttributes */
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::CopyBothResponse => {
-                buf.put_u8(b'W');
-                write_body(buf, |buf| {
-                    // doesn't matter, used only for replication
-                    buf.put_u8(0); /* copy_is_binary */
-                    buf.put_i16(0); /* numAttributes */
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::DataRow(vals) => {
-                buf.put_u8(b'D');
-                write_body(buf, |buf| {
-                    buf.put_u16(vals.len() as u16); // num of cols
-                    for val_opt in vals.iter() {
-                        if let Some(val) = val_opt {
-                            buf.put_u32(val.len() as u32);
-                            buf.put_slice(val);
-                        } else {
-                            buf.put_i32(-1);
-                        }
-                    }
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            // ErrorResponse is a zero-terminated array of zero-terminated fields.
-            // First byte of each field represents type of this field. Set just enough fields
-            // to satisfy rust-postgres client: 'S' -- severity, 'C' -- error, 'M' -- error
-            // message text.
-            BeMessage::ErrorResponse(error_msg) => {
-                // For all the errors set Severity to Error and error code to
-                // 'internal error'.
-
-                // 'E' signalizes ErrorResponse messages
-                buf.put_u8(b'E');
-                write_body(buf, |buf| {
-                    buf.put_u8(b'S'); // severity
-                    write_cstr(&Bytes::from("ERROR"), buf)?;
-
-                    buf.put_u8(b'C'); // SQLSTATE error code
-                    write_cstr(&Bytes::from("CXX000"), buf)?;
-
-                    buf.put_u8(b'M'); // the message
-                    write_cstr(error_msg.as_bytes(), buf)?;
-
-                    buf.put_u8(0); // terminator
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::NoData => {
-                buf.put_u8(b'n');
-                write_body(buf, |_| Ok::<(), io::Error>(())).unwrap();
-            }
-
-            BeMessage::Negotiate => {
-                buf.put_u8(b'N');
-            }
-
-            BeMessage::ParameterStatus => {
-                buf.put_u8(b'S');
-                // parameter names and values are specified by null terminated strings
-                const PARAM_NAME_VALUE: &[u8] = b"client_encoding\0UTF8\0";
-                write_body(buf, |buf| {
-                    buf.put_slice(PARAM_NAME_VALUE);
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::ParameterDescription => {
-                buf.put_u8(b't');
-                write_body(buf, |buf| {
-                    // we don't support params, so always 0
-                    buf.put_i16(0);
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::ParseComplete => {
-                buf.put_u8(b'1');
-                write_body(buf, |_| Ok::<(), io::Error>(())).unwrap();
-            }
-
-            BeMessage::ReadyForQuery => {
-                buf.put_u8(b'Z');
-                write_body(buf, |buf| {
-                    buf.put_u8(b'I');
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-
-            BeMessage::RowDescription(rows) => {
-                buf.put_u8(b'T');
-                write_body(buf, |buf| {
-                    buf.put_i16(rows.len() as i16); // # of fields
-                    for row in rows.iter() {
-                        write_cstr(row.name, buf)?;
-                        buf.put_i32(0); /* table oid */
-                        buf.put_i16(0); /* attnum */
-                        buf.put_u32(row.typoid);
-                        buf.put_i16(row.typlen);
-                        buf.put_i32(-1); /* typmod */
-                        buf.put_i16(0); /* format code */
-                    }
-                    Ok::<_, io::Error>(())
-                })?;
-            }
-
-            BeMessage::XLogData(body) => {
-                buf.put_u8(b'd');
-                write_body(buf, |buf| {
-                    buf.put_u8(b'w');
-                    buf.put_u64(body.wal_start);
-                    buf.put_u64(body.wal_end);
-                    buf.put_u64(body.timestamp);
-                    buf.put_slice(body.data);
-                    Ok::<_, io::Error>(())
-                })
-                .unwrap();
-            }
-        }
-        Ok(())
-    }
-}