diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000..76a2ff549e --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,13 @@ +# The binaries are really slow, if you compile them in 'dev' mode with the defaults. +# Enable some optimizations even in 'dev' mode, to make tests faster. The basic +# optimizations enabled by "opt-level=1" don't affect debuggability too much. +# +# See https://www.reddit.com/r/rust/comments/gvrgca/this_is_a_neat_trick_for_getting_good_runtime/ +# +[profile.dev.package."*"] +# Set the default for dependencies in Development mode. +opt-level = 3 + +[profile.dev] +# Turn on a small amount of optimization in Development mode. +opt-level = 1 diff --git a/.circleci/config.yml b/.circleci/config.yml index 5370e46663..00a51eb906 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,10 +5,10 @@ executors: resource_class: xlarge docker: # NB: when changed, do not forget to update rust image tag in all Dockerfiles - - image: zimg/rust:1.58 + - image: neondatabase/rust:1.58 neon-executor: docker: - - image: zimg/rust:1.58 + - image: neondatabase/rust:1.58 jobs: # A job to build postgres @@ -37,7 +37,7 @@ jobs: name: Restore postgres cache keys: # Restore ONLY if the rev key matches exactly - - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }} + - v05-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }} # Build postgres if the restore_cache didn't find a build. # `make` can't figure out whether the cache is valid, since @@ -54,7 +54,7 @@ jobs: - save_cache: name: Save postgres cache - key: v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }} + key: v05-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }} paths: - tmp_install @@ -85,7 +85,7 @@ jobs: name: Restore postgres cache keys: # Restore ONLY if the rev key matches exactly - - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }} + - v05-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }} - restore_cache: name: Restore rust cache @@ -93,7 +93,7 @@ jobs: # Require an exact match. While an out of date cache might speed up the build, # there's no way to clean out old packages, so the cache grows every time something # changes. - - v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} + - v05-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} # Build the rust code, including test binaries - run: @@ -107,7 +107,7 @@ jobs: export CARGO_INCREMENTAL=0 export CACHEPOT_BUCKET=zenith-rust-cachepot - export RUSTC_WRAPPER=cachepot + export RUSTC_WRAPPER="" export AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" mold -run cargo build $CARGO_FLAGS --features failpoints --bins --tests @@ -115,7 +115,7 @@ jobs: - save_cache: name: Save rust cache - key: v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} + key: v05-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} paths: - ~/.cargo/registry - ~/.cargo/git @@ -142,11 +142,6 @@ jobs: jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name' ) - test_exe_paths=$( - cargo test --message-format=json --no-run | - jq -r '.executable | select(. != null)' - ) - mkdir -p /tmp/zenith/bin mkdir -p /tmp/zenith/test_bin mkdir -p /tmp/zenith/etc @@ -330,274 +325,6 @@ jobs: paths: - "*" - # Build neondatabase/neon:latest image and push it to Docker hub - docker-image: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG=$(git log --oneline|wc -l) - docker build \ - --pull \ - --build-arg GIT_VERSION=${CIRCLE_SHA1} \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:latest . - docker push neondatabase/neon:${DOCKER_TAG} - docker push neondatabase/neon:latest - - # Build neondatabase/compute-node:latest image and push it to Docker hub - docker-image-compute: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Build and push compute-tools Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - docker build \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/compute-tools:local \ - --tag neondatabase/compute-tools:latest \ - -f Dockerfile.compute-tools . - # Only push :latest image - docker push neondatabase/compute-tools:latest - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push compute-node Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG=$(git log --oneline|wc -l) - docker build --tag neondatabase/compute-node:${DOCKER_TAG} \ - --tag neondatabase/compute-node:latest vendor/postgres \ - --build-arg COMPUTE_TOOLS_TAG=local - docker push neondatabase/compute-node:${DOCKER_TAG} - docker push neondatabase/compute-node:latest - - # Build production neondatabase/neon:release image and push it to Docker hub - docker-image-release: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG="release-$(git log --oneline|wc -l)" - docker build \ - --pull \ - --build-arg GIT_VERSION=${CIRCLE_SHA1} \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:release . - docker push neondatabase/neon:${DOCKER_TAG} - docker push neondatabase/neon:release - - # Build production neondatabase/compute-node:release image and push it to Docker hub - docker-image-compute-release: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Build and push compute-tools Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - docker build \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/compute-tools:release \ - --tag neondatabase/compute-tools:local \ - -f Dockerfile.compute-tools . - # Only push :release image - docker push neondatabase/compute-tools:release - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push compute-node Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG="release-$(git log --oneline|wc -l)" - docker build --tag neondatabase/compute-node:${DOCKER_TAG} \ - --tag neondatabase/compute-node:release vendor/postgres \ - --build-arg COMPUTE_TOOLS_TAG=local - docker push neondatabase/compute-node:${DOCKER_TAG} - docker push neondatabase/compute-node:release - - deploy-staging: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - setup_remote_docker - - run: - name: Setup ansible - command: | - pip install --progress-bar off --user ansible boto3 - - run: - name: Redeploy - command: | - cd "$(pwd)/.circleci/ansible" - - ./get_binaries.sh - - echo "${TELEPORT_SSH_KEY}" | tr -d '\n'| base64 --decode >ssh-key - echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub - chmod 0600 ssh-key - ssh-add ssh-key - rm -f ssh-key ssh-key-cert.pub - - ansible-playbook deploy.yaml -i staging.hosts - rm -f neon_install.tar.gz .neon_current_version - - deploy-staging-proxy: - docker: - - image: cimg/base:2021.04 - environment: - KUBECONFIG: .kubeconfig - steps: - - checkout - - run: - name: Store kubeconfig file - command: | - echo "${STAGING_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - run: - name: Setup helm v3 - command: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts - - run: - name: Re-deploy proxy - command: | - DOCKER_TAG=$(git log --oneline|wc -l) - helm upgrade neon-proxy neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s - helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s - - deploy-neon-stress: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - setup_remote_docker - - run: - name: Setup ansible - command: | - pip install --progress-bar off --user ansible boto3 - - run: - name: Redeploy - command: | - cd "$(pwd)/.circleci/ansible" - - ./get_binaries.sh - - echo "${TELEPORT_SSH_KEY}" | tr -d '\n'| base64 --decode >ssh-key - echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub - chmod 0600 ssh-key - ssh-add ssh-key - rm -f ssh-key ssh-key-cert.pub - - ansible-playbook deploy.yaml -i neon-stress.hosts - rm -f neon_install.tar.gz .neon_current_version - - deploy-neon-stress-proxy: - docker: - - image: cimg/base:2021.04 - environment: - KUBECONFIG: .kubeconfig - steps: - - checkout - - run: - name: Store kubeconfig file - command: | - echo "${NEON_STRESS_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - run: - name: Setup helm v3 - command: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts - - run: - name: Re-deploy proxy - command: | - DOCKER_TAG=$(git log --oneline|wc -l) - helm upgrade neon-stress-proxy neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy.yaml --set image.tag=${DOCKER_TAG} --wait - helm upgrade neon-stress-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait - - deploy-release: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - setup_remote_docker - - run: - name: Setup ansible - command: | - pip install --progress-bar off --user ansible boto3 - - run: - name: Redeploy - command: | - cd "$(pwd)/.circleci/ansible" - - RELEASE=true ./get_binaries.sh - - echo "${TELEPORT_SSH_KEY}" | tr -d '\n'| base64 --decode >ssh-key - echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub - chmod 0600 ssh-key - ssh-add ssh-key - rm -f ssh-key ssh-key-cert.pub - - ansible-playbook deploy.yaml -i production.hosts - rm -f neon_install.tar.gz .neon_current_version - - deploy-release-proxy: - docker: - - image: cimg/base:2021.04 - environment: - KUBECONFIG: .kubeconfig - steps: - - checkout - - run: - name: Store kubeconfig file - command: | - echo "${PRODUCTION_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - run: - name: Setup helm v3 - command: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts - - run: - name: Re-deploy proxy - command: | - DOCKER_TAG="release-$(git log --oneline|wc -l)" - helm upgrade neon-proxy neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy.yaml --set image.tag=${DOCKER_TAG} --wait - helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait - workflows: build_and_test: jobs: @@ -640,103 +367,3 @@ workflows: save_perf_report: true requires: - build-neon-release - - docker-image: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - main - requires: - - pg_regress-tests-release - - other-tests-release - - docker-image-compute: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - main - requires: - - pg_regress-tests-release - - other-tests-release - - deploy-staging: - # Context gives an ability to login - context: Docker Hub - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - deploy-staging-proxy: - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - - deploy-neon-stress: - # Context gives an ability to login - context: Docker Hub - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - deploy-neon-stress-proxy: - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - - docker-image-release: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - release - requires: - - pg_regress-tests-release - - other-tests-release - - docker-image-compute-release: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - release - requires: - - pg_regress-tests-release - - other-tests-release - - deploy-release: - # Context gives an ability to login - context: Docker Hub - # deploy only for commits to main - filters: - branches: - only: - - release - requires: - - docker-image-release - - deploy-release-proxy: - # deploy only for commits to main - filters: - branches: - only: - - release - requires: - - docker-image-release diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index accb8896de..f220be2b12 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -37,6 +37,12 @@ runs: name: neon-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.rust_toolchain }}-artifact path: ./neon-artifact/ + - name: Get Postgres artifact for restoration + uses: actions/download-artifact@v3 + with: + name: postgres-${{ runner.os }}-${{ inputs.build_type }}-artifact + path: ./pg-artifact/ + - name: Extract Neon artifact shell: bash -ex {0} run: | @@ -44,6 +50,13 @@ runs: tar -xf ./neon-artifact/neon.tgz -C /tmp/neon/ rm -rf ./neon-artifact/ + - name: Extract Postgres artifact + shell: bash -ex {0} + run: | + mkdir -p /tmp/neon/tmp_install + tar -xf ./pg-artifact/pg.tgz -C /tmp/neon/tmp_install + rm -rf ./pg-artifact/ + - name: Checkout if: inputs.needs_postgres_source == 'true' uses: actions/checkout@v3 @@ -65,7 +78,7 @@ runs: - name: Run pytest env: NEON_BIN: /tmp/neon/bin - POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + POSTGRES_DISTRIB_DIR: /tmp/neon/tmp_install TEST_OUTPUT: /tmp/test_output # this variable will be embedded in perf test report # and is needed to distinguish different environments diff --git a/.circleci/ansible/.gitignore b/.github/ansible/.gitignore similarity index 100% rename from .circleci/ansible/.gitignore rename to .github/ansible/.gitignore diff --git a/.circleci/ansible/ansible.cfg b/.github/ansible/ansible.cfg similarity index 100% rename from .circleci/ansible/ansible.cfg rename to .github/ansible/ansible.cfg diff --git a/.circleci/ansible/ansible.ssh.cfg b/.github/ansible/ansible.ssh.cfg similarity index 100% rename from .circleci/ansible/ansible.ssh.cfg rename to .github/ansible/ansible.ssh.cfg diff --git a/.circleci/ansible/deploy.yaml b/.github/ansible/deploy.yaml similarity index 100% rename from .circleci/ansible/deploy.yaml rename to .github/ansible/deploy.yaml diff --git a/.circleci/ansible/get_binaries.sh b/.github/ansible/get_binaries.sh similarity index 100% rename from .circleci/ansible/get_binaries.sh rename to .github/ansible/get_binaries.sh diff --git a/.circleci/ansible/neon-stress.hosts b/.github/ansible/neon-stress.hosts similarity index 100% rename from .circleci/ansible/neon-stress.hosts rename to .github/ansible/neon-stress.hosts diff --git a/.circleci/ansible/production.hosts b/.github/ansible/production.hosts similarity index 93% rename from .circleci/ansible/production.hosts rename to .github/ansible/production.hosts index 6a3a7791ad..d22ce0e37e 100644 --- a/.circleci/ansible/production.hosts +++ b/.github/ansible/production.hosts @@ -1,6 +1,7 @@ [pageservers] #zenith-1-ps-1 console_region_id=1 zenith-1-ps-2 console_region_id=1 +zenith-1-ps-3 console_region_id=1 [safekeepers] zenith-1-sk-1 console_region_id=1 diff --git a/.circleci/ansible/scripts/init_pageserver.sh b/.github/ansible/scripts/init_pageserver.sh similarity index 100% rename from .circleci/ansible/scripts/init_pageserver.sh rename to .github/ansible/scripts/init_pageserver.sh diff --git a/.circleci/ansible/scripts/init_safekeeper.sh b/.github/ansible/scripts/init_safekeeper.sh similarity index 100% rename from .circleci/ansible/scripts/init_safekeeper.sh rename to .github/ansible/scripts/init_safekeeper.sh diff --git a/.circleci/ansible/staging.hosts b/.github/ansible/staging.hosts similarity index 100% rename from .circleci/ansible/staging.hosts rename to .github/ansible/staging.hosts diff --git a/.circleci/ansible/systemd/pageserver.service b/.github/ansible/systemd/pageserver.service similarity index 100% rename from .circleci/ansible/systemd/pageserver.service rename to .github/ansible/systemd/pageserver.service diff --git a/.circleci/ansible/systemd/safekeeper.service b/.github/ansible/systemd/safekeeper.service similarity index 100% rename from .circleci/ansible/systemd/safekeeper.service rename to .github/ansible/systemd/safekeeper.service diff --git a/.circleci/helm-values/neon-stress.proxy-scram.yaml b/.github/helm-values/neon-stress.proxy-scram.yaml similarity index 100% rename from .circleci/helm-values/neon-stress.proxy-scram.yaml rename to .github/helm-values/neon-stress.proxy-scram.yaml diff --git a/.circleci/helm-values/neon-stress.proxy.yaml b/.github/helm-values/neon-stress.proxy.yaml similarity index 100% rename from .circleci/helm-values/neon-stress.proxy.yaml rename to .github/helm-values/neon-stress.proxy.yaml diff --git a/.circleci/helm-values/production.proxy-scram.yaml b/.github/helm-values/production.proxy-scram.yaml similarity index 100% rename from .circleci/helm-values/production.proxy-scram.yaml rename to .github/helm-values/production.proxy-scram.yaml diff --git a/.circleci/helm-values/production.proxy.yaml b/.github/helm-values/production.proxy.yaml similarity index 100% rename from .circleci/helm-values/production.proxy.yaml rename to .github/helm-values/production.proxy.yaml diff --git a/.circleci/helm-values/staging.proxy-scram.yaml b/.github/helm-values/staging.proxy-scram.yaml similarity index 100% rename from .circleci/helm-values/staging.proxy-scram.yaml rename to .github/helm-values/staging.proxy-scram.yaml diff --git a/.circleci/helm-values/staging.proxy.yaml b/.github/helm-values/staging.proxy.yaml similarity index 100% rename from .circleci/helm-values/staging.proxy.yaml rename to .github/helm-values/staging.proxy.yaml diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 01dd9d00b0..d08c3c50bd 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -104,3 +104,12 @@ jobs: PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" run: | REPORT_FROM=$(realpath perf-report-staging) REPORT_TO=staging scripts/generate_and_push_perf_report.sh + + - name: Post to a Slack channel + if: ${{ github.event.schedule && failure() }} + uses: slackapi/slack-github-action@v1 + with: + channel-id: "C033QLM5P7D" # dev-staging-stream + slack-message: "Periodic perf testing: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 78aa163f3e..776c696f59 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1,9 +1,10 @@ -name: Test +name: Test and Deploy on: push: branches: - main + - release pull_request: defaults: @@ -11,8 +12,9 @@ defaults: shell: bash -ex {0} concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Allow only one workflow per any non-`main` branch. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }} + cancel-in-progress: true env: RUST_BACKTRACE: 1 @@ -93,12 +95,17 @@ jobs: tar -xf ./postgres-artifact/pg.tgz -C ./tmp_install/ rm -rf ./postgres-artifact/ + # Don't include the ~/.cargo/registry/src directory. It contains just + # uncompressed versions of the crates in ~/.cargo/registry/cache + # directory, and it's faster to let 'cargo' to rebuild it from the + # compressed crates. - name: Cache cargo deps id: cache_cargo uses: actions/cache@v3 with: path: | ~/.cargo/registry/ + !~/.cargo/registry/src ~/.cargo/git/ target/ # Fall back to older versions of the key, if no cache for current Cargo.lock was found @@ -170,14 +177,14 @@ jobs: for bin in $test_exe_paths; do SRC=$bin DST=/tmp/neon/test_bin/$(basename $bin) - cp "$SRC" "$DST" + + # We don't need debug symbols for code coverage, so strip them out to make + # the artifact smaller. + strip "$SRC" -o "$DST" echo "$DST" >> /tmp/coverage/binaries.list done fi - - name: Install postgres binaries - run: cp -a tmp_install /tmp/neon/pg_install - - name: Prepare neon artifact run: tar -C /tmp/neon/ -czf ./neon.tgz . @@ -298,6 +305,7 @@ jobs: with: path: | ~/.cargo/registry/ + !~/.cargo/registry/src ~/.cargo/git/ target/ key: v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }} @@ -390,3 +398,253 @@ jobs: \"remote_repo\": \"${{ github.repository }}\" } }" + + docker-image: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ pg_regress-tests, other-tests ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + outputs: + build-tag: ${{steps.build-tag.outputs.tag}} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver: docker + + - name: Get build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::$(git rev-list --count HEAD)" + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release-$(git rev-list --count HEAD)" + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: build-tag + + - name: Get legacy build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::latest + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: legacy-build-tag + + - name: Build neon Docker image + uses: docker/build-push-action@v2 + with: + context: . + build-args: | + GIT_VERSION="${{github.sha}}" + AWS_ACCESS_KEY_ID="${{secrets.CACHEPOT_AWS_ACCESS_KEY_ID}}" + AWS_SECRET_ACCESS_KEY="${{secrets.CACHEPOT_AWS_SECRET_ACCESS_KEY}}" + pull: true + push: true + tags: neondatabase/neon:${{steps.legacy-build-tag.outputs.tag}}, neondatabase/neon:${{steps.build-tag.outputs.tag}} + + docker-image-compute: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ pg_regress-tests, other-tests ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + outputs: + build-tag: ${{steps.build-tag.outputs.tag}} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver: docker + + - name: Get build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::$(git rev-list --count HEAD)" + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release-$(git rev-list --count HEAD)" + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: build-tag + + - name: Get legacy build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::latest + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: legacy-build-tag + + - name: Build compute-tools Docker image + uses: docker/build-push-action@v2 + with: + context: . + build-args: | + GIT_VERSION="${{github.sha}}" + AWS_ACCESS_KEY_ID="${{secrets.CACHEPOT_AWS_ACCESS_KEY_ID}}" + AWS_SECRET_ACCESS_KEY="${{secrets.CACHEPOT_AWS_SECRET_ACCESS_KEY}}" + push: false + file: Dockerfile.compute-tools + tags: neondatabase/compute-tools:local + + - name: Push compute-tools Docker image + uses: docker/build-push-action@v2 + with: + context: . + build-args: | + GIT_VERSION="${{github.sha}}" + AWS_ACCESS_KEY_ID="${{secrets.CACHEPOT_AWS_ACCESS_KEY_ID}}" + AWS_SECRET_ACCESS_KEY="${{secrets.CACHEPOT_AWS_SECRET_ACCESS_KEY}}" + push: true + file: Dockerfile.compute-tools + tags: neondatabase/compute-tools:${{steps.legacy-build-tag.outputs.tag}} + + - name: Build compute-node Docker image + uses: docker/build-push-action@v2 + with: + context: ./vendor/postgres/ + build-args: + COMPUTE_TOOLS_TAG=local + push: true + tags: neondatabase/compute-node:${{steps.legacy-build-tag.outputs.tag}}, neondatabase/compute-node:${{steps.build-tag.outputs.tag}} + + calculate-deploy-targets: + runs-on: [ self-hosted, Linux, k8s-runner ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + outputs: + matrix-include: ${{ steps.set-matrix.outputs.include }} + steps: + - id: set-matrix + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA"}' + NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA"}' + echo "::set-output name=include::[$STAGING, $NEON_STRESS]" + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA"}' + echo "::set-output name=include::[$PRODUCTION]" + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + + deploy: + runs-on: [ self-hosted, Linux, k8s-runner ] + # We need both storage **and** compute images for deploy, because control plane + # picks the compute version based on the storage version. If it notices a fresh + # storage it may bump the compute version. And if compute image failed to build + # it may break things badly. + needs: [ docker-image, docker-image-compute, calculate-deploy-targets ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + strategy: + matrix: + include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Setup ansible + run: | + pip install --progress-bar off --user ansible boto3 + + - name: Redeploy + run: | + cd "$(pwd)/.github/ansible" + + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + ./get_binaries.sh + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + RELEASE=true ./get_binaries.sh + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + + eval $(ssh-agent) + echo "${{ secrets.TELEPORT_SSH_KEY }}" | tr -d '\n'| base64 --decode >ssh-key + echo "${{ secrets.TELEPORT_SSH_CERT }}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub + chmod 0600 ssh-key + ssh-add ssh-key + rm -f ssh-key ssh-key-cert.pub + + ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts + rm -f neon_install.tar.gz .neon_current_version + + deploy-proxy: + runs-on: [ self-hosted, Linux, k8s-runner ] + # Compute image isn't strictly required for proxy deploy, but let's still wait for it + # to run all deploy jobs consistently. + needs: [ docker-image, docker-image-compute, calculate-deploy-targets ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + strategy: + matrix: + include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}} + env: + KUBECONFIG: .kubeconfig + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Store kubeconfig file + run: | + echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG} + chmod 0600 ${KUBECONFIG} + + - name: Setup helm v3 + run: | + curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + helm repo add neondatabase https://neondatabase.github.io/helm-charts + + - name: Re-deploy proxy + run: | + DOCKER_TAG=${{needs.docker-image.outputs.build-tag}} + helm upgrade ${{ matrix.proxy_job }} neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s + helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s diff --git a/.github/workflows/codestyle.yml b/.github/workflows/codestyle.yml index 2b8a01e94e..89bfffd4b9 100644 --- a/.github/workflows/codestyle.yml +++ b/.github/workflows/codestyle.yml @@ -11,8 +11,9 @@ defaults: shell: bash -ex {0} concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Allow only one workflow per any non-`main` branch. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }} + cancel-in-progress: true env: RUST_BACKTRACE: 1 @@ -97,6 +98,7 @@ jobs: with: path: | ~/.cargo/registry + !~/.cargo/registry/src ~/.cargo/git target key: ${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust-${{ matrix.rust_toolchain }} diff --git a/.github/workflows/pg_clients.yml b/.github/workflows/pg_clients.yml index fe4dbea8ac..4ff31ac508 100644 --- a/.github/workflows/pg_clients.yml +++ b/.github/workflows/pg_clients.yml @@ -13,8 +13,9 @@ on: workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Allow only one workflow per any non-`main` branch. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }} + cancel-in-progress: true jobs: test-postgres-client-libs: diff --git a/Cargo.lock b/Cargo.lock index 4f1d727ae1..4f453678e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,9 +48,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.53" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0" +checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" dependencies = [ "backtrace", ] @@ -77,7 +77,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.9", + "time 0.3.11", ] [[package]] @@ -126,9 +126,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ "proc-macro2", "quote", @@ -154,9 +154,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.4" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a" +checksum = "d16705af05732b7d3258ec0f7b73c03a658a28925e050d8852d5b568ee8bcf4e" dependencies = [ "async-trait", "axum-core", @@ -166,7 +166,7 @@ dependencies = [ "http", "http-body", "hyper", - "itoa 1.0.1", + "itoa 1.0.2", "matchit", "memchr", "mime", @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.3" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bdc19781b16e32f8a7200368a336fa4509d4b72ef15dd4e41df5290855ee1e6" +checksum = "e4f44a0e6200e9d11a1cdc989e4b358f6e3d354fbf48478f345a17f4e43f8635" dependencies = [ "async-trait", "bytes", @@ -197,15 +197,15 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" dependencies = [ "addr2line", "cc", "cfg-if", "libc", - "miniz_oxide 0.4.4", + "miniz_oxide", "object", "rustc-demangle", ] @@ -292,15 +292,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.9.1" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" [[package]] name = "bytemuck" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc" +checksum = "c53dfa917ec274df8ed3c572698f381a24eef2efba9492d797301b72b6db408a" [[package]] name = "byteorder" @@ -327,10 +327,16 @@ dependencies = [ ] [[package]] -name = "cc" -version = "1.0.72" +name = "cast" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" [[package]] name = "cexpr" @@ -363,9 +369,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" +checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b" dependencies = [ "glob", "libc", @@ -389,17 +395,26 @@ dependencies = [ [[package]] name = "clap" -version = "3.0.14" +version = "3.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63edc3f163b3c71ec8aa23f9bd6070f77edbf3d1d198b164afa90ff00e4ec62" +checksum = "ab8b79fe3946ceb4a0b1c080b4018992b8d27e9ff363644c1c9b6387c854614d" dependencies = [ "atty", "bitflags", + "clap_lex", "indexmap", - "os_str_bytes", "strsim 0.10.0", "termcolor", - "textwrap 0.14.2", + "textwrap 0.15.0", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", ] [[package]] @@ -423,9 +438,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.3" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b727aacc797f9fc28e355d21f34709ac4fc9adecfe470ad07b8f4464f53062" +checksum = "2a604e93b79d1808327a6fca85a6f2d69de66461e7620f5a4cbf5fb4d1d7c948" dependencies = [ "bytes", "memchr", @@ -449,7 +464,7 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap 3.0.14", + "clap 3.2.12", "env_logger", "hyper", "libc", @@ -467,9 +482,9 @@ dependencies = [ [[package]] name = "const_format" -version = "0.2.22" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22bc6cd49b0ec407b680c3e380182b6ac63b73991cb7602de350352fc309b614" +checksum = "939dc9e2eb9077e0679d2ce32de1ded8531779360b003b4a972a7a39ec263495" dependencies = [ "const_format_proc_macros", ] @@ -534,18 +549,18 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] [[package]] name = "crc32c" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6b9c9389584bcba988bd0836086789b7f87ad91892d6a83d5291dbb24524b5" +checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e" dependencies = [ "rustc_version", ] @@ -561,12 +576,12 @@ dependencies = [ [[package]] name = "criterion" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" dependencies = [ "atty", - "cast", + "cast 0.3.0", "clap 2.34.0", "criterion-plot", "csv", @@ -591,15 +606,15 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" dependencies = [ - "cast", + "cast 0.2.7", "itertools", ] [[package]] name = "crossbeam-channel" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -618,26 +633,26 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "lazy_static", "memoffset", + "once_cell", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" dependencies = [ "cfg-if", - "lazy_static", + "once_cell", ] [[package]] @@ -650,7 +665,7 @@ dependencies = [ "crossterm_winapi", "libc", "mio", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", "winapi", @@ -667,9 +682,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +checksum = "2ccfd8c0ee4cce11e45b3fd6f9d5e69e0cc62912aa6a0cb1bf4617b0eba5a12f" dependencies = [ "generic-array", "typenum", @@ -719,9 +734,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ "darling_core", "darling_macro", @@ -729,9 +744,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", @@ -743,9 +758,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", "quote", @@ -835,15 +850,15 @@ dependencies = [ [[package]] name = "either" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" [[package]] name = "encoding_rs" -version = "0.8.30" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ "cfg-if", ] @@ -863,9 +878,9 @@ dependencies = [ [[package]] name = "etcd-client" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c434d2800b273a506b82397aad2f20971636f65e47b27c027f77d498530c5954" +checksum = "9fb8664f6ea68aba5503d42dd1be786b0f1bd9b7972e7f40208c83ef74db91bf" dependencies = [ "http", "prost", @@ -922,14 +937,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" dependencies = [ "cfg-if", "libc", "redox_syscall", - "winapi", + "windows-sys", ] [[package]] @@ -946,21 +961,9 @@ dependencies = [ [[package]] name = "fixedbitset" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" - -[[package]] -name = "flate2" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" -dependencies = [ - "cfg-if", - "crc32fast", - "libc", - "miniz_oxide 0.5.1", -] +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "fnv" @@ -1104,13 +1107,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.4" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -1149,9 +1152,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.11" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -1162,7 +1165,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.6.9", + "tokio-util", "tracing", ] @@ -1181,6 +1184,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607c8a29735385251a339424dd462993c0fed8fa09d378f259377df08c126022" + [[package]] name = "heck" version = "0.3.3" @@ -1241,20 +1250,20 @@ dependencies = [ [[package]] name = "http" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa 1.0.1", + "itoa 1.0.2", ] [[package]] name = "http-body" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", @@ -1269,9 +1278,9 @@ checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" [[package]] name = "httparse" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -1297,9 +1306,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.17" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ "bytes", "futures-channel", @@ -1310,7 +1319,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.1", + "itoa 1.0.2", "pin-project-lite", "socket2", "tokio", @@ -1376,12 +1385,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.2", ] [[package]] @@ -1393,7 +1402,7 @@ dependencies = [ "ahash", "atty", "indexmap", - "itoa 1.0.1", + "itoa 1.0.2", "lazy_static", "log", "num-format", @@ -1413,9 +1422,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.3.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itertools" @@ -1434,24 +1443,24 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" dependencies = [ "wasm-bindgen", ] [[package]] name = "jsonwebtoken" -version = "8.1.0" +version = "8.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9051c17f81bae79440afa041b3a278e1de71bfb96d32454b477fd4703ccb6f" +checksum = "1aa4b4af834c6cfd35d8763d359661b90f2e45d8f750a0849156c7f4671af09c" dependencies = [ "base64", "pem", @@ -1484,9 +1493,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.117" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libloading" @@ -1500,18 +1509,19 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", "serde", @@ -1566,15 +1576,15 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057a3db23999c867821a7a59feb06a578fcb03685e983dff90daf9e7d24ac08f" +checksum = "3a79b39c93a7a5a27eeaf9a23b5ff43f1b9e0ad6b1cdd441140ae53c35613fc7" dependencies = [ "libc", ] @@ -1613,44 +1623,23 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg", -] - -[[package]] -name = "miniz_oxide" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -1661,9 +1650,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -1682,7 +1671,7 @@ name = "neon_local" version = "0.1.0" dependencies = [ "anyhow", - "clap 3.0.14", + "clap 3.2.12", "comfy-table", "control_plane", "git-version", @@ -1716,22 +1705,12 @@ checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" [[package]] name = "nom" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", - "version_check", -] - -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", ] [[package]] @@ -1757,9 +1736,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -1767,9 +1746,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -1786,18 +1765,18 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.27.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" dependencies = [ "memchr", ] @@ -1813,9 +1792,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "oorandom" @@ -1831,18 +1810,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1851,9 +1842,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" dependencies = [ "autocfg", "cc", @@ -1864,12 +1855,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.0.0" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" -dependencies = [ - "memchr", -] +checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" [[package]] name = "pageserver" @@ -1879,7 +1867,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "clap 3.0.14", + "clap 3.2.12", "close_fds", "const_format", "crc32c", @@ -1939,12 +1927,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.2", + "parking_lot_core 0.9.3", ] [[package]] @@ -1963,9 +1951,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", @@ -1982,9 +1970,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pem" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a3b09a20e374558580a4914d3b7d89bd61b954a5a5e1dcbea98753addb1947" +checksum = "03c64931a1a212348ec4f3b4362585eca7159d0d09cbdf4a7f74f02173596fd4" dependencies = [ "base64", ] @@ -1997,9 +1985,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "petgraph" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" dependencies = [ "fixedbitset", "indexmap", @@ -2025,18 +2013,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" dependencies = [ "proc-macro2", "quote", @@ -2045,9 +2033,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -2057,15 +2045,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plotters" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +checksum = "9428003b84df1496fb9d6eeee9c5f8145cb41ca375eb0dad204328888832811f" dependencies = [ "num-traits", "plotters-backend", @@ -2076,15 +2064,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" [[package]] name = "plotters-svg" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +checksum = "e0918736323d1baff32ee0eade54984f6f201ad7e97d5cfb5d6ab4a358529615" dependencies = [ "plotters-backend", ] @@ -2182,9 +2170,9 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "prettyplease" -version = "0.1.10" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" +checksum = "da6ffbe862780245013cb1c0a48c4e44b7d665548088f91f6b90876d0625e4c2" dependencies = [ "proc-macro2", "syn", @@ -2198,22 +2186,21 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] name = "procfs" -version = "0.10.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e344cafeaeefe487300c361654bcfc85db3ac53619eeccced29f5ea18c4c70" +checksum = "0941606b9934e2d98a3677759a971756eb821f75764d0e0d26946d08e74d9104" dependencies = [ "bitflags", "byteorder", - "flate2", "hex", "lazy_static", "libc", @@ -2221,25 +2208,25 @@ dependencies = [ [[package]] name = "prometheus" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" +checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39" dependencies = [ "cfg-if", "fnv", "lazy_static", "libc", "memchr", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "procfs", "thiserror", ] [[package]] name = "prost" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc03e116981ff7d8da8e5c220e374587b98d294af7ba7dd7fda761158f00086f" +checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" dependencies = [ "bytes", "prost-derive", @@ -2247,9 +2234,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65a1118354442de7feb8a2a76f3d80ef01426bd45542c8c1fdffca41a758f846" +checksum = "8ae5a4388762d5815a9fc0dea33c56b021cdc8dde0c55e0c9ca57197254b0cab" dependencies = [ "bytes", "cfg-if", @@ -2298,17 +2285,17 @@ dependencies = [ "async-trait", "base64", "bytes", - "clap 3.0.14", + "clap 3.2.12", "futures", "git-version", - "hashbrown", + "hashbrown 0.11.2", "hex", "hmac 0.12.1", "hyper", "lazy_static", "md5", "metrics", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "pin-project-lite", "rand", "rcgen", @@ -2316,7 +2303,7 @@ dependencies = [ "routerify", "rstest", "rustls", - "rustls-pemfile", + "rustls-pemfile 0.2.1", "scopeguard", "serde", "serde_json", @@ -2353,9 +2340,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.15" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" dependencies = [ "proc-macro2", ] @@ -2402,9 +2389,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg", "crossbeam-deque", @@ -2414,14 +2401,13 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "lazy_static", "num_cpus", ] @@ -2439,28 +2425,29 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", "redox_syscall", + "thiserror", ] [[package]] name = "regex" -version = "1.5.5" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -2478,9 +2465,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remote_storage" @@ -2496,7 +2483,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", - "tokio-util 0.7.0", + "tokio-util", "toml_edit", "tracing", "workspace_hack", @@ -2513,9 +2500,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" dependencies = [ "base64", "bytes", @@ -2535,12 +2522,13 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile", + "rustls-pemfile 1.0.0", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-rustls", + "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -2551,9 +2539,9 @@ dependencies = [ [[package]] name = "rgb" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74fdc210d8f24a7dbfedc13b04ba5764f5232754ccebfdf5fff1bad791ccbc6" +checksum = "c3b221de559e4a29df3b957eec92bc0de6bc8eaf6ca9cfed43e5e1d67ff65a34" dependencies = [ "bytemuck", ] @@ -2713,9 +2701,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.4" +version = "0.20.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ "log", "ring", @@ -2732,6 +2720,15 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +dependencies = [ + "base64", +] + [[package]] name = "rustls-split" version = "0.3.0" @@ -2743,15 +2740,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" +checksum = "a0a5f7c728f5d284929a1cccb5bc19884422bfe6ef4d6c409da2c41838983fcf" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "safekeeper" @@ -2761,7 +2758,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "clap 3.0.14", + "clap 3.2.12", "const_format", "crc32c", "daemonize", @@ -2787,7 +2784,7 @@ dependencies = [ "tempfile", "tokio", "tokio-postgres", - "tokio-util 0.7.0", + "tokio-util", "toml_edit", "tracing", "url", @@ -2807,12 +2804,12 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "winapi", + "windows-sys", ] [[package]] @@ -2856,15 +2853,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.5" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0486718e92ec9a68fbed73bb5ef687d71103b142595b406835649bebd33f72c7" +checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "0171ebb889e45aa68b44aee0859b3eede84c6f5f5c228e6f140c0b2a0a46cad6" dependencies = [ "serde_derive", ] @@ -2881,9 +2878,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "dc1d3230c1de7932af58ad8ffbe1d784bd55efd5a9d84ac24f69c72d83543dfb" dependencies = [ "proc-macro2", "quote", @@ -2892,11 +2889,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.78" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] @@ -2908,27 +2905,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] [[package]] name = "serde_with" -version = "1.12.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec1e6ec4d8950e5b1e894eac0d360742f3b1407a6078a604a731c4b3f49cefbc" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" dependencies = [ - "rustversion", "serde", "serde_with_macros", ] [[package]] name = "serde_with_macros" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling", "proc-macro2", @@ -2977,9 +2973,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "647c97df271007dcea485bb74ffdb57f2e683f1306c854f468a0c244badabf2d" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" dependencies = [ "libc", "signal-hook-registry", @@ -3007,33 +3003,33 @@ dependencies = [ [[package]] name = "simple_asn1" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a762b1c38b9b990c694b9c2f8abe3372ce6a9ceaae6bca39cfc46e054f45745" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.9", + "time 0.3.11", ] [[package]] name = "siphasher" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a86232ab60fa71287d7f2ddae4a7073f6b7aac33631c3015abb556f08c6d0a3e" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "socket2" @@ -3112,9 +3108,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "symbolic-common" -version = "8.7.0" +version = "8.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6aac7b803adc9ee75344af7681969f76d4b38e4723c6eaacf3b28f5f1d87ff" +checksum = "f551f902d5642e58039aee6a9021a61037926af96e071816361644983966f540" dependencies = [ "debugid", "memmap2", @@ -3124,9 +3120,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "8.7.0" +version = "8.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8143ea5aa546f86c64f9b9aafdd14223ffad4ecd2d58575c63c21335909c99a7" +checksum = "4564ca7b4e6eb14105aa8bbbce26e080f6b5d9c4373e67167ab31f7b86443750" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -3135,13 +3131,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.92" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -3189,9 +3185,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] @@ -3207,24 +3203,24 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -3253,11 +3249,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.2", "libc", "num_threads", "quickcheck", @@ -3282,9 +3278,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -3297,10 +3293,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "57aec3cfa4c296db7255446efb4928a6be304b431a806216105542a67b6ca82e" dependencies = [ + "autocfg", "bytes", "libc", "memchr", @@ -3326,9 +3323,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" dependencies = [ "proc-macro2", "quote", @@ -3356,7 +3353,7 @@ dependencies = [ "fallible-iterator", "futures", "log", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "percent-encoding", "phf", "pin-project-lite", @@ -3364,7 +3361,7 @@ dependencies = [ "postgres-types", "socket2", "tokio", - "tokio-util 0.7.0", + "tokio-util", ] [[package]] @@ -3383,9 +3380,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls", "tokio", @@ -3394,9 +3391,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" dependencies = [ "futures-core", "pin-project-lite", @@ -3405,37 +3402,23 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.9" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", "pin-project-lite", "tokio", + "tracing", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] @@ -3477,7 +3460,7 @@ dependencies = [ "prost-derive", "tokio", "tokio-stream", - "tokio-util 0.7.0", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -3500,9 +3483,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", @@ -3512,7 +3495,7 @@ dependencies = [ "rand", "slab", "tokio", - "tokio-util 0.7.0", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -3520,9 +3503,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" dependencies = [ "bitflags", "bytes", @@ -3545,15 +3528,15 @@ checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" [[package]] name = "tower-service" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.30" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d8d93354fe2a8e50d5953f5ae2e47a3fc2ef03292e7ea46e3cc38f549525fb9" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if", "log", @@ -3564,9 +3547,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.19" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8276d9a4a3a558d7b7ad5303ad50b53d58264641b82914b7ada36bd762e7a716" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" dependencies = [ "proc-macro2", "quote", @@ -3575,9 +3558,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.22" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03cfcb51380632a72d3111cb8d3447a8d908e577d31beeac006f836383d29a23" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", "valuable", @@ -3595,9 +3578,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", @@ -3606,9 +3589,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74786ce43333fcf51efe947aed9718fbe46d5c7328ec3f1029e818083966d9aa" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ "ansi_term", "lazy_static", @@ -3636,15 +3619,21 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" dependencies = [ "tinyvec", ] @@ -3663,9 +3652,9 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "untrusted" @@ -3708,7 +3697,7 @@ dependencies = [ "rand", "routerify", "rustls", - "rustls-pemfile", + "rustls-pemfile 0.2.1", "rustls-split", "serde", "serde_json", @@ -3757,7 +3746,7 @@ name = "wal_craft" version = "0.1.0" dependencies = [ "anyhow", - "clap 3.0.14", + "clap 3.2.12", "env_logger", "log", "once_cell", @@ -3801,9 +3790,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3811,9 +3800,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" dependencies = [ "bumpalo", "lazy_static", @@ -3826,9 +3815,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.29" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f" dependencies = [ "cfg-if", "js-sys", @@ -3838,9 +3827,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3848,9 +3837,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" dependencies = [ "proc-macro2", "quote", @@ -3861,15 +3850,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" dependencies = [ "js-sys", "wasm-bindgen", @@ -3887,18 +3876,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.2" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" dependencies = [ "webpki", ] [[package]] name = "which" -version = "4.2.4" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" dependencies = [ "either", "lazy_static", @@ -3938,9 +3927,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -3951,39 +3940,39 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "winreg" -version = "0.7.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] @@ -4003,7 +3992,7 @@ dependencies = [ "futures-task", "futures-util", "generic-array", - "hashbrown", + "hashbrown 0.11.2", "hex", "hyper", "indexmap", @@ -4011,6 +4000,8 @@ dependencies = [ "libc", "log", "memchr", + "nom", + "num-bigint", "num-integer", "num-traits", "prost", @@ -4020,8 +4011,9 @@ dependencies = [ "scopeguard", "serde", "syn", + "time 0.3.11", "tokio", - "tokio-util 0.7.0", + "tokio-util", "tracing", "tracing-core", ] @@ -4041,14 +4033,14 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.9", + "time 0.3.11", ] [[package]] name = "xattr" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" dependencies = [ "libc", ] @@ -4070,6 +4062,6 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.5.2" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c88870063c39ee00ec285a2f8d6a966e5b6fb2becc4e8dac77ed0d370ed6006" +checksum = "20b578acffd8516a6c3f2a1bdefc1ec37e547bb4e0fb8b6b01a4cafc886b4442" diff --git a/Makefile b/Makefile index 50e2c8ab7f..566f2ecb10 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,8 @@ +ROOT_PROJECT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +# Where to install Postgres, default is ./tmp_install, maybe useful for package managers +POSTGRES_INSTALL_DIR ?= $(ROOT_PROJECT_DIR)/tmp_install + # Seccomp BPF is only available for Linux UNAME_S := $(shell uname -s) ifeq ($(UNAME_S),Linux) @@ -55,55 +60,55 @@ zenith: postgres-headers $(CARGO_CMD_PREFIX) cargo build $(CARGO_BUILD_FLAGS) ### PostgreSQL parts -tmp_install/build/config.status: +$(POSTGRES_INSTALL_DIR)/build/config.status: +@echo "Configuring postgres build" - mkdir -p tmp_install/build - (cd tmp_install/build && \ - ../../vendor/postgres/configure CFLAGS='$(PG_CFLAGS)' \ + mkdir -p $(POSTGRES_INSTALL_DIR)/build + (cd $(POSTGRES_INSTALL_DIR)/build && \ + $(ROOT_PROJECT_DIR)/vendor/postgres/configure CFLAGS='$(PG_CFLAGS)' \ $(PG_CONFIGURE_OPTS) \ $(SECCOMP) \ - --prefix=$(abspath tmp_install) > configure.log) + --prefix=$(abspath $(POSTGRES_INSTALL_DIR)) > configure.log) # nicer alias for running 'configure' .PHONY: postgres-configure -postgres-configure: tmp_install/build/config.status +postgres-configure: $(POSTGRES_INSTALL_DIR)/build/config.status -# Install the PostgreSQL header files into tmp_install/include +# Install the PostgreSQL header files into $(POSTGRES_INSTALL_DIR)/include .PHONY: postgres-headers postgres-headers: postgres-configure +@echo "Installing PostgreSQL headers" - $(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/src/include MAKELEVEL=0 install # Compile and install PostgreSQL and contrib/neon .PHONY: postgres postgres: postgres-configure \ postgres-headers # to prevent `make install` conflicts with zenith's `postgres-headers` +@echo "Compiling PostgreSQL" - $(MAKE) -C tmp_install/build MAKELEVEL=0 install + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build MAKELEVEL=0 install +@echo "Compiling contrib/neon" - $(MAKE) -C tmp_install/build/contrib/neon install + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/neon install +@echo "Compiling contrib/neon_test_utils" - $(MAKE) -C tmp_install/build/contrib/neon_test_utils install + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/neon_test_utils install +@echo "Compiling pg_buffercache" - $(MAKE) -C tmp_install/build/contrib/pg_buffercache install + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/pg_buffercache install +@echo "Compiling pageinspect" - $(MAKE) -C tmp_install/build/contrib/pageinspect install + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/contrib/pageinspect install .PHONY: postgres-clean postgres-clean: - $(MAKE) -C tmp_install/build MAKELEVEL=0 clean + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build MAKELEVEL=0 clean # This doesn't remove the effects of 'configure'. .PHONY: clean clean: - cd tmp_install/build && $(MAKE) clean + cd $(POSTGRES_INSTALL_DIR)/build && $(MAKE) clean $(CARGO_CMD_PREFIX) cargo clean # This removes everything .PHONY: distclean distclean: - rm -rf tmp_install + rm -rf $(POSTGRES_INSTALL_DIR) $(CARGO_CMD_PREFIX) cargo clean .PHONY: fmt @@ -112,4 +117,4 @@ fmt: .PHONY: setup-pre-commit-hook setup-pre-commit-hook: - ln -s -f ../../pre-commit.py .git/hooks/pre-commit + ln -s -f $(ROOT_PROJECT_DIR)/pre-commit.py .git/hooks/pre-commit diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 8bcaf5494a..1e812f2aa0 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -295,7 +295,7 @@ impl ComputeNode { handle_roles(&self.spec, &mut client)?; handle_databases(&self.spec, &mut client)?; handle_role_deletions(self, &mut client)?; - handle_grants(&self.spec, &mut client)?; + handle_grants(self, &mut client)?; create_writablity_check_data(&mut client)?; // 'Close' connection diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 041f42acde..bd47614386 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -349,9 +349,11 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { Ok(()) } -// Grant CREATE ON DATABASE to the database owner -// to allow clients create trusted extensions. -pub fn handle_grants(spec: &ComputeSpec, client: &mut Client) -> Result<()> { +/// Grant CREATE ON DATABASE to the database owner and do some other alters and grants +/// to allow users creating trusted extensions and re-creating `public` schema, for example. +pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> { + let spec = &node.spec; + info!("cluster spec grants:"); // We now have a separate `web_access` role to connect to the database @@ -380,5 +382,47 @@ pub fn handle_grants(spec: &ComputeSpec, client: &mut Client) -> Result<()> { client.execute(query.as_str(), &[])?; } + // Do some per-database access adjustments. We'd better do this at db creation time, + // but CREATE DATABASE isn't transactional. So we cannot create db + do some grants + // atomically. + let mut db_connstr = node.connstr.clone(); + for db in &node.spec.cluster.databases { + // database name is always the last and the only component of the path + db_connstr.set_path(&db.name); + + let mut db_client = Client::connect(db_connstr.as_str(), NoTls)?; + + // This will only change ownership on the schema itself, not the objects + // inside it. Without it owner of the `public` schema will be `cloud_admin` + // and database owner cannot do anything with it. SQL procedure ensures + // that it won't error out if schema `public` doesn't exist. + let alter_query = format!( + "DO $$\n\ + DECLARE\n\ + schema_owner TEXT;\n\ + BEGIN\n\ + IF EXISTS(\n\ + SELECT nspname\n\ + FROM pg_catalog.pg_namespace\n\ + WHERE nspname = 'public'\n\ + )\n\ + THEN\n\ + SELECT nspowner::regrole::text\n\ + FROM pg_catalog.pg_namespace\n\ + WHERE nspname = 'public'\n\ + INTO schema_owner;\n\ + \n\ + IF schema_owner = 'cloud_admin' OR schema_owner = 'zenith_admin'\n\ + THEN\n\ + ALTER SCHEMA public OWNER TO {};\n\ + END IF;\n\ + END IF;\n\ + END\n\ + $$;", + db.owner.quote() + ); + db_client.simple_query(&alter_query)?; + } + Ok(()) } diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000000..7585238efe --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +book diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 60114c5fd5..0000000000 --- a/docs/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Zenith documentation - -## Table of contents - -- [authentication.md](authentication.md) — pageserver JWT authentication. -- [docker.md](docker.md) — Docker images and building pipeline. -- [glossary.md](glossary.md) — Glossary of all the terms used in codebase. -- [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI. -- [sourcetree.md](sourcetree.md) — Overview of the source tree layout. -- [pageserver/README.md](/pageserver/README.md) — pageserver overview. -- [postgres_ffi/README.md](/libs/postgres_ffi/README.md) — Postgres FFI overview. -- [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview. -- [safekeeper/README.md](/safekeeper/README.md) — WAL service overview. -- [core_changes.md](core_changes.md) - Description of Zenith changes in Postgres core diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md new file mode 100644 index 0000000000..cf29ee3c6a --- /dev/null +++ b/docs/SUMMARY.md @@ -0,0 +1,84 @@ +# Summary + +[Introduction]() +- [Separation of Compute and Storage](./separation-compute-storage.md) + +# Architecture + +- [Compute]() + - [WAL proposer]() + - [WAL Backpressure]() + - [Postgres changes](./core_changes.md) + +- [Pageserver](./pageserver.md) + - [Services](./pageserver-services.md) + - [Thread management](./pageserver-thread-mgmt.md) + - [WAL Redo](./pageserver-walredo.md) + - [Page cache](./pageserver-pagecache.md) + - [Storage](./pageserver-storage.md) + - [Datadir mapping]() + - [Layer files]() + - [Branching]() + - [Garbage collection]() + - [Cloud Storage]() + - [Processing a GetPage request](./pageserver-processing-getpage.md) + - [Processing WAL](./pageserver-processing-wal.md) + - [Management API]() + - [Tenant Rebalancing]() + +- [WAL Service](walservice.md) + - [Consensus protocol](safekeeper-protocol.md) + - [Management API]() + - [Rebalancing]() + +- [Control Plane]() + +- [Proxy]() + +- [Source view](./sourcetree.md) + - [docker.md](./docker.md) — Docker images and building pipeline. + - [Error handling and logging]() + - [Testing]() + - [Unit testing]() + - [Integration testing]() + - [Benchmarks]() + + +- [Glossary](./glossary.md) + +# Uncategorized + +- [authentication.md](./authentication.md) +- [multitenancy.md](./multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI. +- [settings.md](./settings.md) +#FIXME: move these under sourcetree.md +#- [pageserver/README.md](/pageserver/README.md) +#- [postgres_ffi/README.md](/libs/postgres_ffi/README.md) +#- [test_runner/README.md](/test_runner/README.md) +#- [safekeeper/README.md](/safekeeper/README.md) + + +# RFCs + +- [RFCs](./rfcs/README.md) + +- [002-storage](rfcs/002-storage.md) +- [003-laptop-cli](rfcs/003-laptop-cli.md) +- [004-durability](rfcs/004-durability.md) +- [005-zenith_local](rfcs/005-zenith_local.md) +- [006-laptop-cli-v2-CLI](rfcs/006-laptop-cli-v2-CLI.md) +- [006-laptop-cli-v2-repository-structure](rfcs/006-laptop-cli-v2-repository-structure.md) +- [007-serverless-on-laptop](rfcs/007-serverless-on-laptop.md) +- [008-push-pull](rfcs/008-push-pull.md) +- [009-snapshot-first-storage-cli](rfcs/009-snapshot-first-storage-cli.md) +- [009-snapshot-first-storage](rfcs/009-snapshot-first-storage.md) +- [009-snapshot-first-storage-pitr](rfcs/009-snapshot-first-storage-pitr.md) +- [010-storage_details](rfcs/010-storage_details.md) +- [011-retention-policy](rfcs/011-retention-policy.md) +- [012-background-tasks](rfcs/012-background-tasks.md) +- [013-term-history](rfcs/013-term-history.md) +- [014-safekeepers-gossip](rfcs/014-safekeepers-gossip.md) +- [014-storage-lsm](rfcs/014-storage-lsm.md) +- [015-storage-messaging](rfcs/015-storage-messaging.md) +- [016-connection-routing](rfcs/016-connection-routing.md) +- [cluster-size-limits](rfcs/cluster-size-limits.md) diff --git a/docs/book.toml b/docs/book.toml new file mode 100644 index 0000000000..f83ac2a6aa --- /dev/null +++ b/docs/book.toml @@ -0,0 +1,5 @@ +[book] +language = "en" +multilingual = false +src = "." +title = "Neon architecture" diff --git a/docs/core_changes.md b/docs/core_changes.md index 82c5addd16..86fdc420f7 100644 --- a/docs/core_changes.md +++ b/docs/core_changes.md @@ -1,3 +1,12 @@ +# Postgres core changes + +This lists all the changes that have been made to the PostgreSQL +source tree, as a somewhat logical set of patches. The long-term goal +is to eliminate all these changes, by submitting patches to upstream +and refactoring code into extensions, so that you can run unmodified +PostgreSQL against Neon storage. + + 1. Add t_cid to XLOG record - Why? The cmin/cmax on a heap page is a real bummer. I don't see any other way to fix that than bite the bullet and modify the WAL-logging routine to include the cmin/cmax. diff --git a/docs/pageserver-page-service.md b/docs/pageserver-page-service.md new file mode 100644 index 0000000000..cea9e5a637 --- /dev/null +++ b/docs/pageserver-page-service.md @@ -0,0 +1,9 @@ +# Page Service + +The Page Service listens for GetPage@LSN requests from the Compute Nodes, +and responds with pages from the repository. On each GetPage@LSN request, +it calls into the Repository function + +A separate thread is spawned for each incoming connection to the page +service. The page service uses the libpq protocol to communicate with +the client. The client is a Compute Postgres instance. diff --git a/docs/pageserver-pagecache.md b/docs/pageserver-pagecache.md new file mode 100644 index 0000000000..d9b120bbb9 --- /dev/null +++ b/docs/pageserver-pagecache.md @@ -0,0 +1,8 @@ +# Page cache + +TODO: + +- shared across tenants +- store pages from layer files +- store pages from "in-memory layer" +- store materialized pages diff --git a/docs/pageserver-processing-getpage.md b/docs/pageserver-processing-getpage.md new file mode 100644 index 0000000000..be99ab82d4 --- /dev/null +++ b/docs/pageserver-processing-getpage.md @@ -0,0 +1,4 @@ +# Processing a GetPage request + +TODO: +- sequence diagram that shows how a GetPage@LSN request is processed diff --git a/docs/pageserver-processing-wal.md b/docs/pageserver-processing-wal.md new file mode 100644 index 0000000000..f8c43b6085 --- /dev/null +++ b/docs/pageserver-processing-wal.md @@ -0,0 +1,5 @@ +# Processing WAL + +TODO: +- diagram that shows how incoming WAL is processed +- explain durability, what is fsync'd when, disk_consistent_lsn diff --git a/pageserver/README.md b/docs/pageserver-services.md similarity index 75% rename from pageserver/README.md rename to docs/pageserver-services.md index cb752881af..4e85413513 100644 --- a/pageserver/README.md +++ b/docs/pageserver-services.md @@ -1,15 +1,4 @@ -## Page server architecture - -The Page Server has a few different duties: - -- Respond to GetPage@LSN requests from the Compute Nodes -- Receive WAL from WAL safekeeper -- Replay WAL that's applicable to the chunks that the Page Server maintains -- Backup to S3 - -S3 is the main fault-tolerant storage of all data, as there are no Page Server -replicas. We use a separate fault-tolerant WAL service to reduce latency. It -keeps track of WAL records which are not synced to S3 yet. +# Services The Page Server consists of multiple threads that operate on a shared repository of page versions: @@ -21,18 +10,22 @@ repository of page versions: | WAL receiver | | | +--------------+ - +----+ - +---------+ .......... | | - | | . . | | - GetPage@LSN | | . backup . -------> | S3 | --------------> | Page | repository . . | | - | Service | .......... | | - page | | +----+ + ...... + +---------+ +--------+ . . + | | | | . . + GetPage@LSN | | | backup | -------> . S3 . +-------------> | Page | repository | | . . + | Service | +--------+ . . + page | | ...... <------------- | | - +---------+ +--------------------+ - | Checkpointing / | - | Garbage collection | - +--------------------+ + +---------+ +-----------+ +--------------------+ + | WAL redo | | Checkpointing, | + +----------+ | processes | | Garbage collection | + | | +-----------+ +--------------------+ + | HTTP | + | mgmt API | + | | + +----------+ Legend: @@ -40,28 +33,77 @@ Legend: | | A thread or multi-threaded service +--+ -.... -. . Component at its early development phase. -.... - ---> Data flow <--- ``` -Page Service ------------- +## Page Service The Page Service listens for GetPage@LSN requests from the Compute Nodes, -and responds with pages from the repository. +and responds with pages from the repository. On each GetPage@LSN request, +it calls into the Repository function + +A separate thread is spawned for each incoming connection to the page +service. The page service uses the libpq protocol to communicate with +the client. The client is a Compute Postgres instance. + +## WAL Receiver + +The WAL receiver connects to the external WAL safekeeping service +using PostgreSQL physical streaming replication, and continuously +receives WAL. It decodes the WAL records, and stores them to the +repository. -WAL Receiver ------------- +## Backup service -The WAL receiver connects to the external WAL safekeeping service (or -directly to the primary) using PostgreSQL physical streaming -replication, and continuously receives WAL. It decodes the WAL records, -and stores them to the repository. +The backup service, responsible for storing pageserver recovery data externally. + +Currently, pageserver stores its files in a filesystem directory it's pointed to. +That working directory could be rather ephemeral for such cases as "a pageserver pod running in k8s with no persistent volumes attached". +Therefore, the server interacts with external, more reliable storage to back up and restore its state. + +The code for storage support is extensible and can support arbitrary ones as long as they implement a certain Rust trait. +There are the following implementations present: +* local filesystem — to use in tests mainly +* AWS S3 - to use in production + +Implementation details are covered in the [backup readme](./src/remote_storage/README.md) and corresponding Rust file docs, parameters documentation can be found at [settings docs](../docs/settings.md). + +The backup service is disabled by default and can be enabled to interact with a single remote storage. + +CLI examples: +* Local FS: `${PAGESERVER_BIN} -c "remote_storage={local_path='/some/local/path/'}"` +* AWS S3 : `env AWS_ACCESS_KEY_ID='SOMEKEYAAAAASADSAH*#' AWS_SECRET_ACCESS_KEY='SOMEsEcReTsd292v' ${PAGESERVER_BIN} -c "remote_storage={bucket_name='some-sample-bucket',bucket_region='eu-north-1', prefix_in_bucket='/test_prefix/'}"` + +For Amazon AWS S3, a key id and secret access key could be located in `~/.aws/credentials` if awscli was ever configured to work with the desired bucket, on the AWS Settings page for a certain user. Also note, that the bucket names does not contain any protocols when used on AWS. +For local S3 installations, refer to the their documentation for name format and credentials. + +Similar to other pageserver settings, toml config file can be used to configure either of the storages as backup targets. +Required sections are: + +```toml +[remote_storage] +local_path = '/Users/someonetoignore/Downloads/tmp_dir/' +``` + +or + +```toml +[remote_storage] +bucket_name = 'some-sample-bucket' +bucket_region = 'eu-north-1' +prefix_in_bucket = '/test_prefix/' +``` + +`AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID` env variables can be used to specify the S3 credentials if needed. + + +## Repository background tasks + +The Repository also has a few different background threads and tokio tasks that perform +background duties like dumping accumulated WAL data from memory to disk, reorganizing +files for performance (compaction), and garbage collecting old files. Repository @@ -116,48 +158,6 @@ Remove old on-disk layer files that are no longer needed according to the PITR retention policy -### Backup service - -The backup service, responsible for storing pageserver recovery data externally. - -Currently, pageserver stores its files in a filesystem directory it's pointed to. -That working directory could be rather ephemeral for such cases as "a pageserver pod running in k8s with no persistent volumes attached". -Therefore, the server interacts with external, more reliable storage to back up and restore its state. - -The code for storage support is extensible and can support arbitrary ones as long as they implement a certain Rust trait. -There are the following implementations present: -* local filesystem — to use in tests mainly -* AWS S3 - to use in production - -Implementation details are covered in the [backup readme](./src/remote_storage/README.md) and corresponding Rust file docs, parameters documentation can be found at [settings docs](../docs/settings.md). - -The backup service is disabled by default and can be enabled to interact with a single remote storage. - -CLI examples: -* Local FS: `${PAGESERVER_BIN} -c "remote_storage={local_path='/some/local/path/'}"` -* AWS S3 : `env AWS_ACCESS_KEY_ID='SOMEKEYAAAAASADSAH*#' AWS_SECRET_ACCESS_KEY='SOMEsEcReTsd292v' ${PAGESERVER_BIN} -c "remote_storage={bucket_name='some-sample-bucket',bucket_region='eu-north-1', prefix_in_bucket='/test_prefix/'}"` - -For Amazon AWS S3, a key id and secret access key could be located in `~/.aws/credentials` if awscli was ever configured to work with the desired bucket, on the AWS Settings page for a certain user. Also note, that the bucket names does not contain any protocols when used on AWS. -For local S3 installations, refer to the their documentation for name format and credentials. - -Similar to other pageserver settings, toml config file can be used to configure either of the storages as backup targets. -Required sections are: - -```toml -[remote_storage] -local_path = '/Users/someonetoignore/Downloads/tmp_dir/' -``` - -or - -```toml -[remote_storage] -bucket_name = 'some-sample-bucket' -bucket_region = 'eu-north-1' -prefix_in_bucket = '/test_prefix/' -``` - -`AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID` env variables can be used to specify the S3 credentials if needed. TODO: Sharding -------------------- diff --git a/pageserver/src/layered_repository/README.md b/docs/pageserver-storage.md similarity index 99% rename from pageserver/src/layered_repository/README.md rename to docs/pageserver-storage.md index bd5fa59257..8d03e68ac7 100644 --- a/pageserver/src/layered_repository/README.md +++ b/docs/pageserver-storage.md @@ -1,4 +1,4 @@ -# Overview +# Pageserver storage The main responsibility of the Page Server is to process the incoming WAL, and reprocess it into a format that allows reasonably quick access to any page diff --git a/docs/pageserver-thread-mgmt.md b/docs/pageserver-thread-mgmt.md new file mode 100644 index 0000000000..9ee3e40085 --- /dev/null +++ b/docs/pageserver-thread-mgmt.md @@ -0,0 +1,26 @@ +## Thread management + +Each thread in the system is tracked by the `thread_mgr` module. It +maintains a registry of threads, and which tenant or timeline they are +operating on. This is used for safe shutdown of a tenant, or the whole +system. + +### Handling shutdown + +When a tenant or timeline is deleted, we need to shut down all threads +operating on it, before deleting the data on disk. A thread registered +in the thread registry can check if it has been requested to shut down, +by calling `is_shutdown_requested()`. For async operations, there's also +a `shudown_watcher()` async task that can be used to wake up on shutdown. + +### Sync vs async + +The primary programming model in the page server is synchronous, +blocking code. However, there are some places where async code is +used. Be very careful when mixing sync and async code. + +Async is primarily used to wait for incoming data on network +connections. For example, all WAL receivers have a shared thread pool, +with one async Task for each connection. Once a piece of WAL has been +received from the network, the thread calls the blocking functions in +the Repository to process the WAL. diff --git a/docs/pageserver-walredo.md b/docs/pageserver-walredo.md new file mode 100644 index 0000000000..1de9c177cc --- /dev/null +++ b/docs/pageserver-walredo.md @@ -0,0 +1,77 @@ +# WAL Redo + +To reconstruct a particular page version from an image of the page and +some WAL records, the pageserver needs to replay the WAL records. This +happens on-demand, when a GetPage@LSN request comes in, or as part of +background jobs that reorganize data for faster access. + +It's important that data cannot leak from one tenant to another, and +that a corrupt WAL record on one timeline doesn't affect other tenants +or timelines. + +## Multi-tenant security + +If you have direct access to the WAL directory, or if you have +superuser access to a running PostgreSQL server, it's easy to +construct a malicious or corrupt WAL record that causes the WAL redo +functions to crash, or to execute arbitrary code. That is not a +security problem for PostgreSQL; if you have superuser access, you +have full access to the system anyway. + +The Neon pageserver, however, is multi-tenant. It needs to execute WAL +belonging to different tenants in the same system, and malicious WAL +in one tenant must not affect other tenants. + +A separate WAL redo process is launched for each tenant, and the +process uses the seccomp(2) system call to restrict its access to the +bare minimum needed to replay WAL records. The process does not have +access to the filesystem or network. It can only communicate with the +parent pageserver process through a pipe. + +If an attacker creates a malicious WAL record and injects it into the +WAL stream of a timeline, he can take control of the WAL redo process +in the pageserver. However, the WAL redo process cannot access the +rest of the system. And because there is a separate WAL redo process +for each tenant, the hijacked WAL redo process can only see WAL and +data belonging to the same tenant, which the attacker would have +access to anyway. + +## WAL-redo process communication + +The WAL redo process runs the 'postgres' executable, launched with a +Neon-specific command-line option to put it into WAL-redo process +mode. The pageserver controls the lifetime of the WAL redo processes, +launching them as needed. If a tenant is detached from the pageserver, +any WAL redo processes for that tenant are killed. + +The pageserver communicates with each WAL redo process over its +stdin/stdout/stderr. It works in request-response model with a simple +custom protocol, described in walredo.rs. To replay a set of WAL +records for a page, the pageserver sends the "before" image of the +page and the WAL records over 'stdin', followed by a command to +perform the replay. The WAL redo process responds with an "after" +image of the page. + +## Special handling of some records + +Some WAL record types are handled directly in the pageserver, by +bespoken Rust code, and are not sent over to the WAL redo process. +This includes SLRU-related WAL records, like commit records. SLRUs +don't use the standard Postgres buffer manager, so dealing with them +in the Neon WAL redo mode would require quite a few changes to +Postgres code and special handling in the protocol anyway. + +Some record types that include a full-page-image (e.g. XLOG_FPI) are +also handled specially when incoming WAL is processed already, and are +stored as page images rather than WAL records. + + +## Records that modify multiple pages + +Some Postgres WAL records modify multiple pages. Such WAL records are +duplicated, so that a copy is stored for each affected page. This is +somewhat wasteful, but because most WAL records only affect one page, +the overhead is acceptable. + +The WAL redo always happens for one particular page. If the WAL record +coantains changes to other pages, they are ignored. diff --git a/docs/pageserver.md b/docs/pageserver.md new file mode 100644 index 0000000000..ee70032396 --- /dev/null +++ b/docs/pageserver.md @@ -0,0 +1,11 @@ +# Page server architecture + +The Page Server has a few different duties: + +- Respond to GetPage@LSN requests from the Compute Nodes +- Receive WAL from WAL safekeeper, and store it +- Upload data to S3 to make it durable, download files from S3 as needed + +S3 is the main fault-tolerant storage of all data, as there are no Page Server +replicas. We use a separate fault-tolerant WAL service to reduce latency. It +keeps track of WAL records which are not synced to S3 yet. diff --git a/safekeeper/README_PROTO.md b/docs/safekeeper-protocol.md similarity index 100% rename from safekeeper/README_PROTO.md rename to docs/safekeeper-protocol.md diff --git a/docs/separation-compute-storage.md b/docs/separation-compute-storage.md new file mode 100644 index 0000000000..f07fa8b6dc --- /dev/null +++ b/docs/separation-compute-storage.md @@ -0,0 +1,8 @@ +# Separation of Compute and Storage + +TODO: + +- Read path +- Write path +- Durability model +- API auth diff --git a/safekeeper/README.md b/docs/walservice.md similarity index 100% rename from safekeeper/README.md rename to docs/walservice.md diff --git a/libs/postgres_ffi/build.rs b/libs/postgres_ffi/build.rs index 0043b9ab58..c6df4fc0b0 100644 --- a/libs/postgres_ffi/build.rs +++ b/libs/postgres_ffi/build.rs @@ -2,6 +2,7 @@ extern crate bindgen; use std::env; use std::path::PathBuf; +use std::process::Command; use bindgen::callbacks::ParseCallbacks; @@ -45,6 +46,43 @@ fn main() { // Tell cargo to invalidate the built crate whenever the wrapper changes println!("cargo:rerun-if-changed=pg_control_ffi.h"); + // Finding the location of C headers for the Postgres server: + // - if POSTGRES_INSTALL_DIR is set look into it, otherwise look into `/tmp_install` + // - if there's a `bin/pg_config` file use it for getting include server, otherwise use `/tmp_install/include/postgresql/server` + let mut pg_install_dir: PathBuf; + if let Some(postgres_install_dir) = env::var_os("POSTGRES_INSTALL_DIR") { + pg_install_dir = postgres_install_dir.into(); + } else { + pg_install_dir = PathBuf::from("tmp_install") + } + + if pg_install_dir.is_relative() { + let cwd = env::current_dir().unwrap(); + pg_install_dir = cwd.join("..").join("..").join(pg_install_dir); + } + + let pg_config_bin = pg_install_dir.join("bin").join("pg_config"); + let inc_server_path: String = if pg_config_bin.exists() { + let output = Command::new(pg_config_bin) + .arg("--includedir-server") + .output() + .expect("failed to execute `pg_config --includedir-server`"); + + if !output.status.success() { + panic!("`pg_config --includedir-server` failed") + } + + String::from_utf8(output.stdout).unwrap().trim_end().into() + } else { + pg_install_dir + .join("include") + .join("postgresql") + .join("server") + .into_os_string() + .into_string() + .unwrap() + }; + // The bindgen::Builder is the main entry point // to bindgen, and lets you build up options for // the resulting bindings. @@ -81,15 +119,7 @@ fn main() { // explicit padding fields. .explicit_padding(true) // - // Path the server include dir. It is in tmp_install/include/server, if you did - // "configure --prefix=". But if you used "configure --prefix=/", - // and used DESTDIR to move it into tmp_install, then it's in - // tmp_install/include/postgres/server - // 'pg_config --includedir-server' would perhaps be the more proper way to find it, - // but this will do for now. - // - .clang_arg("-I../../tmp_install/include/server") - .clang_arg("-I../../tmp_install/include/postgresql/server") + .clang_arg(format!("-I{inc_server_path}")) // // Finish the builder and generate the bindings. // diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index 17891fb94f..520870cc53 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -15,6 +15,7 @@ use crate::XLogPageHeaderData; use crate::XLogRecord; use crate::XLOG_PAGE_MAGIC; +use crate::pg_constants::WAL_SEGMENT_SIZE; use anyhow::{bail, ensure}; use byteorder::{ByteOrder, LittleEndian}; use bytes::BytesMut; @@ -461,8 +462,7 @@ pub fn find_end_of_wal( pub fn main() { let mut data_dir = PathBuf::new(); data_dir.push("."); - let wal_seg_size = 16 * 1024 * 1024; - let (wal_end, tli) = find_end_of_wal(&data_dir, wal_seg_size, true, Lsn(0)).unwrap(); + let (wal_end, tli) = find_end_of_wal(&data_dir, WAL_SEGMENT_SIZE, true, Lsn(0)).unwrap(); println!( "wal_end={:>08X}{:>08X}, tli={}", (wal_end >> 32) as u32, @@ -606,10 +606,9 @@ mod tests { fn test_end_of_wal( test_name: &str, expected_end_of_wal_non_partial: Lsn, - last_segment: &str, ) { use wal_craft::*; - // 1. Generate some WAL + // Craft some WAL let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("..") .join(".."); @@ -622,24 +621,71 @@ mod tests { } cfg.initdb().unwrap(); let srv = cfg.start_server().unwrap(); - let expected_wal_end: Lsn = - u64::from(C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap()).into(); + let (intermediate_lsns, expected_end_of_wal_partial) = + C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap(); + let intermediate_lsns: Vec = intermediate_lsns + .iter() + .map(|&lsn| u64::from(lsn).into()) + .collect(); + let expected_end_of_wal_partial: Lsn = u64::from(expected_end_of_wal_partial).into(); srv.kill(); - // 2. Pick WAL generated by initdb - let wal_dir = cfg.datadir.join("pg_wal"); - let wal_seg_size = 16 * 1024 * 1024; + // Check find_end_of_wal on the initial WAL + let last_segment = cfg + .wal_dir() + .read_dir() + .unwrap() + .map(|f| f.unwrap().file_name().into_string().unwrap()) + .filter(|fname| IsXLogFileName(fname)) + .max() + .unwrap(); + check_pg_waldump_end_of_wal(&cfg, &last_segment, expected_end_of_wal_partial); + for start_lsn in std::iter::once(Lsn(0)) + .chain(intermediate_lsns) + .chain(std::iter::once(expected_end_of_wal_partial)) + { + // Erase all WAL before `start_lsn` to ensure it's not used by `find_end_of_wal`. + // We assume that `start_lsn` is non-decreasing. + info!( + "Checking with start_lsn={}, erasing WAL before it", + start_lsn + ); + for file in fs::read_dir(cfg.wal_dir()).unwrap().flatten() { + let fname = file.file_name().into_string().unwrap(); + if !IsXLogFileName(&fname) { + continue; + } + let (segno, _) = XLogFromFileName(&fname, WAL_SEGMENT_SIZE); + let seg_start_lsn = XLogSegNoOffsetToRecPtr(segno, 0, WAL_SEGMENT_SIZE); + if seg_start_lsn > u64::from(start_lsn) { + continue; + } + let mut f = File::options().write(true).open(file.path()).unwrap(); + const ZEROS: [u8; WAL_SEGMENT_SIZE] = [0u8; WAL_SEGMENT_SIZE]; + f.write_all( + &ZEROS[0..min( + WAL_SEGMENT_SIZE, + (u64::from(start_lsn) - seg_start_lsn) as usize, + )], + ) + .unwrap(); + } + check_end_of_wal( + &cfg, + &last_segment, + start_lsn, + expected_end_of_wal_non_partial, + expected_end_of_wal_partial, + ); + } + } - // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated) - let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap(); - let wal_end = Lsn(wal_end); - info!( - "find_end_of_wal returned (wal_end={}, tli={})", - wal_end, tli - ); - assert_eq!(wal_end, expected_end_of_wal_non_partial); - - // 4. Get the actual end of WAL by pg_waldump + fn check_pg_waldump_end_of_wal( + cfg: &wal_craft::Conf, + last_segment: &str, + expected_end_of_wal: Lsn, + ) { + // Get the actual end of WAL by pg_waldump let waldump_output = cfg .pg_waldump("000000010000000000000001", last_segment) .unwrap() @@ -658,32 +704,57 @@ mod tests { let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap(); info!( "waldump erred on {}, expected wal end at {}", - waldump_wal_end, expected_wal_end + waldump_wal_end, expected_end_of_wal ); - assert_eq!(waldump_wal_end, expected_wal_end); + assert_eq!(waldump_wal_end, expected_end_of_wal); + } - // 5. Rename file to partial to actually find last valid lsn - fs::rename( - wal_dir.join(last_segment), - wal_dir.join(format!("{}.partial", last_segment)), - ) - .unwrap(); - let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap(); + fn check_end_of_wal( + cfg: &wal_craft::Conf, + last_segment: &str, + start_lsn: Lsn, + expected_end_of_wal_non_partial: Lsn, + expected_end_of_wal_partial: Lsn, + ) { + // Check end_of_wal on non-partial WAL segment (we treat it as fully populated) + let (wal_end, tli) = + find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, true, start_lsn).unwrap(); let wal_end = Lsn(wal_end); info!( - "find_end_of_wal returned (wal_end={}, tli={})", + "find_end_of_wal returned (wal_end={}, tli={}) with non-partial WAL segment", wal_end, tli ); - assert_eq!(wal_end, waldump_wal_end); + assert_eq!(wal_end, expected_end_of_wal_non_partial); + + // Rename file to partial to actually find last valid lsn, then rename it back. + fs::rename( + cfg.wal_dir().join(&last_segment), + cfg.wal_dir().join(format!("{}.partial", last_segment)), + ) + .unwrap(); + let (wal_end, tli) = + find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, true, start_lsn).unwrap(); + let wal_end = Lsn(wal_end); + info!( + "find_end_of_wal returned (wal_end={}, tli={}) with partial WAL segment", + wal_end, tli + ); + assert_eq!(wal_end, expected_end_of_wal_partial); + fs::rename( + cfg.wal_dir().join(format!("{}.partial", last_segment)), + cfg.wal_dir().join(last_segment), + ) + .unwrap(); } + const_assert!(WAL_SEGMENT_SIZE == 16 * 1024 * 1024); + #[test] pub fn test_find_end_of_wal_simple() { init_logging(); test_end_of_wal::( "test_find_end_of_wal_simple", "0/2000000".parse::().unwrap(), - "000000010000000000000001", ); } @@ -693,7 +764,6 @@ mod tests { test_end_of_wal::( "test_find_end_of_wal_crossing_segment_followed_by_small_one", "0/3000000".parse::().unwrap(), - "000000010000000000000002", ); } @@ -704,7 +774,6 @@ mod tests { test_end_of_wal::( "test_find_end_of_wal_last_crossing_segment", "0/3000000".parse::().unwrap(), - "000000010000000000000002", ); } diff --git a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs index 13892538d0..938f8f421b 100644 --- a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs +++ b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs @@ -55,7 +55,7 @@ fn main() -> Result<()> { .get_matches(); let wal_craft = |arg_matches: &ArgMatches, client| { - let lsn = match arg_matches.value_of("type").unwrap() { + let (intermediate_lsns, end_of_wal_lsn) = match arg_matches.value_of("type").unwrap() { Simple::NAME => Simple::craft(client)?, LastWalRecordXlogSwitch::NAME => LastWalRecordXlogSwitch::craft(client)?, LastWalRecordXlogSwitchEndsOnPageBoundary::NAME => { @@ -67,7 +67,10 @@ fn main() -> Result<()> { LastWalRecordCrossingSegment::NAME => LastWalRecordCrossingSegment::craft(client)?, a => panic!("Unknown --type argument: {}", a), }; - println!("end_of_wal = {}", lsn); + for lsn in intermediate_lsns { + println!("intermediate_lsn = {}", lsn); + } + println!("end_of_wal = {}", end_of_wal_lsn); Ok(()) }; diff --git a/libs/postgres_ffi/wal_craft/src/lib.rs b/libs/postgres_ffi/wal_craft/src/lib.rs index 51482137c8..e3b666da41 100644 --- a/libs/postgres_ffi/wal_craft/src/lib.rs +++ b/libs/postgres_ffi/wal_craft/src/lib.rs @@ -4,6 +4,7 @@ use log::*; use once_cell::sync::Lazy; use postgres::types::PgLsn; use postgres::Client; +use postgres_ffi::pg_constants::WAL_SEGMENT_SIZE; use postgres_ffi::xlog_utils::{ XLOG_BLCKSZ, XLOG_SIZE_OF_XLOG_RECORD, XLOG_SIZE_OF_XLOG_SHORT_PHD, }; @@ -45,6 +46,10 @@ impl Conf { self.pg_distrib_dir.join("lib") } + pub fn wal_dir(&self) -> PathBuf { + self.datadir.join("pg_wal") + } + fn new_pg_command(&self, command: impl AsRef) -> Result { let path = self.pg_bin_dir().join(command); ensure!(path.exists(), "Command {:?} does not exist", path); @@ -211,7 +216,7 @@ pub fn ensure_server_config(client: &mut impl postgres::GenericClient) -> Result "Unexpected wal_segment_size unit" ); ensure!( - wal_segment_size.get::<_, i64>("setting") == 16 * 1024 * 1024, + wal_segment_size.get::<_, i64>("setting") == WAL_SEGMENT_SIZE as i64, "Unexpected wal_segment_size in bytes" ); @@ -221,20 +226,24 @@ pub fn ensure_server_config(client: &mut impl postgres::GenericClient) -> Result pub trait Crafter { const NAME: &'static str; - /// Generates WAL using the client `client`. Returns the expected end-of-wal LSN. - fn craft(client: &mut impl postgres::GenericClient) -> Result; + /// Generates WAL using the client `client`. Returns a pair of: + /// * A vector of some valid "interesting" intermediate LSNs which one may start reading from. + /// May include or exclude Lsn(0) and the end-of-wal. + /// * The expected end-of-wal LSN. + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)>; } fn craft_internal( client: &mut C, - f: impl Fn(&mut C, PgLsn) -> Result>, -) -> Result { + f: impl Fn(&mut C, PgLsn) -> Result<(Vec, Option)>, +) -> Result<(Vec, PgLsn)> { ensure_server_config(client)?; let initial_lsn = client.pg_current_wal_insert_lsn()?; info!("LSN initial = {}", initial_lsn); - let last_lsn = match f(client, initial_lsn)? { + let (mut intermediate_lsns, last_lsn) = f(client, initial_lsn)?; + let last_lsn = match last_lsn { None => client.pg_current_wal_insert_lsn()?, Some(last_lsn) => match last_lsn.cmp(&client.pg_current_wal_insert_lsn()?) { Ordering::Less => bail!("Some records were inserted after the crafted WAL"), @@ -242,6 +251,9 @@ fn craft_internal( Ordering::Greater => bail!("Reported LSN is greater than insert_lsn"), }, }; + if !intermediate_lsns.starts_with(&[initial_lsn]) { + intermediate_lsns.insert(0, initial_lsn); + } // Some records may be not flushed, e.g. non-transactional logical messages. client.execute("select neon_xlogflush(pg_current_wal_insert_lsn())", &[])?; @@ -250,16 +262,16 @@ fn craft_internal( Ordering::Equal => {} Ordering::Greater => bail!("Reported LSN is greater than flush_lsn"), } - Ok(last_lsn) + Ok((intermediate_lsns, last_lsn)) } pub struct Simple; impl Crafter for Simple { const NAME: &'static str = "simple"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { craft_internal(client, |client, _| { client.execute("CREATE table t(x int)", &[])?; - Ok(None) + Ok((Vec::new(), None)) }) } } @@ -267,12 +279,13 @@ impl Crafter for Simple { pub struct LastWalRecordXlogSwitch; impl Crafter for LastWalRecordXlogSwitch { const NAME: &'static str = "last_wal_record_xlog_switch"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { // Do not use generate_internal because here we end up with flush_lsn exactly on // the segment boundary and insert_lsn after the initial page header, which is unusual. ensure_server_config(client)?; client.execute("CREATE table t(x int)", &[])?; + let before_xlog_switch = client.pg_current_wal_insert_lsn()?; let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); let next_segment = PgLsn::from(0x0200_0000); ensure!( @@ -281,14 +294,14 @@ impl Crafter for LastWalRecordXlogSwitch { after_xlog_switch, next_segment ); - Ok(next_segment) + Ok((vec![before_xlog_switch, after_xlog_switch], next_segment)) } } pub struct LastWalRecordXlogSwitchEndsOnPageBoundary; impl Crafter for LastWalRecordXlogSwitchEndsOnPageBoundary { const NAME: &'static str = "last_wal_record_xlog_switch_ends_on_page_boundary"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { // Do not use generate_internal because here we end up with flush_lsn exactly on // the segment boundary and insert_lsn after the initial page header, which is unusual. ensure_server_config(client)?; @@ -334,6 +347,7 @@ impl Crafter for LastWalRecordXlogSwitchEndsOnPageBoundary { ); // Emit the XLOG_SWITCH + let before_xlog_switch = client.pg_current_wal_insert_lsn()?; let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); let next_segment = PgLsn::from(0x0200_0000); ensure!( @@ -347,14 +361,14 @@ impl Crafter for LastWalRecordXlogSwitchEndsOnPageBoundary { "XLOG_SWITCH message ended not on page boundary: {}", after_xlog_switch ); - Ok(next_segment) + Ok((vec![before_xlog_switch, after_xlog_switch], next_segment)) } } fn craft_single_logical_message( client: &mut impl postgres::GenericClient, transactional: bool, -) -> Result { +) -> Result<(Vec, PgLsn)> { craft_internal(client, |client, initial_lsn| { ensure!( initial_lsn < PgLsn::from(0x0200_0000 - 1024 * 1024), @@ -386,9 +400,9 @@ fn craft_single_logical_message( message_lsn < after_message_lsn, "No record found after the emitted message" ); - Ok(Some(after_message_lsn)) + Ok((vec![message_lsn], Some(after_message_lsn))) } else { - Ok(Some(message_lsn)) + Ok((Vec::new(), Some(message_lsn))) } }) } @@ -396,7 +410,7 @@ fn craft_single_logical_message( pub struct WalRecordCrossingSegmentFollowedBySmallOne; impl Crafter for WalRecordCrossingSegmentFollowedBySmallOne { const NAME: &'static str = "wal_record_crossing_segment_followed_by_small_one"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { craft_single_logical_message(client, true) } } @@ -404,7 +418,7 @@ impl Crafter for WalRecordCrossingSegmentFollowedBySmallOne { pub struct LastWalRecordCrossingSegment; impl Crafter for LastWalRecordCrossingSegment { const NAME: &'static str = "last_wal_record_crossing_segment"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { craft_single_logical_message(client, false) } } diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index e977329822..cead2e9222 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -1768,24 +1768,23 @@ impl LayeredTimeline { /// Flush one frozen in-memory layer to disk, as a new delta layer. fn flush_frozen_layer(&self, frozen_layer: Arc) -> Result<()> { - let layer_paths_to_upload; - // As a special case, when we have just imported an image into the repository, // instead of writing out a L0 delta layer, we directly write out image layer // files instead. This is possible as long as *all* the data imported into the // repository have the same LSN. let lsn_range = frozen_layer.get_lsn_range(); - if lsn_range.start == self.initdb_lsn && lsn_range.end == Lsn(self.initdb_lsn.0 + 1) { + let layer_paths_to_upload = if lsn_range.start == self.initdb_lsn + && lsn_range.end == Lsn(self.initdb_lsn.0 + 1) + { let pgdir = tenant_mgr::get_local_timeline_with_load(self.tenant_id, self.timeline_id)?; let (partitioning, _lsn) = pgdir.repartition(self.initdb_lsn, self.get_compaction_target_size())?; - layer_paths_to_upload = - self.create_image_layers(&partitioning, self.initdb_lsn, true)?; + self.create_image_layers(&partitioning, self.initdb_lsn, true)? } else { // normal case, write out a L0 delta layer file. let delta_path = self.create_delta_layer(&frozen_layer)?; - layer_paths_to_upload = HashSet::from([delta_path]); - } + HashSet::from([delta_path]) + }; fail_point!("flush-frozen-before-sync"); diff --git a/pageserver/src/storage_sync.rs b/pageserver/src/storage_sync.rs index d6e3741bc0..ac5fb0bc8c 100644 --- a/pageserver/src/storage_sync.rs +++ b/pageserver/src/storage_sync.rs @@ -928,7 +928,7 @@ fn storage_sync_loop( ); let mut sync_status_updates: HashMap> = HashMap::new(); - let index_accessor = runtime.block_on(index.write()); + let index_accessor = runtime.block_on(index.read()); for tenant_id in updated_tenants { let tenant_entry = match index_accessor.tenant_entry(&tenant_id) { Some(tenant_entry) => tenant_entry, @@ -1557,6 +1557,7 @@ fn schedule_first_sync_tasks( local_timeline_init_statuses } +/// bool in return value stands for awaits_download fn compare_local_and_remote_timeline( new_sync_tasks: &mut VecDeque<(ZTenantTimelineId, SyncTask)>, sync_id: ZTenantTimelineId, @@ -1566,14 +1567,6 @@ fn compare_local_and_remote_timeline( ) -> (LocalTimelineInitStatus, bool) { let remote_files = remote_entry.stored_files(); - // TODO probably here we need more sophisticated logic, - // if more data is available remotely can we just download what's there? - // without trying to upload something. It may be tricky, needs further investigation. - // For now looks strange that we can request upload - // and download for the same timeline simultaneously. - // (upload needs to be only for previously unsynced files, not whole timeline dir). - // If one of the tasks fails they will be reordered in the queue which can lead - // to timeline being stuck in evicted state let number_of_layers_to_download = remote_files.difference(&local_files).count(); let (initial_timeline_status, awaits_download) = if number_of_layers_to_download > 0 { new_sync_tasks.push_back(( diff --git a/pageserver/src/storage_sync/download.rs b/pageserver/src/storage_sync/download.rs index 0f2bdd3bcb..a91eaaa7ca 100644 --- a/pageserver/src/storage_sync/download.rs +++ b/pageserver/src/storage_sync/download.rs @@ -3,12 +3,13 @@ use std::{ collections::{HashMap, HashSet}, fmt::Debug, + mem, path::Path, }; use anyhow::Context; use futures::stream::{FuturesUnordered, StreamExt}; -use remote_storage::{path_with_suffix_extension, RemoteObjectName, RemoteStorage}; +use remote_storage::{path_with_suffix_extension, DownloadError, RemoteObjectName, RemoteStorage}; use tokio::{ fs, io::{self, AsyncWriteExt}, @@ -27,28 +28,50 @@ use super::{ pub const TEMP_DOWNLOAD_EXTENSION: &str = "temp_download"; -/// FIXME: Needs cleanup. Currently it swallows errors. Here we need to ensure that -/// we successfully downloaded all metadata parts for one tenant. -/// And successful includes absence of index_part in the remote. Because it is valid situation -/// when timeline was just created and pageserver restarted before upload of index part was completed. -/// But currently RemoteStorage interface does not provide this knowledge because it uses -/// anyhow::Error as an error type. So this needs a refactoring. -/// -/// In other words we need to yield only complete sets of tenant timelines. -/// Failure for one timeline of a tenant should exclude whole tenant from returned hashmap. -/// So there are two requirements: keep everything in one futures unordered -/// to allow higher concurrency. Mark tenants as failed independently. -/// That requires some bookeeping. +// We collect timelines remotely available for each tenant +// in case we failed to gather all index parts (due to an error) +// Poisoned variant is returned. +// When data is received succesfully without errors Present variant is used. +pub enum TenantIndexParts { + Poisoned { + present: HashMap, + missing: HashSet, + }, + Present(HashMap), +} + +impl TenantIndexParts { + fn add_poisoned(&mut self, timeline_id: ZTimelineId) { + match self { + TenantIndexParts::Poisoned { missing, .. } => { + missing.insert(timeline_id); + } + TenantIndexParts::Present(present) => { + *self = TenantIndexParts::Poisoned { + present: mem::take(present), + missing: HashSet::from([timeline_id]), + } + } + } + } +} + +impl Default for TenantIndexParts { + fn default() -> Self { + TenantIndexParts::Present(HashMap::default()) + } +} + pub async fn download_index_parts( conf: &'static PageServerConf, storage: &S, keys: HashSet, -) -> HashMap> +) -> HashMap where P: Debug + Send + Sync + 'static, S: RemoteStorage + Send + Sync + 'static, { - let mut index_parts: HashMap> = HashMap::new(); + let mut index_parts: HashMap = HashMap::new(); let mut part_downloads = keys .into_iter() @@ -59,12 +82,29 @@ where match part_upload_result { Ok(index_part) => { debug!("Successfully fetched index part for {id}"); - index_parts - .entry(id.tenant_id) - .or_default() - .insert(id.timeline_id, index_part); + match index_parts.entry(id.tenant_id).or_default() { + TenantIndexParts::Poisoned { present, .. } => { + present.insert(id.timeline_id, index_part); + } + TenantIndexParts::Present(parts) => { + parts.insert(id.timeline_id, index_part); + } + } + } + Err(download_error) => { + match download_error { + DownloadError::NotFound => { + // thats ok because it means that we didnt upload something we have locally for example + } + e => { + let tenant_parts = index_parts.entry(id.tenant_id).or_default(); + tenant_parts.add_poisoned(id.timeline_id); + error!( + "Failed to fetch index part for {id}: {e} poisoning tenant index parts" + ); + } + } } - Err(e) => error!("Failed to fetch index part for {id}: {e}"), } } @@ -119,12 +159,16 @@ where }); } - download_index_parts(conf, storage, sync_ids) + match download_index_parts(conf, storage, sync_ids) .await .remove(&tenant_id) - .ok_or(anyhow::anyhow!( - "Missing tenant index parts. This is a bug." - )) + .ok_or_else(|| anyhow::anyhow!("Missing tenant index parts. This is a bug."))? + { + TenantIndexParts::Poisoned { missing, .. } => { + anyhow::bail!("Failed to download index parts for all timelines. Missing {missing:?}") + } + TenantIndexParts::Present(parts) => Ok(parts), + } } /// Retrieves index data from the remote storage for a given timeline. @@ -132,7 +176,7 @@ async fn download_index_part( conf: &'static PageServerConf, storage: &S, sync_id: ZTenantTimelineId, -) -> anyhow::Result +) -> Result where P: Debug + Send + Sync + 'static, S: RemoteStorage + Send + Sync + 'static, @@ -147,15 +191,11 @@ where "Failed to get the index part storage path for local path '{}'", index_part_path.display() ) - })?; + }) + .map_err(DownloadError::BadInput)?; + + let mut index_part_download = storage.download(&part_storage_path).await?; - let mut index_part_download = - storage - .download(&part_storage_path) - .await - .with_context(|| { - format!("Failed to open download stream for for storage path {part_storage_path:?}") - })?; let mut index_part_bytes = Vec::new(); io::copy( &mut index_part_download.download_stream, @@ -164,11 +204,16 @@ where .await .with_context(|| { format!("Failed to download an index part from storage path {part_storage_path:?}") - })?; + }) + .map_err(DownloadError::Other)?; - let index_part: IndexPart = serde_json::from_slice(&index_part_bytes).with_context(|| { - format!("Failed to deserialize index part file from storage path '{part_storage_path:?}'") - })?; + let index_part: IndexPart = serde_json::from_slice(&index_part_bytes) + .with_context(|| { + format!( + "Failed to deserialize index part file from storage path '{part_storage_path:?}'" + ) + }) + .map_err(DownloadError::Other)?; let missing_files = index_part.missing_files(); if !missing_files.is_empty() { diff --git a/pageserver/src/storage_sync/index.rs b/pageserver/src/storage_sync/index.rs index 54be3d0f8c..134ae893bc 100644 --- a/pageserver/src/storage_sync/index.rs +++ b/pageserver/src/storage_sync/index.rs @@ -13,6 +13,7 @@ use anyhow::{anyhow, Context, Ok}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use tokio::sync::RwLock; +use tracing::log::warn; use crate::{config::PageServerConf, layered_repository::metadata::TimelineMetadata}; use utils::{ @@ -20,6 +21,8 @@ use utils::{ zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}, }; +use super::download::TenantIndexParts; + /// A part of the filesystem path, that needs a root to become a path again. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(transparent)] @@ -88,21 +91,27 @@ pub struct RemoteIndex(Arc>); impl RemoteIndex { pub fn from_parts( conf: &'static PageServerConf, - index_parts: HashMap>, + index_parts: HashMap, ) -> anyhow::Result { let mut entries: HashMap = HashMap::new(); - for (tenant_id, timelines) in index_parts { - for (timeline_id, index_part) in timelines { - let timeline_path = conf.timeline_path(&timeline_id, &tenant_id); - let remote_timeline = - RemoteTimeline::from_index_part(&timeline_path, index_part) - .context("Failed to restore remote timeline data from index part")?; + for (tenant_id, index_parts) in index_parts { + match index_parts { + // TODO: should we schedule a retry so it can be recovered? otherwise we can revive it only through detach/attach or pageserver restart + TenantIndexParts::Poisoned { missing, ..} => warn!("skipping tenant_id set up for remote index because the index download has failed for timeline(s): {missing:?}"), + TenantIndexParts::Present(timelines) => { + for (timeline_id, index_part) in timelines { + let timeline_path = conf.timeline_path(&timeline_id, &tenant_id); + let remote_timeline = + RemoteTimeline::from_index_part(&timeline_path, index_part) + .context("Failed to restore remote timeline data from index part")?; - entries - .entry(tenant_id) - .or_default() - .insert(timeline_id, remote_timeline); + entries + .entry(tenant_id) + .or_default() + .insert(timeline_id, remote_timeline); + } + }, } } diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index cad211b1bd..db4620417c 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -623,6 +623,7 @@ impl PostgresRedoProcess { .env_clear() .env("LD_LIBRARY_PATH", conf.pg_lib_dir()) .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir()) + .close_fds() .output() .map_err(|e| Error::new(e.kind(), format!("failed to execute initdb: {}", e)))?; diff --git a/poetry.lock b/poetry.lock index 6e552d2cd3..4963390718 100644 --- a/poetry.lock +++ b/poetry.lock @@ -544,20 +544,21 @@ test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pr [[package]] name = "docker" -version = "5.0.3" +version = "4.2.2" description = "A Python library for the Docker Engine API." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -pywin32 = {version = "227", markers = "sys_platform == \"win32\""} +pypiwin32 = {version = "223", markers = "sys_platform == \"win32\" and python_version >= \"3.6\""} requests = ">=2.14.2,<2.18.0 || >2.18.0" +six = ">=1.4.0" websocket-client = ">=0.32.0" [package.extras] ssh = ["paramiko (>=2.4.2)"] -tls = ["pyOpenSSL (>=17.5.0)", "cryptography (>=3.4.7)", "idna (>=2.0.0)"] +tls = ["pyOpenSSL (>=17.5.0)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] [[package]] name = "ecdsa" @@ -1003,6 +1004,17 @@ python-versions = ">=3.6" [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pypiwin32" +version = "223" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pywin32 = ">=223" + [[package]] name = "pyrsistent" version = "0.18.1" @@ -1124,7 +1136,7 @@ python-versions = "*" [[package]] name = "pywin32" -version = "227" +version = "301" description = "Python for Window Extensions" category = "main" optional = false @@ -1501,8 +1513,8 @@ cryptography = [ {file = "cryptography-36.0.1.tar.gz", hash = "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638"}, ] docker = [ - {file = "docker-5.0.3-py2.py3-none-any.whl", hash = "sha256:7a79bb439e3df59d0a72621775d600bc8bc8b422d285824cb37103eab91d1ce0"}, - {file = "docker-5.0.3.tar.gz", hash = "sha256:d916a26b62970e7c2f554110ed6af04c7ccff8e9f81ad17d0d40c75637e227fb"}, + {file = "docker-4.2.2-py2.py3-none-any.whl", hash = "sha256:03a46400c4080cb6f7aa997f881ddd84fef855499ece219d75fbdb53289c17ab"}, + {file = "docker-4.2.2.tar.gz", hash = "sha256:26eebadce7e298f55b76a88c4f8802476c5eaddbdbe38dbc6cce8781c47c9b54"}, ] ecdsa = [ {file = "ecdsa-0.17.0-py2.py3-none-any.whl", hash = "sha256:5cf31d5b33743abe0dfc28999036c849a69d548f994b535e527ee3cb7f3ef676"}, @@ -1802,6 +1814,10 @@ pyparsing = [ {file = "pyparsing-3.0.6-py3-none-any.whl", hash = "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4"}, {file = "pyparsing-3.0.6.tar.gz", hash = "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"}, ] +pypiwin32 = [ + {file = "pypiwin32-223-py3-none-any.whl", hash = "sha256:67adf399debc1d5d14dffc1ab5acacb800da569754fafdc576b2a039485aa775"}, + {file = "pypiwin32-223.tar.gz", hash = "sha256:71be40c1fbd28594214ecaecb58e7aa8b708eabfa0125c8a109ebd51edbd776a"}, +] pyrsistent = [ {file = "pyrsistent-0.18.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1"}, {file = "pyrsistent-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26"}, @@ -1858,18 +1874,16 @@ pytz = [ {file = "pytz-2021.3.tar.gz", hash = "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"}, ] pywin32 = [ - {file = "pywin32-227-cp27-cp27m-win32.whl", hash = "sha256:371fcc39416d736401f0274dd64c2302728c9e034808e37381b5e1b22be4a6b0"}, - {file = "pywin32-227-cp27-cp27m-win_amd64.whl", hash = "sha256:4cdad3e84191194ea6d0dd1b1b9bdda574ff563177d2adf2b4efec2a244fa116"}, - {file = "pywin32-227-cp35-cp35m-win32.whl", hash = "sha256:f4c5be1a293bae0076d93c88f37ee8da68136744588bc5e2be2f299a34ceb7aa"}, - {file = "pywin32-227-cp35-cp35m-win_amd64.whl", hash = "sha256:a929a4af626e530383a579431b70e512e736e9588106715215bf685a3ea508d4"}, - {file = "pywin32-227-cp36-cp36m-win32.whl", hash = "sha256:300a2db938e98c3e7e2093e4491439e62287d0d493fe07cce110db070b54c0be"}, - {file = "pywin32-227-cp36-cp36m-win_amd64.whl", hash = "sha256:9b31e009564fb95db160f154e2aa195ed66bcc4c058ed72850d047141b36f3a2"}, - {file = "pywin32-227-cp37-cp37m-win32.whl", hash = "sha256:47a3c7551376a865dd8d095a98deba954a98f326c6fe3c72d8726ca6e6b15507"}, - {file = "pywin32-227-cp37-cp37m-win_amd64.whl", hash = "sha256:31f88a89139cb2adc40f8f0e65ee56a8c585f629974f9e07622ba80199057511"}, - {file = "pywin32-227-cp38-cp38-win32.whl", hash = "sha256:7f18199fbf29ca99dff10e1f09451582ae9e372a892ff03a28528a24d55875bc"}, - {file = "pywin32-227-cp38-cp38-win_amd64.whl", hash = "sha256:7c1ae32c489dc012930787f06244426f8356e129184a02c25aef163917ce158e"}, - {file = "pywin32-227-cp39-cp39-win32.whl", hash = "sha256:c054c52ba46e7eb6b7d7dfae4dbd987a1bb48ee86debe3f245a2884ece46e295"}, - {file = "pywin32-227-cp39-cp39-win_amd64.whl", hash = "sha256:f27cec5e7f588c3d1051651830ecc00294f90728d19c3bf6916e6dba93ea357c"}, + {file = "pywin32-301-cp35-cp35m-win32.whl", hash = "sha256:93367c96e3a76dfe5003d8291ae16454ca7d84bb24d721e0b74a07610b7be4a7"}, + {file = "pywin32-301-cp35-cp35m-win_amd64.whl", hash = "sha256:9635df6998a70282bd36e7ac2a5cef9ead1627b0a63b17c731312c7a0daebb72"}, + {file = "pywin32-301-cp36-cp36m-win32.whl", hash = "sha256:c866f04a182a8cb9b7855de065113bbd2e40524f570db73ef1ee99ff0a5cc2f0"}, + {file = "pywin32-301-cp36-cp36m-win_amd64.whl", hash = "sha256:dafa18e95bf2a92f298fe9c582b0e205aca45c55f989937c52c454ce65b93c78"}, + {file = "pywin32-301-cp37-cp37m-win32.whl", hash = "sha256:98f62a3f60aa64894a290fb7494bfa0bfa0a199e9e052e1ac293b2ad3cd2818b"}, + {file = "pywin32-301-cp37-cp37m-win_amd64.whl", hash = "sha256:fb3b4933e0382ba49305cc6cd3fb18525df7fd96aa434de19ce0878133bf8e4a"}, + {file = "pywin32-301-cp38-cp38-win32.whl", hash = "sha256:88981dd3cfb07432625b180f49bf4e179fb8cbb5704cd512e38dd63636af7a17"}, + {file = "pywin32-301-cp38-cp38-win_amd64.whl", hash = "sha256:8c9d33968aa7fcddf44e47750e18f3d034c3e443a707688a008a2e52bbef7e96"}, + {file = "pywin32-301-cp39-cp39-win32.whl", hash = "sha256:595d397df65f1b2e0beaca63a883ae6d8b6df1cdea85c16ae85f6d2e648133fe"}, + {file = "pywin32-301-cp39-cp39-win_amd64.whl", hash = "sha256:87604a4087434cd814ad8973bd47d6524bd1fa9e971ce428e76b62a5e0860fdf"}, ] pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, diff --git a/safekeeper/src/broker.rs b/safekeeper/src/broker.rs index 8e0eb971f3..ce66131700 100644 --- a/safekeeper/src/broker.rs +++ b/safekeeper/src/broker.rs @@ -83,7 +83,9 @@ impl ElectionLeader { ) -> Result { let resp = self.client.leader(election_name).await?; - let kv = resp.kv().ok_or(anyhow!("failed to get leader response"))?; + let kv = resp + .kv() + .ok_or_else(|| anyhow!("failed to get leader response"))?; let leader = kv.value_str()?; Ok(leader == candidate_name) diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 331baffbca..fd4761505d 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -637,6 +637,17 @@ where &mut self, msg: &VoteRequest, ) -> Result> { + // Once voted, we won't accept data from older proposers; flush + // everything we've already received so that new proposer starts + // streaming at end of our WAL, without overlap. Currently we truncate + // WAL at streaming point, so this avoids truncating already committed + // WAL. + // + // TODO: it would be smoother to not truncate committed piece at + // handle_elected instead. Currently not a big deal, as proposer is the + // only source of WAL; with peer2peer recovery it would be more + // important. + self.wal_store.flush_wal()?; // initialize with refusal let mut resp = VoteResponse { term: self.state.acceptor_state.term, diff --git a/test_runner/batch_others/test_branch_and_gc.py b/test_runner/batch_others/test_branch_and_gc.py new file mode 100644 index 0000000000..a6210b9176 --- /dev/null +++ b/test_runner/batch_others/test_branch_and_gc.py @@ -0,0 +1,101 @@ +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnv +from fixtures.utils import lsn_from_hex + + +# Test the GC implementation when running with branching. +# This test reproduces the issue https://github.com/neondatabase/neon/issues/707. +# +# Consider two LSNs `lsn1` and `lsn2` with some delta files as follows: +# ... +# p -> has an image layer xx_p with p < lsn1 +# ... +# lsn1 +# ... +# q -> has an image layer yy_q with lsn1 < q < lsn2 +# ... +# lsn2 +# +# Consider running a GC iteration such that the GC horizon is between p and lsn1 +# ... +# p -> has an image layer xx_p with p < lsn1 +# D_start -> is a delta layer D's start (e.g D = '...-...-D_start-D_end') +# ... +# GC_h -> is a gc horizon such that p < GC_h < lsn1 +# ... +# lsn1 +# ... +# D_end -> is a delta layer D's end +# ... +# q -> has an image layer yy_q with lsn1 < q < lsn2 +# ... +# lsn2 +# +# As described in the issue #707, the image layer xx_p will be deleted as +# its range is below the GC horizon and there exists a newer image layer yy_q (q > p). +# However, removing xx_p will corrupt any delta layers that depend on xx_p that +# are not deleted by GC. For example, the delta layer D is corrupted in the +# above example because D depends on the image layer xx_p for value reconstruction. +# +# Because the delta layer D covering lsn1 is corrupted, creating a branch +# starting from lsn1 should return an error as follows: +# could not find data for key ... at LSN ..., for request at LSN ... +def test_branch_and_gc(neon_simple_env: NeonEnv): + env = neon_simple_env + + tenant, _ = env.neon_cli.create_tenant( + conf={ + # disable background GC + 'gc_period': '10 m', + 'gc_horizon': f'{10 * 1024 ** 3}', + + # small checkpoint distance to create more delta layer files + 'checkpoint_distance': f'{1024 ** 2}', + + # set the target size to be large to allow the image layer to cover the whole key space + 'compaction_target_size': f'{1024 ** 3}', + + # tweak the default settings to allow quickly create image layers and L1 layers + 'compaction_period': '1 s', + 'compaction_threshold': '2', + 'image_creation_threshold': '1', + + # set PITR interval to be small, so we can do GC + 'pitr_interval': '1 s' + }) + + timeline_main = env.neon_cli.create_timeline(f'test_main', tenant_id=tenant) + pg_main = env.postgres.create_start('test_main', tenant_id=tenant) + + main_cur = pg_main.connect().cursor() + + main_cur.execute( + "CREATE TABLE foo(key serial primary key, t text default 'foooooooooooooooooooooooooooooooooooooooooooooooooooo')" + ) + main_cur.execute('INSERT INTO foo SELECT FROM generate_series(1, 100000)') + main_cur.execute('SELECT pg_current_wal_insert_lsn()') + lsn1 = main_cur.fetchone()[0] + log.info(f'LSN1: {lsn1}') + + main_cur.execute('INSERT INTO foo SELECT FROM generate_series(1, 100000)') + main_cur.execute('SELECT pg_current_wal_insert_lsn()') + lsn2 = main_cur.fetchone()[0] + log.info(f'LSN2: {lsn2}') + + # Set the GC horizon so that lsn1 is inside the horizon, which means + # we can create a new branch starting from lsn1. + env.pageserver.safe_psql( + f'''do_gc {tenant.hex} {timeline_main.hex} {lsn_from_hex(lsn2) - lsn_from_hex(lsn1) + 1024}''' + ) + + env.neon_cli.create_branch('test_branch', + 'test_main', + tenant_id=tenant, + ancestor_start_lsn=lsn1) + pg_branch = env.postgres.create_start('test_branch', tenant_id=tenant) + + branch_cur = pg_branch.connect().cursor() + branch_cur.execute('INSERT INTO foo SELECT FROM generate_series(1, 100000)') + + branch_cur.execute('SELECT count(*) FROM foo') + assert branch_cur.fetchone() == (200000, ) diff --git a/test_runner/batch_others/test_branching.py b/test_runner/batch_others/test_branching.py new file mode 100644 index 0000000000..c61bac7a58 --- /dev/null +++ b/test_runner/batch_others/test_branching.py @@ -0,0 +1,89 @@ +from typing import List +import threading +import pytest +from fixtures.neon_fixtures import NeonEnv, PgBin, Postgres +import time +import random +from fixtures.log_helper import log +from performance.test_perf_pgbench import get_scales_matrix + + +# Test branch creation +# +# This test spawns pgbench in a thread in the background, and creates a branch while +# pgbench is running. Then it launches pgbench on the new branch, and creates another branch. +# Repeat `n_branches` times. +# +# If 'ty' == 'cascade', each branch is created from the previous branch, so that you end +# up with a branch of a branch of a branch ... of a branch. With 'ty' == 'flat', +# each branch is created from the root. +@pytest.mark.parametrize("n_branches", [10]) +@pytest.mark.parametrize("scale", get_scales_matrix(1)) +@pytest.mark.parametrize("ty", ["cascade", "flat"]) +def test_branching_with_pgbench(neon_simple_env: NeonEnv, + pg_bin: PgBin, + n_branches: int, + scale: int, + ty: str): + env = neon_simple_env + + # Use aggressive GC and checkpoint settings, so that we also exercise GC during the test + tenant, _ = env.neon_cli.create_tenant( + conf={ + 'gc_period': '5 s', + 'gc_horizon': f'{1024 ** 2}', + 'checkpoint_distance': f'{1024 ** 2}', + 'compaction_target_size': f'{1024 ** 2}', + # set PITR interval to be small, so we can do GC + 'pitr_interval': '5 s' + }) + + def run_pgbench(pg: Postgres): + connstr = pg.connstr() + + log.info(f"Start a pgbench workload on pg {connstr}") + + pg_bin.run_capture(['pgbench', '-i', f'-s{scale}', connstr]) + pg_bin.run_capture(['pgbench', '-T15', connstr]) + + env.neon_cli.create_branch('b0', tenant_id=tenant) + pgs: List[Postgres] = [] + pgs.append(env.postgres.create_start('b0', tenant_id=tenant)) + + threads: List[threading.Thread] = [] + threads.append(threading.Thread(target=run_pgbench, args=(pgs[0], ), daemon=True)) + threads[-1].start() + + thread_limit = 4 + + for i in range(n_branches): + # random a delay between [0, 5] + delay = random.random() * 5 + time.sleep(delay) + log.info(f"Sleep {delay}s") + + # If the number of concurrent threads exceeds a threshold, + # wait for all the threads to finish before spawning a new one. + # Because tests defined in `batch_others` are run concurrently in CI, + # we want to avoid the situation that one test exhausts resources for other tests. + if len(threads) >= thread_limit: + for thread in threads: + thread.join() + threads = [] + + if ty == "cascade": + env.neon_cli.create_branch('b{}'.format(i + 1), 'b{}'.format(i), tenant_id=tenant) + else: + env.neon_cli.create_branch('b{}'.format(i + 1), 'b0', tenant_id=tenant) + + pgs.append(env.postgres.create_start('b{}'.format(i + 1), tenant_id=tenant)) + + threads.append(threading.Thread(target=run_pgbench, args=(pgs[-1], ), daemon=True)) + threads[-1].start() + + for thread in threads: + thread.join() + + for pg in pgs: + res = pg.safe_psql('SELECT count(*) from pgbench_accounts') + assert res[0] == (100000 * scale, ) diff --git a/test_runner/batch_others/test_close_fds.py b/test_runner/batch_others/test_close_fds.py new file mode 100644 index 0000000000..9521b1bb4a --- /dev/null +++ b/test_runner/batch_others/test_close_fds.py @@ -0,0 +1,51 @@ +from contextlib import closing +import shutil +import time +import subprocess +import os.path + +from cached_property import threading +from fixtures.neon_fixtures import NeonEnv +from fixtures.log_helper import log + + +def lsof_path() -> str: + path_output = shutil.which("lsof") + if path_output is None: + raise RuntimeError('lsof not found in PATH') + else: + return path_output + + +# Makes sure that `pageserver.pid` is only held by `pageserve` command, not other commands. +# This is to test the changes in https://github.com/neondatabase/neon/pull/1834. +def test_lsof_pageserver_pid(neon_simple_env: NeonEnv): + env = neon_simple_env + + def start_workload(): + env.neon_cli.create_branch("test_lsof_pageserver_pid") + pg = env.postgres.create_start("test_lsof_pageserver_pid") + with closing(pg.connect()) as conn: + with conn.cursor() as cur: + cur.execute("CREATE TABLE foo as SELECT x FROM generate_series(1,100000) x") + cur.execute("update foo set x=x+1") + + workload_thread = threading.Thread(target=start_workload, args=(), daemon=True) + workload_thread.start() + + path = os.path.join(env.repo_dir, "pageserver.pid") + lsof = lsof_path() + while workload_thread.is_alive(): + res = subprocess.run([lsof, path], + check=False, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # parse the `lsof` command's output to get only the list of commands + commands = [line.split(' ')[0] for line in res.stdout.strip().split('\n')[1:]] + if len(commands) > 0: + log.info(f"lsof commands: {commands}") + assert commands == ['pageserve'] + + time.sleep(1.0) diff --git a/test_runner/batch_others/test_wal_acceptor_async.py b/test_runner/batch_others/test_wal_acceptor_async.py index 4664c332fc..d74ef8840a 100644 --- a/test_runner/batch_others/test_wal_acceptor_async.py +++ b/test_runner/batch_others/test_wal_acceptor_async.py @@ -302,6 +302,8 @@ def test_compute_restarts(neon_env_builder: NeonEnvBuilder): class BackgroundCompute(object): + MAX_QUERY_GAP_SECONDS = 2 + def __init__(self, index: int, env: NeonEnv, branch: str): self.index = index self.env = env @@ -339,7 +341,7 @@ class BackgroundCompute(object): # With less sleep, there is a very big chance of not committing # anything or only 1 xact during test run. - await asyncio.sleep(2 * random.random()) + await asyncio.sleep(random.uniform(0, self.MAX_QUERY_GAP_SECONDS)) self.running = False @@ -356,20 +358,34 @@ async def run_concurrent_computes(env: NeonEnv, background_tasks = [asyncio.create_task(compute.run()) for compute in computes] await asyncio.sleep(run_seconds) + log.info("stopping all tasks but one") for compute in computes[1:]: compute.stopped = True + await asyncio.gather(*background_tasks[1:]) log.info("stopped all tasks but one") # work for some time with only one compute -- it should be able to make some xacts - await asyncio.sleep(8) + TIMEOUT_SECONDS = computes[0].MAX_QUERY_GAP_SECONDS + 3 + initial_queries_by_0 = len(computes[0].successful_queries) + log.info(f'Waiting for another query by computes[0], ' + f'it already had {initial_queries_by_0}, timeout is {TIMEOUT_SECONDS}s') + for _ in range(10 * TIMEOUT_SECONDS): + current_queries_by_0 = len(computes[0].successful_queries) - initial_queries_by_0 + if current_queries_by_0 >= 1: + log.info(f'Found {current_queries_by_0} successful queries ' + f'by computes[0], completing the test') + break + await asyncio.sleep(0.1) + else: + assert False, "Timed out while waiting for another query by computes[0]" computes[0].stopped = True - await asyncio.gather(*background_tasks) + await asyncio.gather(background_tasks[0]) result = await exec_compute_query(env, branch, 'SELECT * FROM query_log') # we should have inserted something while single compute was running - assert len(result) >= 4 - log.info(f'Executed {len(result)} queries') + log.info(f'Executed {len(result)} queries, {current_queries_by_0} of them ' + f'by computes[0] after we started stopping the others') for row in result: log.info(f'{row[0]} {row[1]} {row[2]}') diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index e2bf7da79d..3a6a233208 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1276,12 +1276,9 @@ class WalCraft(AbstractNeonCli): res.check_returncode() return res.stdout.split('\n') - def in_existing(self, type: str, connection: str) -> int: + def in_existing(self, type: str, connection: str) -> None: res = self.raw_cli(["in-existing", type, connection]) res.check_returncode() - m = re.fullmatch(r'end_of_wal = (.*)\n', res.stdout) - assert m - return lsn_from_hex(m.group(1)) class NeonPageserver(PgProtocol): diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index 05d1a6634d..c49fa08d77 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -83,6 +83,9 @@ def get_dir_size(path: str) -> int: totalbytes = 0 for root, dirs, files in os.walk(path): for name in files: - totalbytes += os.path.getsize(os.path.join(root, name)) + try: + totalbytes += os.path.getsize(os.path.join(root, name)) + except FileNotFoundError as e: + pass # file could be concurrently removed return totalbytes diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 92877faef7..4dc7e4e157 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -33,7 +33,9 @@ itoa = { version = "0.4", features = ["i128", "std"] } libc = { version = "0.2", features = ["extra_traits", "std"] } log = { version = "0.4", default-features = false, features = ["serde", "std"] } memchr = { version = "2", features = ["std", "use_std"] } -num-integer = { version = "0.1", default-features = false, features = ["i128"] } +nom = { version = "7", features = ["alloc", "std"] } +num-bigint = { version = "0.4", features = ["std"] } +num-integer = { version = "0.1", default-features = false, features = ["i128", "std"] } num-traits = { version = "0.2", features = ["i128", "std"] } prost = { version = "0.10", features = ["prost-derive", "std"] } rand = { version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha", "rand_hc", "small_rng", "std", "std_rng"] } @@ -41,10 +43,11 @@ regex = { version = "1", features = ["aho-corasick", "memchr", "perf", "perf-cac regex-syntax = { version = "0.6", features = ["unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] } scopeguard = { version = "1", features = ["use_std"] } serde = { version = "1", features = ["alloc", "derive", "serde_derive", "std"] } -tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "process", "rt", "rt-multi-thread", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros"] } +time = { version = "0.3", features = ["alloc", "formatting", "itoa", "macros", "parsing", "quickcheck", "quickcheck-dep", "std", "time-macros"] } +tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "process", "rt", "rt-multi-thread", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros", "winapi"] } tokio-util = { version = "0.7", features = ["codec", "io"] } tracing = { version = "0.1", features = ["attributes", "log", "std", "tracing-attributes"] } -tracing-core = { version = "0.1", features = ["lazy_static", "std"] } +tracing-core = { version = "0.1", features = ["lazy_static", "std", "valuable"] } [build-dependencies] ahash = { version = "0.7", features = ["std"] } @@ -57,6 +60,7 @@ indexmap = { version = "1", default-features = false, features = ["std"] } libc = { version = "0.2", features = ["extra_traits", "std"] } log = { version = "0.4", default-features = false, features = ["serde", "std"] } memchr = { version = "2", features = ["std", "use_std"] } +nom = { version = "7", features = ["alloc", "std"] } prost = { version = "0.10", features = ["prost-derive", "std"] } regex = { version = "1", features = ["aho-corasick", "memchr", "perf", "perf-cache", "perf-dfa", "perf-inline", "perf-literal", "std", "unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] } regex-syntax = { version = "0.6", features = ["unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] }