Compare commits

..

3 Commits

Author SHA1 Message Date
Christian Schwarz
1bc5ae03d9 clippy 2025-07-24 13:47:41 +00:00
Christian Schwarz
0a353051ac run the test that was allegedly fixed by that check a 100 times 2025-07-24 12:08:30 +00:00
Christian Schwarz
e56c46c2f8 I think this check makes no sense 2025-07-24 12:08:00 +00:00
47 changed files with 540 additions and 3013 deletions

View File

@@ -1,384 +0,0 @@
name: TPC-C like benchmark using benchbase
on:
schedule:
# * is a special character in YAML so you have to quote this string
# ┌───────────── minute (0 - 59)
# │ ┌───────────── hour (0 - 23)
# │ │ ┌───────────── day of the month (1 - 31)
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
- cron: '0 6 * * *' # run once a day at 6 AM UTC
workflow_dispatch: # adds ability to run this manually
defaults:
run:
shell: bash -euxo pipefail {0}
concurrency:
# Allow only one workflow globally because we do not want to be too noisy in production environment
group: benchbase-tpcc-workflow
cancel-in-progress: false
permissions:
contents: read
jobs:
benchbase-tpcc:
strategy:
fail-fast: false # allow other variants to continue even if one fails
matrix:
include:
- warehouses: 50 # defines number of warehouses and is used to compute number of terminals
max_rate: 800 # measured max TPS at scale factor based on experiments. Adjust if performance is better/worse
min_cu: 0.25 # simulate free tier plan (0.25 -2 CU)
max_cu: 2
- warehouses: 500 # serverless plan (2-8 CU)
max_rate: 2000
min_cu: 2
max_cu: 8
- warehouses: 1000 # business plan (2-16 CU)
max_rate: 2900
min_cu: 2
max_cu: 16
max-parallel: 1 # we want to run each workload size sequentially to avoid noisy neighbors
permissions:
contents: write
statuses: write
id-token: write # aws-actions/configure-aws-credentials
env:
PG_CONFIG: /tmp/neon/pg_install/v17/bin/pg_config
PSQL: /tmp/neon/pg_install/v17/bin/psql
PG_17_LIB_PATH: /tmp/neon/pg_install/v17/lib
POSTGRES_VERSION: 17
runs-on: [ self-hosted, us-east-2, x64 ]
timeout-minutes: 1440
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Configure AWS credentials # necessary to download artefacts
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-region: eu-central-1
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
role-duration-seconds: 18000 # 5 hours is currently max associated with IAM role
- name: Download Neon artifact
uses: ./.github/actions/download
with:
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
path: /tmp/neon/
prefix: latest
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
- name: Create Neon Project
id: create-neon-project-tpcc
uses: ./.github/actions/neon-project-create
with:
region_id: aws-us-east-2
postgres_version: ${{ env.POSTGRES_VERSION }}
compute_units: '[${{ matrix.min_cu }}, ${{ matrix.max_cu }}]'
api_key: ${{ secrets.NEON_PRODUCTION_API_KEY_4_BENCHMARKS }}
api_host: console.neon.tech # production (!)
- name: Initialize Neon project
env:
BENCHMARK_TPCC_CONNSTR: ${{ steps.create-neon-project-tpcc.outputs.dsn }}
PROJECT_ID: ${{ steps.create-neon-project-tpcc.outputs.project_id }}
run: |
echo "Initializing Neon project with project_id: ${PROJECT_ID}"
export LD_LIBRARY_PATH=${PG_17_LIB_PATH}
# Retry logic for psql connection with 1 minute sleep between attempts
for attempt in {1..3}; do
echo "Attempt ${attempt}/3: Creating extensions in Neon project"
if ${PSQL} "${BENCHMARK_TPCC_CONNSTR}" -c "CREATE EXTENSION IF NOT EXISTS neon; CREATE EXTENSION IF NOT EXISTS neon_utils;"; then
echo "Successfully created extensions"
break
else
echo "Failed to create extensions on attempt ${attempt}"
if [ ${attempt} -lt 3 ]; then
echo "Waiting 60 seconds before retry..."
sleep 60
else
echo "All attempts failed, exiting"
exit 1
fi
fi
done
echo "BENCHMARK_TPCC_CONNSTR=${BENCHMARK_TPCC_CONNSTR}" >> $GITHUB_ENV
- name: Generate BenchBase workload configuration
env:
WAREHOUSES: ${{ matrix.warehouses }}
MAX_RATE: ${{ matrix.max_rate }}
run: |
echo "Generating BenchBase configs for warehouses: ${WAREHOUSES}, max_rate: ${MAX_RATE}"
# Extract hostname and password from connection string
# Format: postgresql://username:password@hostname/database?params (no port for Neon)
HOSTNAME=$(echo "${BENCHMARK_TPCC_CONNSTR}" | sed -n 's|.*://[^:]*:[^@]*@\([^/]*\)/.*|\1|p')
PASSWORD=$(echo "${BENCHMARK_TPCC_CONNSTR}" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p')
echo "Extracted hostname: ${HOSTNAME}"
# Use runner temp (NVMe) as working directory
cd "${RUNNER_TEMP}"
# Copy the generator script
cp "${GITHUB_WORKSPACE}/test_runner/performance/benchbase_tpc_c_helpers/generate_workload_size.py" .
# Generate configs and scripts
python3 generate_workload_size.py \
--warehouses ${WAREHOUSES} \
--max-rate ${MAX_RATE} \
--hostname ${HOSTNAME} \
--password ${PASSWORD} \
--runner-arch ${{ runner.arch }}
# Fix path mismatch: move generated configs and scripts to expected locations
mv ../configs ./configs
mv ../scripts ./scripts
- name: Prepare database (load data)
env:
WAREHOUSES: ${{ matrix.warehouses }}
run: |
cd "${RUNNER_TEMP}"
echo "Loading ${WAREHOUSES} warehouses into database..."
# Run the loader script and capture output to log file while preserving stdout/stderr
./scripts/load_${WAREHOUSES}_warehouses.sh 2>&1 | tee "load_${WAREHOUSES}_warehouses.log"
echo "Database loading completed"
- name: Run TPC-C benchmark (warmup phase, then benchmark at 70% of configuredmax TPS)
env:
WAREHOUSES: ${{ matrix.warehouses }}
run: |
cd "${RUNNER_TEMP}"
echo "Running TPC-C benchmark with ${WAREHOUSES} warehouses..."
# Run the optimal rate benchmark
./scripts/execute_${WAREHOUSES}_warehouses_opt_rate.sh
echo "Benchmark execution completed"
- name: Run TPC-C benchmark (warmup phase, then ramp down TPS and up again in 5 minute intervals)
env:
WAREHOUSES: ${{ matrix.warehouses }}
run: |
cd "${RUNNER_TEMP}"
echo "Running TPC-C ramp-down-up with ${WAREHOUSES} warehouses..."
# Run the optimal rate benchmark
./scripts/execute_${WAREHOUSES}_warehouses_ramp_up.sh
echo "Benchmark execution completed"
- name: Process results (upload to test results database and generate diagrams)
env:
WAREHOUSES: ${{ matrix.warehouses }}
MIN_CU: ${{ matrix.min_cu }}
MAX_CU: ${{ matrix.max_cu }}
PROJECT_ID: ${{ steps.create-neon-project-tpcc.outputs.project_id }}
REVISION: ${{ github.sha }}
PERF_DB_CONNSTR: ${{ secrets.PERF_TEST_RESULT_CONNSTR }}
run: |
cd "${RUNNER_TEMP}"
echo "Creating temporary Python environment for results processing..."
# Create temporary virtual environment
python3 -m venv temp_results_env
source temp_results_env/bin/activate
# Install required packages in virtual environment
pip install matplotlib pandas psycopg2-binary
echo "Copying results processing scripts..."
# Copy both processing scripts
cp "${GITHUB_WORKSPACE}/test_runner/performance/benchbase_tpc_c_helpers/generate_diagrams.py" .
cp "${GITHUB_WORKSPACE}/test_runner/performance/benchbase_tpc_c_helpers/upload_results_to_perf_test_results.py" .
echo "Processing load phase metrics..."
# Find and process load log
LOAD_LOG=$(find . -name "load_${WAREHOUSES}_warehouses.log" -type f | head -1)
if [ -n "$LOAD_LOG" ]; then
echo "Processing load metrics from: $LOAD_LOG"
python upload_results_to_perf_test_results.py \
--load-log "$LOAD_LOG" \
--run-type "load" \
--warehouses "${WAREHOUSES}" \
--min-cu "${MIN_CU}" \
--max-cu "${MAX_CU}" \
--project-id "${PROJECT_ID}" \
--revision "${REVISION}" \
--connection-string "${PERF_DB_CONNSTR}"
else
echo "Warning: Load log file not found: load_${WAREHOUSES}_warehouses.log"
fi
echo "Processing warmup results for optimal rate..."
# Find and process warmup results
WARMUP_CSV=$(find results_warmup -name "*.results.csv" -type f | head -1)
WARMUP_JSON=$(find results_warmup -name "*.summary.json" -type f | head -1)
if [ -n "$WARMUP_CSV" ] && [ -n "$WARMUP_JSON" ]; then
echo "Generating warmup diagram from: $WARMUP_CSV"
python generate_diagrams.py \
--input-csv "$WARMUP_CSV" \
--output-svg "warmup_${WAREHOUSES}_warehouses_performance.svg" \
--title-suffix "Warmup at max TPS"
echo "Uploading warmup metrics from: $WARMUP_JSON"
python upload_results_to_perf_test_results.py \
--summary-json "$WARMUP_JSON" \
--results-csv "$WARMUP_CSV" \
--run-type "warmup" \
--min-cu "${MIN_CU}" \
--max-cu "${MAX_CU}" \
--project-id "${PROJECT_ID}" \
--revision "${REVISION}" \
--connection-string "${PERF_DB_CONNSTR}"
else
echo "Warning: Missing warmup results files (CSV: $WARMUP_CSV, JSON: $WARMUP_JSON)"
fi
echo "Processing optimal rate results..."
# Find and process optimal rate results
OPTRATE_CSV=$(find results_opt_rate -name "*.results.csv" -type f | head -1)
OPTRATE_JSON=$(find results_opt_rate -name "*.summary.json" -type f | head -1)
if [ -n "$OPTRATE_CSV" ] && [ -n "$OPTRATE_JSON" ]; then
echo "Generating optimal rate diagram from: $OPTRATE_CSV"
python generate_diagrams.py \
--input-csv "$OPTRATE_CSV" \
--output-svg "benchmark_${WAREHOUSES}_warehouses_performance.svg" \
--title-suffix "70% of max TPS"
echo "Uploading optimal rate metrics from: $OPTRATE_JSON"
python upload_results_to_perf_test_results.py \
--summary-json "$OPTRATE_JSON" \
--results-csv "$OPTRATE_CSV" \
--run-type "opt-rate" \
--min-cu "${MIN_CU}" \
--max-cu "${MAX_CU}" \
--project-id "${PROJECT_ID}" \
--revision "${REVISION}" \
--connection-string "${PERF_DB_CONNSTR}"
else
echo "Warning: Missing optimal rate results files (CSV: $OPTRATE_CSV, JSON: $OPTRATE_JSON)"
fi
echo "Processing warmup 2 results for ramp down/up phase..."
# Find and process warmup results
WARMUP_CSV=$(find results_warmup -name "*.results.csv" -type f | tail -1)
WARMUP_JSON=$(find results_warmup -name "*.summary.json" -type f | tail -1)
if [ -n "$WARMUP_CSV" ] && [ -n "$WARMUP_JSON" ]; then
echo "Generating warmup diagram from: $WARMUP_CSV"
python generate_diagrams.py \
--input-csv "$WARMUP_CSV" \
--output-svg "warmup_2_${WAREHOUSES}_warehouses_performance.svg" \
--title-suffix "Warmup at max TPS"
echo "Uploading warmup metrics from: $WARMUP_JSON"
python upload_results_to_perf_test_results.py \
--summary-json "$WARMUP_JSON" \
--results-csv "$WARMUP_CSV" \
--run-type "warmup" \
--min-cu "${MIN_CU}" \
--max-cu "${MAX_CU}" \
--project-id "${PROJECT_ID}" \
--revision "${REVISION}" \
--connection-string "${PERF_DB_CONNSTR}"
else
echo "Warning: Missing warmup results files (CSV: $WARMUP_CSV, JSON: $WARMUP_JSON)"
fi
echo "Processing ramp results..."
# Find and process ramp results
RAMPUP_CSV=$(find results_ramp_up -name "*.results.csv" -type f | head -1)
RAMPUP_JSON=$(find results_ramp_up -name "*.summary.json" -type f | head -1)
if [ -n "$RAMPUP_CSV" ] && [ -n "$RAMPUP_JSON" ]; then
echo "Generating ramp diagram from: $RAMPUP_CSV"
python generate_diagrams.py \
--input-csv "$RAMPUP_CSV" \
--output-svg "ramp_${WAREHOUSES}_warehouses_performance.svg" \
--title-suffix "ramp TPS down and up in 5 minute intervals"
echo "Uploading ramp metrics from: $RAMPUP_JSON"
python upload_results_to_perf_test_results.py \
--summary-json "$RAMPUP_JSON" \
--results-csv "$RAMPUP_CSV" \
--run-type "ramp-up" \
--min-cu "${MIN_CU}" \
--max-cu "${MAX_CU}" \
--project-id "${PROJECT_ID}" \
--revision "${REVISION}" \
--connection-string "${PERF_DB_CONNSTR}"
else
echo "Warning: Missing ramp results files (CSV: $RAMPUP_CSV, JSON: $RAMPUP_JSON)"
fi
# Deactivate and clean up virtual environment
deactivate
rm -rf temp_results_env
rm upload_results_to_perf_test_results.py
echo "Results processing completed and environment cleaned up"
- name: Set date for upload
id: set-date
run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT
- name: Configure AWS credentials # necessary to upload results
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-region: us-east-2
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
role-duration-seconds: 900 # 900 is minimum value
- name: Upload benchmark results to S3
env:
S3_BUCKET: neon-public-benchmark-results
S3_PREFIX: benchbase-tpc-c/${{ steps.set-date.outputs.date }}/${{ github.run_id }}/${{ matrix.warehouses }}-warehouses
run: |
echo "Redacting passwords from configuration files before upload..."
# Mask all passwords in XML config files
find "${RUNNER_TEMP}/configs" -name "*.xml" -type f -exec sed -i 's|<password>[^<]*</password>|<password>redacted</password>|g' {} \;
echo "Uploading benchmark results to s3://${S3_BUCKET}/${S3_PREFIX}/"
# Upload the entire benchmark directory recursively
aws s3 cp --only-show-errors --recursive "${RUNNER_TEMP}" s3://${S3_BUCKET}/${S3_PREFIX}/
echo "Upload completed"
- name: Delete Neon Project
if: ${{ always() }}
uses: ./.github/actions/neon-project-delete
with:
project_id: ${{ steps.create-neon-project-tpcc.outputs.project_id }}
api_key: ${{ secrets.NEON_PRODUCTION_API_KEY_4_BENCHMARKS }}
api_host: console.neon.tech # production (!)

View File

@@ -26,13 +26,7 @@ commands:
- name: postgres-exporter
user: nobody
sysvInitAction: respawn
# Turn off database collector (`--no-collector.database`), we don't use `pg_database_size_bytes` metric anyway, see
# https://github.com/neondatabase/flux-fleet/blob/5e19b3fd897667b70d9a7ad4aa06df0ca22b49ff/apps/base/compute-metrics/scrape-compute-pg-exporter-neon.yaml#L29
# but it's enabled by default and it doesn't filter out invalid databases, see
# https://github.com/prometheus-community/postgres_exporter/blob/06a553c8166512c9d9c5ccf257b0f9bba8751dbc/collector/pg_database.go#L67
# so if it hits one, it starts spamming logs
# ERROR: [NEON_SMGR] [reqid d9700000018] could not read db size of db 705302 from page server at lsn 5/A2457EB0
shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --no-collector.database --config.file=/etc/postgres_exporter.yml'
shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --config.file=/etc/postgres_exporter.yml'
- name: pgbouncer-exporter
user: postgres
sysvInitAction: respawn

View File

@@ -26,13 +26,7 @@ commands:
- name: postgres-exporter
user: nobody
sysvInitAction: respawn
# Turn off database collector (`--no-collector.database`), we don't use `pg_database_size_bytes` metric anyway, see
# https://github.com/neondatabase/flux-fleet/blob/5e19b3fd897667b70d9a7ad4aa06df0ca22b49ff/apps/base/compute-metrics/scrape-compute-pg-exporter-neon.yaml#L29
# but it's enabled by default and it doesn't filter out invalid databases, see
# https://github.com/prometheus-community/postgres_exporter/blob/06a553c8166512c9d9c5ccf257b0f9bba8751dbc/collector/pg_database.go#L67
# so if it hits one, it starts spamming logs
# ERROR: [NEON_SMGR] [reqid d9700000018] could not read db size of db 705302 from page server at lsn 5/A2457EB0
shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --no-collector.database --config.file=/etc/postgres_exporter.yml'
shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --config.file=/etc/postgres_exporter.yml'
- name: pgbouncer-exporter
user: postgres
sysvInitAction: respawn

View File

@@ -52,14 +52,8 @@ stateDiagram-v2
Init --> Running : Started Postgres
Running --> TerminationPendingFast : Requested termination
Running --> TerminationPendingImmediate : Requested termination
Running --> ConfigurationPending : Received a /configure request with spec
Running --> RefreshConfigurationPending : Received a /refresh_configuration request, compute node will pull a new spec and reconfigure
RefreshConfigurationPending --> RefreshConfiguration: Received compute spec and started configuration
RefreshConfiguration --> Running : Compute has been re-configured
RefreshConfiguration --> RefreshConfigurationPending : Configuration failed and to be retried
TerminationPendingFast --> Terminated compute with 30s delay for cplane to inspect status
TerminationPendingImmediate --> Terminated : Terminated compute immediately
Failed --> RefreshConfigurationPending : Received a /refresh_configuration request
Failed --> [*] : Compute exited
Terminated --> [*] : Compute exited
```

View File

@@ -49,10 +49,10 @@ use compute_tools::compute::{
BUILD_TAG, ComputeNode, ComputeNodeParams, forward_termination_signal,
};
use compute_tools::extension_server::get_pg_version_string;
use compute_tools::logger::*;
use compute_tools::params::*;
use compute_tools::pg_isready::get_pg_isready_bin;
use compute_tools::spec::*;
use compute_tools::{hadron_metrics, installed_extensions, logger::*};
use rlimit::{Resource, setrlimit};
use signal_hook::consts::{SIGINT, SIGQUIT, SIGTERM};
use signal_hook::iterator::Signals;
@@ -205,9 +205,6 @@ fn main() -> Result<()> {
// enable core dumping for all child processes
setrlimit(Resource::CORE, rlimit::INFINITY, rlimit::INFINITY)?;
installed_extensions::initialize_metrics();
hadron_metrics::initialize_metrics();
let connstr = Url::parse(&cli.connstr).context("cannot parse connstr as a URL")?;
let config = get_config(&cli)?;
@@ -238,9 +235,6 @@ fn main() -> Result<()> {
pg_isready_bin: get_pg_isready_bin(&cli.pgbin),
instance_id: std::env::var("INSTANCE_ID").ok(),
lakebase_mode: cli.lakebase_mode,
build_tag: BUILD_TAG.to_string(),
control_plane_uri: cli.control_plane_uri,
config_path_test_only: cli.config,
},
config,
)?;

View File

@@ -21,7 +21,6 @@ use postgres::NoTls;
use postgres::error::SqlState;
use remote_storage::{DownloadError, RemotePath};
use std::collections::{HashMap, HashSet};
use std::ffi::OsString;
use std::os::unix::fs::{PermissionsExt, symlink};
use std::path::Path;
use std::process::{Command, Stdio};
@@ -41,9 +40,8 @@ use utils::shard::{ShardCount, ShardIndex, ShardNumber};
use crate::configurator::launch_configurator;
use crate::disk_quota::set_disk_quota;
use crate::hadron_metrics::COMPUTE_ATTACHED;
use crate::installed_extensions::get_installed_extensions;
use crate::logger::{self, startup_context_from_env};
use crate::logger::startup_context_from_env;
use crate::lsn_lease::launch_lsn_lease_bg_task_for_static;
use crate::metrics::COMPUTE_CTL_UP;
use crate::monitor::launch_monitor;
@@ -122,10 +120,6 @@ pub struct ComputeNodeParams {
// Path to the `pg_isready` binary.
pub pg_isready_bin: String,
pub lakebase_mode: bool,
pub build_tag: String,
pub control_plane_uri: Option<String>,
pub config_path_test_only: Option<OsString>,
}
type TaskHandle = Mutex<Option<JoinHandle<()>>>;
@@ -1802,12 +1796,12 @@ impl ComputeNode {
let states_allowing_configuration_refresh = [
ComputeStatus::Running,
ComputeStatus::Failed,
ComputeStatus::RefreshConfigurationPending,
// ComputeStatus::RefreshConfigurationPending,
];
let mut state = self.state.lock().expect("state lock poisoned");
let state = self.state.lock().expect("state lock poisoned");
if states_allowing_configuration_refresh.contains(&state.status) {
state.status = ComputeStatus::RefreshConfigurationPending;
// state.status = ComputeStatus::RefreshConfigurationPending;
self.state_changed.notify_all();
Ok(())
} else if state.status == ComputeStatus::Init {
@@ -1994,8 +1988,6 @@ impl ComputeNode {
// wait
ComputeStatus::Init
| ComputeStatus::Configuration
| ComputeStatus::RefreshConfiguration
| ComputeStatus::RefreshConfigurationPending
| ComputeStatus::Empty => {
state = self.state_changed.wait(state).unwrap();
}
@@ -2552,34 +2544,6 @@ LIMIT 100",
);
}
}
/// Set the compute spec and update related metrics.
/// This is the central place where pspec is updated.
pub fn set_spec(params: &ComputeNodeParams, state: &mut ComputeState, pspec: ParsedSpec) {
state.pspec = Some(pspec);
ComputeNode::update_attached_metric(params, state);
let _ = logger::update_ids(&params.instance_id, &Some(params.compute_id.clone()));
}
pub fn update_attached_metric(params: &ComputeNodeParams, state: &mut ComputeState) {
// Update the pg_cctl_attached gauge when all identifiers are available.
if let Some(instance_id) = &params.instance_id {
if let Some(pspec) = &state.pspec {
// Clear all values in the metric
COMPUTE_ATTACHED.reset();
// Set new metric value
COMPUTE_ATTACHED
.with_label_values(&[
&params.compute_id,
instance_id,
&pspec.tenant_id.to_string(),
&pspec.timeline_id.to_string(),
])
.set(1);
}
}
}
}
pub async fn installed_extensions(conf: tokio_postgres::Config) -> Result<()> {

View File

@@ -1,40 +1,23 @@
use std::fs::File;
use std::sync::Arc;
use std::thread;
use std::{path::Path, sync::Arc};
use anyhow::Result;
use compute_api::responses::{ComputeConfig, ComputeStatus};
use compute_api::responses::ComputeStatus;
use tracing::{error, info, instrument};
use crate::compute::{ComputeNode, ParsedSpec};
use crate::spec::get_config_from_control_plane;
use crate::compute::ComputeNode;
#[instrument(skip_all)]
fn configurator_main_loop(compute: &Arc<ComputeNode>) {
info!("waiting for reconfiguration requests");
loop {
let mut state = compute.state.lock().unwrap();
/* BEGIN_HADRON */
// RefreshConfiguration should only be used inside the loop
assert_ne!(state.status, ComputeStatus::RefreshConfiguration);
/* END_HADRON */
if compute.params.lakebase_mode {
while state.status != ComputeStatus::ConfigurationPending
&& state.status != ComputeStatus::RefreshConfigurationPending
&& state.status != ComputeStatus::Failed
{
info!("configurator: compute status: {:?}, sleeping", state.status);
state = compute.state_changed.wait(state).unwrap();
}
} else {
// We have to re-check the status after re-acquiring the lock because it could be that
// the status has changed while we were waiting for the lock, and we might not need to
// wait on the condition variable. Otherwise, we might end up in some soft-/deadlock, i.e.
// we are waiting for a condition variable that will never be signaled.
if state.status != ComputeStatus::ConfigurationPending {
state = compute.state_changed.wait(state).unwrap();
}
// We have to re-check the status after re-acquiring the lock because it could be that
// the status has changed while we were waiting for the lock, and we might not need to
// wait on the condition variable. Otherwise, we might end up in some soft-/deadlock, i.e.
// we are waiting for a condition variable that will never be signaled.
if state.status != ComputeStatus::ConfigurationPending {
state = compute.state_changed.wait(state).unwrap();
}
// Re-check the status after waking up
@@ -54,133 +37,6 @@ fn configurator_main_loop(compute: &Arc<ComputeNode>) {
// XXX: used to test that API is blocking
// std::thread::sleep(std::time::Duration::from_millis(10000));
compute.set_status(new_status);
} else if state.status == ComputeStatus::RefreshConfigurationPending {
info!(
"compute node suspects its configuration is out of date, now refreshing configuration"
);
state.set_status(ComputeStatus::RefreshConfiguration, &compute.state_changed);
// Drop the lock guard here to avoid holding the lock while downloading config from the control plane / HCC.
// This is the only thread that can move compute_ctl out of the `RefreshConfiguration` state, so it
// is safe to drop the lock like this.
drop(state);
let get_config_result: anyhow::Result<ComputeConfig> =
if let Some(config_path) = &compute.params.config_path_test_only {
// This path is only to make testing easier. In production we always get the config from the HCC.
info!(
"reloading config.json from path: {}",
config_path.to_string_lossy()
);
let path = Path::new(config_path);
if let Ok(file) = File::open(path) {
match serde_json::from_reader::<File, ComputeConfig>(file) {
Ok(config) => Ok(config),
Err(e) => {
error!("could not parse config file: {}", e);
Err(anyhow::anyhow!("could not parse config file: {}", e))
}
}
} else {
error!(
"could not open config file at path: {:?}",
config_path.to_string_lossy()
);
Err(anyhow::anyhow!(
"could not open config file at path: {}",
config_path.to_string_lossy()
))
}
} else if let Some(control_plane_uri) = &compute.params.control_plane_uri {
get_config_from_control_plane(control_plane_uri, &compute.params.compute_id)
} else {
Err(anyhow::anyhow!("config_path_test_only is not set"))
};
// Parse any received ComputeSpec and transpose the result into a Result<Option<ParsedSpec>>.
let parsed_spec_result: Result<Option<ParsedSpec>> =
get_config_result.and_then(|config| {
if let Some(spec) = config.spec {
if let Ok(pspec) = ParsedSpec::try_from(spec) {
Ok(Some(pspec))
} else {
Err(anyhow::anyhow!("could not parse spec"))
}
} else {
Ok(None)
}
});
let new_status: ComputeStatus;
match parsed_spec_result {
// Control plane (HCM) returned a spec and we were able to parse it.
Ok(Some(pspec)) => {
{
let mut state = compute.state.lock().unwrap();
// Defensive programming to make sure this thread is indeed the only one that can move the compute
// node out of the `RefreshConfiguration` state. Would be nice if we can encode this invariant
// into the type system.
assert_eq!(state.status, ComputeStatus::RefreshConfiguration);
if state.pspec.as_ref().map(|ps| ps.pageserver_connstr.clone())
== Some(pspec.pageserver_connstr.clone())
{
info!(
"Refresh configuration: Retrieved spec is the same as the current spec. Waiting for control plane to update the spec before attempting reconfiguration."
);
state.status = ComputeStatus::Running;
compute.state_changed.notify_all();
drop(state);
std::thread::sleep(std::time::Duration::from_secs(5));
continue;
}
// state.pspec is consumed by compute.reconfigure() below. Note that compute.reconfigure() will acquire
// the compute.state lock again so we need to have the lock guard go out of scope here. We could add a
// "locked" variant of compute.reconfigure() that takes the lock guard as an argument to make this cleaner,
// but it's not worth forking the codebase too much for this minor point alone right now.
state.pspec = Some(pspec);
}
match compute.reconfigure() {
Ok(_) => {
info!("Refresh configuration: compute node configured");
new_status = ComputeStatus::Running;
}
Err(e) => {
error!(
"Refresh configuration: could not configure compute node: {}",
e
);
// Set the compute node back to the `RefreshConfigurationPending` state if the configuration
// was not successful. It should be okay to treat this situation the same as if the loop
// hasn't executed yet as long as the detection side keeps notifying.
new_status = ComputeStatus::RefreshConfigurationPending;
}
}
}
// Control plane (HCM)'s response does not contain a spec. This is the "Empty" attachment case.
Ok(None) => {
info!(
"Compute Manager signaled that this compute is no longer attached to any storage. Exiting."
);
// We just immediately terminate the whole compute_ctl in this case. It's not necessary to attempt a
// clean shutdown as Postgres is probably not responding anyway (which is why we are in this refresh
// configuration state).
std::process::exit(1);
}
// Various error cases:
// - The request to the control plane (HCM) either failed or returned a malformed spec.
// - compute_ctl itself is configured incorrectly (e.g., compute_id is not set).
Err(e) => {
error!(
"Refresh configuration: error getting a parsed spec: {:?}",
e
);
new_status = ComputeStatus::RefreshConfigurationPending;
// We may be dealing with an overloaded HCM if we end up in this path. Backoff 5 seconds before
// retrying to avoid hammering the HCM.
std::thread::sleep(std::time::Duration::from_secs(5));
}
}
compute.set_status(new_status);
} else if state.status == ComputeStatus::Failed {
info!("compute node is now in Failed state, exiting");

View File

@@ -43,12 +43,7 @@ pub(in crate::http) async fn configure(
// configure request for tracing purposes.
state.startup_span = Some(tracing::Span::current());
if compute.params.lakebase_mode {
ComputeNode::set_spec(&compute.params, &mut state, pspec);
} else {
state.pspec = Some(pspec);
}
state.pspec = Some(pspec);
state.set_status(ComputeStatus::ConfigurationPending, &compute.state_changed);
drop(state);
}

View File

@@ -13,7 +13,6 @@ use metrics::{Encoder, TextEncoder};
use crate::communicator_socket_client::connect_communicator_socket;
use crate::compute::ComputeNode;
use crate::hadron_metrics;
use crate::http::JsonResponse;
use crate::metrics::collect;
@@ -22,18 +21,11 @@ pub(in crate::http) async fn get_metrics() -> Response {
// When we call TextEncoder::encode() below, it will immediately return an
// error if a metric family has no metrics, so we need to preemptively
// filter out metric families with no metrics.
let mut metrics = collect()
let metrics = collect()
.into_iter()
.filter(|m| !m.get_metric().is_empty())
.collect::<Vec<MetricFamily>>();
// Add Hadron metrics.
let hadron_metrics: Vec<MetricFamily> = hadron_metrics::collect()
.into_iter()
.filter(|m| !m.get_metric().is_empty())
.collect();
metrics.extend(hadron_metrics);
let encoder = TextEncoder::new();
let mut buffer = vec![];

View File

@@ -7,23 +7,28 @@ use axum::{
response::{IntoResponse, Response},
};
use http::StatusCode;
use tracing::debug;
use crate::compute::ComputeNode;
use crate::hadron_metrics::POSTGRES_PAGESTREAM_REQUEST_ERRORS;
// use crate::hadron_metrics::POSTGRES_PAGESTREAM_REQUEST_ERRORS;
use crate::http::JsonResponse;
/// The /refresh_configuration POST method is used to nudge compute_ctl to pull a new spec
/// from the HCC and attempt to reconfigure Postgres with the new spec. The method does not wait
/// for the reconfiguration to complete. Rather, it simply delivers a signal that will cause
/// configuration to be reloaded in a best effort manner. Invocation of this method does not
/// guarantee that a reconfiguration will occur. The caller should consider keep sending this
/// request while it believes that the compute configuration is out of date.
// The /refresh_configuration POST method is used to nudge compute_ctl to pull a new spec
// from the HCC and attempt to reconfigure Postgres with the new spec. The method does not wait
// for the reconfiguration to complete. Rather, it simply delivers a signal that will cause
// configuration to be reloaded in a best effort manner. Invocation of this method does not
// guarantee that a reconfiguration will occur. The caller should consider keep sending this
// request while it believes that the compute configuration is out of date.
pub(in crate::http) async fn refresh_configuration(
State(compute): State<Arc<ComputeNode>>,
) -> Response {
POSTGRES_PAGESTREAM_REQUEST_ERRORS.inc();
debug!("serving /refresh_configuration POST request");
// POSTGRES_PAGESTREAM_REQUEST_ERRORS.inc();
match compute.signal_refresh_configuration().await {
Ok(_) => StatusCode::OK.into_response(),
Err(e) => JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e),
Err(e) => {
tracing::error!("error handling /refresh_configuration request: {}", e);
JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e)
}
}
}

View File

@@ -1,7 +1,7 @@
use crate::compute::{ComputeNode, forward_termination_signal};
use crate::http::JsonResponse;
use axum::extract::State;
use axum::response::{IntoResponse, Response};
use axum::response::Response;
use axum_extra::extract::OptionalQuery;
use compute_api::responses::{ComputeStatus, TerminateMode, TerminateResponse};
use http::StatusCode;
@@ -33,29 +33,7 @@ pub(in crate::http) async fn terminate(
if !matches!(state.status, ComputeStatus::Empty | ComputeStatus::Running) {
return JsonResponse::invalid_status(state.status);
}
// If compute is Empty, there's no Postgres to terminate. The regular compute_ctl termination path
// assumes Postgres to be configured and running, so we just special-handle this case by exiting
// the process directly.
if compute.params.lakebase_mode && state.status == ComputeStatus::Empty {
drop(state);
info!("terminating empty compute - will exit process");
// Queue a task to exit the process after 5 seconds. The 5-second delay aims to
// give enough time for the HTTP response to be sent so that HCM doesn't get an abrupt
// connection termination.
tokio::spawn(async {
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
info!("exiting process after terminating empty compute");
std::process::exit(0);
});
return StatusCode::OK.into_response();
}
// For Running status, proceed with normal termination
state.set_status(mode.into(), &compute.state_changed);
drop(state);
}
forward_termination_signal(false);

View File

@@ -23,11 +23,11 @@ use super::{
middleware::authorize::Authorize,
routes::{
check_writability, configure, database_schema, dbs_and_roles, extension_server, extensions,
grants, hadron_liveness_probe, insights, lfc, metrics, metrics_json, promote,
refresh_configuration, status, terminate,
grants, insights, lfc, metrics, metrics_json, promote, status, terminate,
},
};
use crate::compute::ComputeNode;
use crate::http::routes::{hadron_liveness_probe, refresh_configuration};
/// `compute_ctl` has two servers: internal and external. The internal server
/// binds to the loopback interface and handles communication from clients on

View File

@@ -560,9 +560,7 @@ enum EndpointCmd {
Create(EndpointCreateCmdArgs),
Start(EndpointStartCmdArgs),
Reconfigure(EndpointReconfigureCmdArgs),
RefreshConfiguration(EndpointRefreshConfigurationArgs),
Stop(EndpointStopCmdArgs),
UpdatePageservers(EndpointUpdatePageserversCmdArgs),
GenerateJwt(EndpointGenerateJwtCmdArgs),
}
@@ -723,13 +721,6 @@ struct EndpointReconfigureCmdArgs {
safekeepers: Option<String>,
}
#[derive(clap::Args)]
#[clap(about = "Refresh the endpoint's configuration by forcing it reload it's spec")]
struct EndpointRefreshConfigurationArgs {
#[clap(help = "Postgres endpoint id")]
endpoint_id: String,
}
#[derive(clap::Args)]
#[clap(about = "Stop an endpoint")]
struct EndpointStopCmdArgs {
@@ -747,16 +738,6 @@ struct EndpointStopCmdArgs {
mode: EndpointTerminateMode,
}
#[derive(clap::Args)]
#[clap(about = "Update the pageservers in the spec file of the compute endpoint")]
struct EndpointUpdatePageserversCmdArgs {
#[clap(help = "Postgres endpoint id")]
endpoint_id: String,
#[clap(short = 'p', long, help = "Specified pageserver id")]
pageserver_id: Option<NodeId>,
}
#[derive(clap::Args)]
#[clap(about = "Generate a JWT for an endpoint")]
struct EndpointGenerateJwtCmdArgs {
@@ -1644,44 +1625,6 @@ async fn handle_endpoint(subcmd: &EndpointCmd, env: &local_env::LocalEnv) -> Res
println!("Starting existing endpoint {endpoint_id}...");
endpoint.start(args).await?;
}
EndpointCmd::UpdatePageservers(args) => {
let endpoint_id = &args.endpoint_id;
let endpoint = cplane
.endpoints
.get(endpoint_id.as_str())
.with_context(|| format!("postgres endpoint {endpoint_id} is not found"))?;
let pageservers = match args.pageserver_id {
Some(pageserver_id) => {
let pageserver =
PageServerNode::from_env(env, env.get_pageserver_conf(pageserver_id)?);
vec![(
PageserverProtocol::Libpq,
pageserver.pg_connection_config.host().clone(),
pageserver.pg_connection_config.port(),
)]
}
None => {
let storage_controller = StorageController::from_env(env);
storage_controller
.tenant_locate(endpoint.tenant_id)
.await?
.shards
.into_iter()
.map(|shard| {
(
PageserverProtocol::Libpq,
Host::parse(&shard.listen_pg_addr)
.expect("Storage controller reported malformed host"),
shard.listen_pg_port,
)
})
.collect::<Vec<_>>()
}
};
endpoint.update_pageservers_in_config(pageservers).await?;
}
EndpointCmd::Reconfigure(args) => {
let endpoint_id = &args.endpoint_id;
let endpoint = cplane
@@ -1735,14 +1678,6 @@ async fn handle_endpoint(subcmd: &EndpointCmd, env: &local_env::LocalEnv) -> Res
.reconfigure(Some(pageservers), None, safekeepers, None)
.await?;
}
EndpointCmd::RefreshConfiguration(args) => {
let endpoint_id = &args.endpoint_id;
let endpoint = cplane
.endpoints
.get(endpoint_id.as_str())
.with_context(|| format!("postgres endpoint {endpoint_id} is not found"))?;
endpoint.refresh_configuration().await?;
}
EndpointCmd::Stop(args) => {
let endpoint_id = &args.endpoint_id;
let endpoint = cplane

View File

@@ -937,9 +937,7 @@ impl Endpoint {
| ComputeStatus::Configuration
| ComputeStatus::TerminationPendingFast
| ComputeStatus::TerminationPendingImmediate
| ComputeStatus::Terminated
| ComputeStatus::RefreshConfigurationPending
| ComputeStatus::RefreshConfiguration => {
| ComputeStatus::Terminated => {
bail!("unexpected compute status: {:?}", state.status)
}
}
@@ -962,29 +960,6 @@ impl Endpoint {
Ok(())
}
// Update the pageservers in the spec file of the endpoint. This is useful to test the spec refresh scenario.
pub async fn update_pageservers_in_config(
&self,
pageservers: Vec<(PageserverProtocol, Host, u16)>,
) -> Result<()> {
let config_path = self.endpoint_path().join("config.json");
let mut config: ComputeConfig = {
let file = std::fs::File::open(&config_path)?;
serde_json::from_reader(file)?
};
let pageserver_connstring = Self::build_pageserver_connstr(&pageservers);
assert!(!pageserver_connstring.is_empty());
let mut spec = config.spec.unwrap();
spec.pageserver_connstring = Some(pageserver_connstring);
config.spec = Some(spec);
let file = std::fs::File::create(&config_path)?;
serde_json::to_writer_pretty(file, &config)?;
Ok(())
}
// Call the /status HTTP API
pub async fn get_status(&self) -> Result<ComputeStatusResponse> {
let client = reqwest::Client::new();
@@ -1150,33 +1125,6 @@ impl Endpoint {
Ok(response)
}
pub async fn refresh_configuration(&self) -> Result<()> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.unwrap();
let response = client
.post(format!(
"http://{}:{}/refresh_configuration",
self.internal_http_address.ip(),
self.internal_http_address.port()
))
.send()
.await?;
let status = response.status();
if !(status.is_client_error() || status.is_server_error()) {
Ok(())
} else {
let url = response.url().to_owned();
let msg = match response.text().await {
Ok(err_body) => format!("Error: {err_body}"),
Err(_) => format!("Http error ({}) at {}.", status.as_u16(), url),
};
Err(anyhow::anyhow!(msg))
}
}
pub fn connstr(&self, user: &str, db_name: &str) -> String {
format!(
"postgresql://{}@{}:{}/{}",

View File

@@ -172,11 +172,6 @@ pub enum ComputeStatus {
TerminationPendingImmediate,
// Terminated Postgres
Terminated,
// A spec refresh is being requested
RefreshConfigurationPending,
// A spec refresh is being applied. We cannot refresh configuration again until the current
// refresh is done, i.e., signal_refresh_configuration() will return 500 error.
RefreshConfiguration,
}
#[derive(Deserialize, Serialize)]
@@ -189,10 +184,6 @@ impl Display for ComputeStatus {
match self {
ComputeStatus::Empty => f.write_str("empty"),
ComputeStatus::ConfigurationPending => f.write_str("configuration-pending"),
ComputeStatus::RefreshConfiguration => f.write_str("refresh-configuration"),
ComputeStatus::RefreshConfigurationPending => {
f.write_str("refresh-configuration-pending")
}
ComputeStatus::Init => f.write_str("init"),
ComputeStatus::Running => f.write_str("running"),
ComputeStatus::Configuration => f.write_str("configuration"),

View File

@@ -7,7 +7,7 @@ use tokio::net::TcpStream;
use tokio::sync::mpsc;
use crate::client::SocketConfig;
use crate::config::{Host, SslMode};
use crate::config::Host;
use crate::connect_raw::StartupStream;
use crate::connect_socket::connect_socket;
use crate::tls::{MakeTlsConnect, TlsConnect};
@@ -45,36 +45,14 @@ where
T: TlsConnect<TcpStream>,
{
let socket = connect_socket(host_addr, host, port, config.connect_timeout).await?;
let stream = config.tls_and_authenticate(socket, tls).await?;
managed(
stream,
host_addr,
host.clone(),
port,
config.ssl_mode,
config.connect_timeout,
)
.await
}
pub async fn managed<TlsStream>(
mut stream: StartupStream<TcpStream, TlsStream>,
host_addr: Option<IpAddr>,
host: Host,
port: u16,
ssl_mode: SslMode,
connect_timeout: Option<std::time::Duration>,
) -> Result<(Client, Connection<TcpStream, TlsStream>), Error>
where
TlsStream: AsyncRead + AsyncWrite + Unpin,
{
let mut stream = config.tls_and_authenticate(socket, tls).await?;
let (process_id, secret_key) = wait_until_ready(&mut stream).await?;
let socket_config = SocketConfig {
host_addr,
host,
host: host.clone(),
port,
connect_timeout,
connect_timeout: config.connect_timeout,
};
let (client_tx, conn_rx) = mpsc::unbounded_channel();
@@ -83,7 +61,7 @@ where
client_tx,
client_rx,
socket_config,
ssl_mode,
config.ssl_mode,
process_id,
secret_key,
);

View File

@@ -48,7 +48,7 @@ mod cancel_token;
mod client;
mod codec;
pub mod config;
pub mod connect;
mod connect;
pub mod connect_raw;
mod connect_socket;
mod connect_tls;

View File

@@ -535,7 +535,6 @@ impl timeline::handle::TenantManager<TenantManagerTypes> for TenantManagerWrappe
match resolved {
ShardResolveResult::Found(tenant_shard) => break tenant_shard,
ShardResolveResult::NotFound => {
MISROUTED_PAGESTREAM_REQUESTS.inc();
return Err(GetActiveTimelineError::Tenant(
GetActiveTenantError::NotFound(GetTenantError::NotFound(*tenant_id)),
));
@@ -2194,7 +2193,7 @@ impl PageServerHandler {
//
// We may have older data available, but we make a best effort to detect this case and return an error,
// to distinguish a misbehaving client (asking for old LSN) from a storage issue (data missing at a legitimate LSN).
if request_lsn < **latest_gc_cutoff_lsn && !timeline.is_gc_blocked_by_lsn_lease_deadline() {
if request_lsn < **latest_gc_cutoff_lsn {
let gc_info = &timeline.gc_info.read().unwrap();
if !gc_info.lsn_covered_by_lease(request_lsn) {
return Err(

View File

@@ -2799,11 +2799,6 @@ impl Timeline {
.unwrap_or(self.conf.default_tenant_conf.lsn_lease_length_for_ts)
}
pub(crate) fn is_gc_blocked_by_lsn_lease_deadline(&self) -> bool {
let tenant_conf = self.tenant_conf.load();
tenant_conf.is_gc_blocked_by_lsn_lease_deadline()
}
pub(crate) fn get_lazy_slru_download(&self) -> bool {
let tenant_conf = self.tenant_conf.load();
tenant_conf

View File

@@ -33,10 +33,6 @@ SHLIB_LINK = -lcurl
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S), Darwin)
SHLIB_LINK += -framework Security -framework CoreFoundation -framework SystemConfiguration
# Link against object files for the current macOS version, to avoid spurious linker warnings.
MACOSX_DEPLOYMENT_TARGET := $(shell xcrun --sdk macosx --show-sdk-version)
export MACOSX_DEPLOYMENT_TARGET
endif
EXTENSION = neon

View File

@@ -14,7 +14,7 @@
#include "extension_server.h"
#include "neon_utils.h"
int hadron_extension_server_port = 0;
static int extension_server_port = 0;
static int extension_server_request_timeout = 60;
static int extension_server_connect_timeout = 60;
@@ -47,7 +47,7 @@ neon_download_extension_file_http(const char *filename, bool is_library)
curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, (long)extension_server_connect_timeout /* seconds */ );
compute_ctl_url = psprintf("http://localhost:%d/extension_server/%s%s",
hadron_extension_server_port, filename, is_library ? "?is_library=true" : "");
extension_server_port, filename, is_library ? "?is_library=true" : "");
elog(LOG, "Sending request to compute_ctl: %s", compute_ctl_url);
@@ -82,7 +82,7 @@ pg_init_extension_server()
DefineCustomIntVariable("neon.extension_server_port",
"connection string to the compute_ctl",
NULL,
&hadron_extension_server_port,
&extension_server_port,
0, 0, INT_MAX,
PGC_POSTMASTER,
0, /* no flags required */

View File

@@ -13,8 +13,6 @@
#include <math.h>
#include <sys/socket.h>
#include <curl/curl.h>
#include "libpq-int.h"
#include "access/xlog.h"
@@ -88,10 +86,6 @@ static int pageserver_response_log_timeout = 10000;
/* 2.5 minutes. A bit higher than highest default TCP retransmission timeout */
static int pageserver_response_disconnect_timeout = 150000;
static int conf_refresh_reconnect_attempt_threshold = 16;
// Hadron: timeout for refresh errors (1 minute)
static uint64 kRefreshErrorTimeoutUSec = 1 * USECS_PER_MINUTE;
typedef struct
{
char connstring[MAX_SHARDS][MAX_PAGESERVER_CONNSTRING_SIZE];
@@ -136,7 +130,7 @@ static uint64 pagestore_local_counter = 0;
typedef enum PSConnectionState {
PS_Disconnected, /* no connection yet */
PS_Connecting_Startup, /* connection starting up */
PS_Connecting_PageStream, /* negotiating pagestream */
PS_Connecting_PageStream, /* negotiating pagestream */
PS_Connected, /* connected, pagestream established */
} PSConnectionState;
@@ -407,7 +401,7 @@ get_shard_number(BufferTag *tag)
}
static inline void
CLEANUP_AND_DISCONNECT(PageServer *shard)
CLEANUP_AND_DISCONNECT(PageServer *shard)
{
if (shard->wes_read)
{
@@ -429,7 +423,7 @@ CLEANUP_AND_DISCONNECT(PageServer *shard)
* complete the connection (e.g. due to receiving an earlier cancellation
* during connection start).
* Returns true if successfully connected; false if the connection failed.
*
*
* Throws errors in unrecoverable situations, or when this backend's query
* is canceled.
*/
@@ -1036,101 +1030,6 @@ pageserver_disconnect_shard(shardno_t shard_no)
shard->state = PS_Disconnected;
}
// BEGIN HADRON
/*
* Nudge compute_ctl to refresh our configuration. Called when we suspect we may be
* connecting to the wrong pageservers due to a stale configuration.
*
* This is a best-effort operation. If we couldn't send the local loopback HTTP request
* to compute_ctl or if the request fails for any reason, we just log the error and move
* on.
*/
extern int hadron_extension_server_port;
// The timestamp (usec) of the first error that occurred while trying to refresh the configuration.
// Will be reset to 0 after a successful refresh.
static uint64 first_recorded_refresh_error_usec = 0;
// Request compute_ctl to refresh the configuration. This operation may fail, e.g., if the compute_ctl
// is already in the configuration state. The function returns true if the caller needs to cancel the
// current query to avoid dead/live lock.
static bool
hadron_request_configuration_refresh() {
static CURL *handle = NULL;
CURLcode res;
char *compute_ctl_url;
bool cancel_query = false;
if (!lakebase_mode)
return false;
if (handle == NULL)
{
handle = alloc_curl_handle();
curl_easy_setopt(handle, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(handle, CURLOPT_TIMEOUT, 3L /* seconds */ );
curl_easy_setopt(handle, CURLOPT_POSTFIELDS, "");
}
// Set the URL
compute_ctl_url = psprintf("http://localhost:%d/refresh_configuration", hadron_extension_server_port);
elog(LOG, "Sending refresh configuration request to compute_ctl: %s", compute_ctl_url);
curl_easy_setopt(handle, CURLOPT_URL, compute_ctl_url);
res = curl_easy_perform(handle);
if (res != CURLE_OK )
{
elog(WARNING, "refresh_configuration request failed: %s\n", curl_easy_strerror(res));
}
else
{
long http_code = 0;
curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_code);
if ( res != CURLE_OK )
{
elog(WARNING, "compute_ctl refresh_configuration request getinfo failed: %s\n", curl_easy_strerror(res));
}
else
{
elog(LOG, "compute_ctl refresh_configuration got HTTP response: %ld\n", http_code);
if( http_code == 200 )
{
first_recorded_refresh_error_usec = 0;
}
else
{
if (first_recorded_refresh_error_usec == 0)
{
first_recorded_refresh_error_usec = GetCurrentTimestamp();
}
else if(GetCurrentTimestamp() - first_recorded_refresh_error_usec > kRefreshErrorTimeoutUSec)
{
{
first_recorded_refresh_error_usec = 0;
cancel_query = true;
}
}
}
}
}
// In regular Postgres usage, it is not necessary to manually free memory allocated by palloc (psprintf) because
// it will be cleaned up after the "memory context" is reset (e.g. after the query or the transaction is finished).
// However, the number of times this function gets called during a single query/transaction can be unbounded due to
// the various retry loops around calls to pageservers. Therefore, we need to manually free this memory here.
if (compute_ctl_url != NULL)
{
pfree(compute_ctl_url);
}
return cancel_query;
}
// END HADRON
static bool
pageserver_send(shardno_t shard_no, NeonRequest *request)
{
@@ -1165,11 +1064,6 @@ pageserver_send(shardno_t shard_no, NeonRequest *request)
while (!pageserver_connect(shard_no, shard->n_reconnect_attempts < max_reconnect_attempts ? LOG : ERROR))
{
shard->n_reconnect_attempts += 1;
if (shard->n_reconnect_attempts > conf_refresh_reconnect_attempt_threshold
&& hadron_request_configuration_refresh() )
{
neon_shard_log(shard_no, ERROR, "request failed too many times, cancelling query");
}
}
shard->n_reconnect_attempts = 0;
} else {
@@ -1277,26 +1171,17 @@ pageserver_receive(shardno_t shard_no)
pfree(msg);
pageserver_disconnect(shard_no);
resp = NULL;
/*
* Always poke compute_ctl to request a configuration refresh if we have issues receiving data from pageservers after
* successfully connecting to it. It could be an indication that we are connecting to the wrong pageservers (e.g. PS
* is in secondary mode or otherwise refuses to respond our request).
*/
hadron_request_configuration_refresh();
}
else if (rc == -2)
{
char *msg = pchomp(PQerrorMessage(pageserver_conn));
pageserver_disconnect(shard_no);
hadron_request_configuration_refresh();
neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: could not read COPY data: %s", msg);
}
else
{
pageserver_disconnect(shard_no);
hadron_request_configuration_refresh();
neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: unexpected PQgetCopyData return value: %d", rc);
}
@@ -1364,34 +1249,21 @@ pageserver_try_receive(shardno_t shard_no)
neon_shard_log(shard_no, LOG, "pageserver_receive disconnect: psql end of copy data: %s", pchomp(PQerrorMessage(pageserver_conn)));
pageserver_disconnect(shard_no);
resp = NULL;
hadron_request_configuration_refresh();
}
else if (rc == -2)
{
char *msg = pchomp(PQerrorMessage(pageserver_conn));
pageserver_disconnect(shard_no);
hadron_request_configuration_refresh();
neon_shard_log(shard_no, LOG, "pageserver_receive disconnect: could not read COPY data: %s", msg);
resp = NULL;
}
else
{
pageserver_disconnect(shard_no);
hadron_request_configuration_refresh();
neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: unexpected PQgetCopyData return value: %d", rc);
}
/*
* Always poke compute_ctl to request a configuration refresh if we have issues receiving data from pageservers after
* successfully connecting to it. It could be an indication that we are connecting to the wrong pageservers (e.g. PS
* is in secondary mode or otherwise refuses to respond our request).
*/
if ( rc < 0 && hadron_request_configuration_refresh() )
{
neon_shard_log(shard_no, ERROR, "refresh_configuration request failed, cancelling query");
}
shard->nresponses_received++;
return (NeonResponse *) resp;
}
@@ -1588,16 +1460,6 @@ pg_init_libpagestore(void)
PGC_SU_BACKEND,
0, /* no flags required */
NULL, NULL, NULL);
DefineCustomIntVariable("hadron.conf_refresh_reconnect_attempt_threshold",
"Threshold of the number of consecutive failed pageserver "
"connection attempts (per shard) before signaling "
"compute_ctl for a configuration refresh.",
NULL,
&conf_refresh_reconnect_attempt_threshold,
16, 0, INT_MAX,
PGC_USERSET,
0,
NULL, NULL, NULL);
DefineCustomIntVariable("neon.pageserver_response_log_timeout",
"pageserver response log timeout",

View File

@@ -507,45 +507,19 @@ backpressure_lag_impl(void)
LSN_FORMAT_ARGS(flushPtr),
LSN_FORMAT_ARGS(applyPtr));
if (lakebase_mode)
if ((writePtr != InvalidXLogRecPtr && max_replication_write_lag > 0 && myFlushLsn > writePtr + max_replication_write_lag * MB))
{
// in case PG does not have shard map initialized, we assume PG always has 1 shard at minimum.
shardno_t num_shards = Max(1, get_num_shards());
int tenant_max_replication_apply_lag = num_shards * max_replication_apply_lag;
int tenant_max_replication_flush_lag = num_shards * max_replication_flush_lag;
int tenant_max_replication_write_lag = num_shards * max_replication_write_lag;
if ((writePtr != InvalidXLogRecPtr && tenant_max_replication_write_lag > 0 && myFlushLsn > writePtr + tenant_max_replication_write_lag * MB))
{
return (myFlushLsn - writePtr - tenant_max_replication_write_lag * MB);
}
if ((flushPtr != InvalidXLogRecPtr && tenant_max_replication_flush_lag > 0 && myFlushLsn > flushPtr + tenant_max_replication_flush_lag * MB))
{
return (myFlushLsn - flushPtr - tenant_max_replication_flush_lag * MB);
}
if ((applyPtr != InvalidXLogRecPtr && tenant_max_replication_apply_lag > 0 && myFlushLsn > applyPtr + tenant_max_replication_apply_lag * MB))
{
return (myFlushLsn - applyPtr - tenant_max_replication_apply_lag * MB);
}
return (myFlushLsn - writePtr - max_replication_write_lag * MB);
}
else
if ((flushPtr != InvalidXLogRecPtr && max_replication_flush_lag > 0 && myFlushLsn > flushPtr + max_replication_flush_lag * MB))
{
if ((writePtr != InvalidXLogRecPtr && max_replication_write_lag > 0 && myFlushLsn > writePtr + max_replication_write_lag * MB))
{
return (myFlushLsn - writePtr - max_replication_write_lag * MB);
}
return (myFlushLsn - flushPtr - max_replication_flush_lag * MB);
}
if ((flushPtr != InvalidXLogRecPtr && max_replication_flush_lag > 0 && myFlushLsn > flushPtr + max_replication_flush_lag * MB))
{
return (myFlushLsn - flushPtr - max_replication_flush_lag * MB);
}
if ((applyPtr != InvalidXLogRecPtr && max_replication_apply_lag > 0 && myFlushLsn > applyPtr + max_replication_apply_lag * MB))
{
return (myFlushLsn - applyPtr - max_replication_apply_lag * MB);
}
if ((applyPtr != InvalidXLogRecPtr && max_replication_apply_lag > 0 && myFlushLsn > applyPtr + max_replication_apply_lag * MB))
{
return (myFlushLsn - applyPtr - max_replication_apply_lag * MB);
}
}
return 0;

View File

@@ -25,7 +25,6 @@ use crate::control_plane::messages::MetricsAuxInfo;
use crate::error::{ReportableError, UserFacingError};
use crate::metrics::{Metrics, NumDbConnectionsGuard};
use crate::pqproto::StartupMessageParams;
use crate::proxy::connect_compute::TlsNegotiation;
use crate::proxy::neon_option;
use crate::types::Host;
@@ -85,14 +84,6 @@ pub(crate) enum ConnectionError {
#[error("error acquiring resource permit: {0}")]
TooManyConnectionAttempts(#[from] ApiLockError),
#[cfg(test)]
#[error("retryable: {retryable}, wakeable: {wakeable}, kind: {kind:?}")]
TestError {
retryable: bool,
wakeable: bool,
kind: crate::error::ErrorKind,
},
}
impl UserFacingError for ConnectionError {
@@ -103,8 +94,6 @@ impl UserFacingError for ConnectionError {
"Failed to acquire permit to connect to the database. Too many database connection attempts are currently ongoing.".to_owned()
}
ConnectionError::TlsError(_) => COULD_NOT_CONNECT.to_owned(),
#[cfg(test)]
ConnectionError::TestError { .. } => self.to_string(),
}
}
}
@@ -115,8 +104,6 @@ impl ReportableError for ConnectionError {
ConnectionError::TlsError(_) => crate::error::ErrorKind::Compute,
ConnectionError::WakeComputeError(e) => e.get_error_kind(),
ConnectionError::TooManyConnectionAttempts(e) => e.get_error_kind(),
#[cfg(test)]
ConnectionError::TestError { kind, .. } => *kind,
}
}
}
@@ -269,7 +256,6 @@ impl ConnectInfo {
async fn connect_raw(
&self,
config: &ComputeConfig,
tls: TlsNegotiation,
) -> Result<(SocketAddr, MaybeTlsStream<TcpStream, RustlsStream>), TlsError> {
let timeout = config.timeout;
@@ -312,7 +298,7 @@ impl ConnectInfo {
match connect_once(&*addrs).await {
Ok((sockaddr, stream)) => Ok((
sockaddr,
tls::connect_tls(stream, self.ssl_mode, config, host, tls).await?,
tls::connect_tls(stream, self.ssl_mode, config, host).await?,
)),
Err(err) => {
warn!("couldn't connect to compute node at {host}:{port}: {err}");
@@ -343,10 +329,9 @@ impl ConnectInfo {
ctx: &RequestContext,
aux: &MetricsAuxInfo,
config: &ComputeConfig,
tls: TlsNegotiation,
) -> Result<ComputeConnection, ConnectionError> {
let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Compute);
let (socket_addr, stream) = self.connect_raw(config, tls).await?;
let (socket_addr, stream) = self.connect_raw(config).await?;
drop(pause);
tracing::Span::current().record("compute_id", tracing::field::display(&aux.compute_id));

View File

@@ -7,7 +7,6 @@ use thiserror::Error;
use tokio::io::{AsyncRead, AsyncWrite};
use crate::pqproto::request_tls;
use crate::proxy::connect_compute::TlsNegotiation;
use crate::proxy::retry::CouldRetry;
#[derive(Debug, Error)]
@@ -36,7 +35,6 @@ pub async fn connect_tls<S, T>(
mode: SslMode,
tls: &T,
host: &str,
negotiation: TlsNegotiation,
) -> Result<MaybeTlsStream<S, T::Stream>, TlsError>
where
S: AsyncRead + AsyncWrite + Unpin + Send,
@@ -51,15 +49,12 @@ where
SslMode::Prefer | SslMode::Require => {}
}
match negotiation {
// No TLS request needed
TlsNegotiation::Direct => {}
// TLS request successful
TlsNegotiation::Postgres if request_tls(&mut stream).await? => {}
// TLS request failed but is required
TlsNegotiation::Postgres if SslMode::Require == mode => return Err(TlsError::Required),
// TLS request failed but is not required
TlsNegotiation::Postgres => return Ok(MaybeTlsStream::Raw(stream)),
if !request_tls(&mut stream).await? {
if SslMode::Require == mode {
return Err(TlsError::Required);
}
return Ok(MaybeTlsStream::Raw(stream));
}
Ok(MaybeTlsStream::Tls(

View File

@@ -16,9 +16,8 @@ use crate::pglb::ClientRequestError;
use crate::pglb::handshake::{HandshakeData, handshake};
use crate::pglb::passthrough::ProxyPassthrough;
use crate::protocol2::{ConnectHeader, ConnectionInfo, read_proxy_protocol};
use crate::proxy::{
ErrorSource, connect_compute, forward_compute_params_to_client, send_client_greeting,
};
use crate::proxy::connect_compute::{TcpMechanism, connect_to_compute};
use crate::proxy::{ErrorSource, forward_compute_params_to_client, send_client_greeting};
use crate::util::run_until_cancelled;
pub async fn task_main(
@@ -216,11 +215,14 @@ pub(crate) async fn handle_client<S: AsyncRead + AsyncWrite + Unpin + Send>(
};
auth_info.set_startup_params(&params, true);
let mut node = connect_compute::connect_to_compute(
let mut node = connect_to_compute(
ctx,
config,
&TcpMechanism {
locks: &config.connect_compute_locks,
},
&node_info,
connect_compute::TlsNegotiation::Postgres,
config.wake_compute_retry_config,
&config.connect_to_compute,
)
.or_else(|e| async { Err(stream.throw_error(e, Some(ctx)).await) })
.await?;

View File

@@ -17,6 +17,7 @@ use crate::auth::backend::ComputeUserInfo;
use crate::auth::backend::jwt::AuthRule;
use crate::auth::{AuthError, IpPattern, check_peer_addr_is_in_list};
use crate::cache::{Cached, TimedLru};
use crate::config::ComputeConfig;
use crate::context::RequestContext;
use crate::control_plane::messages::{ControlPlaneErrorMessage, MetricsAuxInfo};
use crate::intern::{AccountIdInt, EndpointIdInt, ProjectIdInt};
@@ -71,6 +72,16 @@ pub(crate) struct NodeInfo {
pub(crate) aux: MetricsAuxInfo,
}
impl NodeInfo {
pub(crate) async fn connect(
&self,
ctx: &RequestContext,
config: &ComputeConfig,
) -> Result<compute::ComputeConnection, compute::ConnectionError> {
self.conn_info.connect(ctx, &self.aux, config).await
}
}
#[derive(Copy, Clone, Default, Debug)]
pub(crate) struct AccessBlockerFlags {
pub public_access_blocked: bool,

View File

@@ -1,82 +0,0 @@
use thiserror::Error;
use crate::auth::Backend;
use crate::auth::backend::ComputeUserInfo;
use crate::cache::Cache;
use crate::compute::{AuthInfo, ComputeConnection, ConnectionError, PostgresError};
use crate::config::ProxyConfig;
use crate::context::RequestContext;
use crate::control_plane::client::ControlPlaneClient;
use crate::error::{ReportableError, UserFacingError};
use crate::proxy::connect_compute::{TlsNegotiation, connect_to_compute};
use crate::proxy::retry::ShouldRetryWakeCompute;
#[derive(Debug, Error)]
pub enum AuthError {
#[error(transparent)]
Auth(#[from] PostgresError),
#[error(transparent)]
Connect(#[from] ConnectionError),
}
impl UserFacingError for AuthError {
fn to_string_client(&self) -> String {
match self {
AuthError::Auth(postgres_error) => postgres_error.to_string_client(),
AuthError::Connect(connection_error) => connection_error.to_string_client(),
}
}
}
impl ReportableError for AuthError {
fn get_error_kind(&self) -> crate::error::ErrorKind {
match self {
AuthError::Auth(postgres_error) => postgres_error.get_error_kind(),
AuthError::Connect(connection_error) => connection_error.get_error_kind(),
}
}
}
/// Try to connect to the compute node, retrying if necessary.
#[tracing::instrument(skip_all)]
pub(crate) async fn connect_to_compute_and_auth(
ctx: &RequestContext,
config: &ProxyConfig,
user_info: &Backend<'_, ComputeUserInfo>,
auth_info: AuthInfo,
tls: TlsNegotiation,
) -> Result<ComputeConnection, AuthError> {
let mut attempt = 0;
// NOTE: This is messy, but should hopefully be detangled with PGLB.
// We wanted to separate the concerns of **connect** to compute (a PGLB operation),
// from **authenticate** to compute (a NeonKeeper operation).
//
// This unfortunately removed retry handling for one error case where
// the compute was cached, and we connected, but the compute cache was actually stale
// and is associated with the wrong endpoint. We detect this when the **authentication** fails.
// As such, we retry once here if the `authenticate` function fails and the error is valid to retry.
loop {
attempt += 1;
let mut node = connect_to_compute(ctx, config, user_info, tls).await?;
let res = auth_info.authenticate(ctx, &mut node).await;
match res {
Ok(()) => return Ok(node),
Err(e) => {
if attempt < 2
&& let Backend::ControlPlane(cplane, user_info) = user_info
&& let ControlPlaneClient::ProxyV1(cplane_proxy_v1) = &**cplane
&& e.should_retry_wake_compute()
{
tracing::warn!(error = ?e, "retrying wake compute");
let key = user_info.endpoint_cache_key();
cplane_proxy_v1.caches.node_info.invalidate(&key);
continue;
}
return Err(e)?;
}
}
}
}

View File

@@ -1,15 +1,18 @@
use async_trait::async_trait;
use tokio::time;
use tracing::{debug, info, warn};
use crate::compute::{self, COULD_NOT_CONNECT, ComputeConnection};
use crate::config::{ComputeConfig, ProxyConfig, RetryConfig};
use crate::config::{ComputeConfig, RetryConfig};
use crate::context::RequestContext;
use crate::control_plane::errors::WakeComputeError;
use crate::control_plane::locks::ApiLocks;
use crate::control_plane::{self, NodeInfo};
use crate::error::ReportableError;
use crate::metrics::{
ConnectOutcome, ConnectionFailureKind, Metrics, RetriesMetricGroup, RetryType,
};
use crate::proxy::retry::{ShouldRetryWakeCompute, retry_after, should_retry};
use crate::proxy::retry::{CouldRetry, ShouldRetryWakeCompute, retry_after, should_retry};
use crate::proxy::wake_compute::{WakeComputeBackend, wake_compute};
use crate::types::Host;
@@ -32,32 +35,29 @@ pub(crate) fn invalidate_cache(node_info: control_plane::CachedNodeInfo) -> Node
node_info.invalidate()
}
#[async_trait]
pub(crate) trait ConnectMechanism {
type Connection;
type ConnectError: ReportableError;
type Error: From<Self::ConnectError>;
async fn connect_once(
&self,
ctx: &RequestContext,
node_info: &control_plane::CachedNodeInfo,
config: &ComputeConfig,
) -> Result<Self::Connection, compute::ConnectionError>;
) -> Result<Self::Connection, Self::ConnectError>;
}
struct TcpMechanism<'a> {
pub(crate) struct TcpMechanism {
/// connect_to_compute concurrency lock
locks: &'a ApiLocks<Host>,
tls: TlsNegotiation,
pub(crate) locks: &'static ApiLocks<Host>,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum TlsNegotiation {
/// TLS is assumed
Direct,
/// We must ask for TLS using the postgres SSLRequest message
Postgres,
}
impl ConnectMechanism for TcpMechanism<'_> {
#[async_trait]
impl ConnectMechanism for TcpMechanism {
type Connection = ComputeConnection;
type ConnectError = compute::ConnectionError;
type Error = compute::ConnectionError;
#[tracing::instrument(skip_all, fields(
pid = tracing::field::Empty,
@@ -68,47 +68,25 @@ impl ConnectMechanism for TcpMechanism<'_> {
ctx: &RequestContext,
node_info: &control_plane::CachedNodeInfo,
config: &ComputeConfig,
) -> Result<ComputeConnection, compute::ConnectionError> {
) -> Result<ComputeConnection, Self::Error> {
let permit = self.locks.get_permit(&node_info.conn_info.host).await?;
permit.release_result(
node_info
.conn_info
.connect(ctx, &node_info.aux, config, self.tls)
.await,
)
permit.release_result(node_info.connect(ctx, config).await)
}
}
/// Try to connect to the compute node, retrying if necessary.
#[tracing::instrument(skip_all)]
pub(crate) async fn connect_to_compute<B: WakeComputeBackend>(
ctx: &RequestContext,
config: &ProxyConfig,
user_info: &B,
tls: TlsNegotiation,
) -> Result<ComputeConnection, compute::ConnectionError> {
connect_to_compute_inner(
ctx,
&TcpMechanism {
locks: &config.connect_compute_locks,
tls,
},
user_info,
config.wake_compute_retry_config,
&config.connect_to_compute,
)
.await
}
/// Try to connect to the compute node, retrying if necessary.
pub(crate) async fn connect_to_compute_inner<M: ConnectMechanism, B: WakeComputeBackend>(
pub(crate) async fn connect_to_compute<M: ConnectMechanism, B: WakeComputeBackend>(
ctx: &RequestContext,
mechanism: &M,
user_info: &B,
wake_compute_retry_config: RetryConfig,
compute: &ComputeConfig,
) -> Result<M::Connection, compute::ConnectionError> {
) -> Result<M::Connection, M::Error>
where
M::ConnectError: CouldRetry + ShouldRetryWakeCompute + std::fmt::Debug,
M::Error: From<WakeComputeError>,
{
let mut num_retries = 0;
let node_info =
wake_compute(&mut num_retries, ctx, user_info, wake_compute_retry_config).await?;
@@ -142,7 +120,7 @@ pub(crate) async fn connect_to_compute_inner<M: ConnectMechanism, B: WakeCompute
},
num_retries.into(),
);
return Err(err);
return Err(err.into());
}
node_info
} else {
@@ -183,7 +161,7 @@ pub(crate) async fn connect_to_compute_inner<M: ConnectMechanism, B: WakeCompute
},
num_retries.into(),
);
return Err(e);
return Err(e.into());
}
warn!(error = ?e, num_retries, retriable = true, COULD_NOT_CONNECT);

View File

@@ -1,7 +1,6 @@
#[cfg(test)]
mod tests;
pub(crate) mod connect_auth;
pub(crate) mod connect_compute;
pub(crate) mod retry;
pub(crate) mod wake_compute;
@@ -24,13 +23,17 @@ use tokio::net::TcpStream;
use tokio::sync::oneshot;
use tracing::Instrument;
use crate::cache::Cache;
use crate::cancellation::{CancelClosure, CancellationHandler};
use crate::compute::{ComputeConnection, PostgresError, RustlsStream};
use crate::config::ProxyConfig;
use crate::context::RequestContext;
use crate::control_plane::client::ControlPlaneClient;
pub use crate::pglb::copy_bidirectional::{ErrorSource, copy_bidirectional_client_compute};
use crate::pglb::{ClientMode, ClientRequestError};
use crate::pqproto::{BeMessage, CancelKeyData, StartupMessageParams};
use crate::proxy::connect_compute::{TcpMechanism, connect_to_compute};
use crate::proxy::retry::ShouldRetryWakeCompute;
use crate::rate_limiter::EndpointRateLimiter;
use crate::stream::{PqStream, Stream};
use crate::types::EndpointCacheKey;
@@ -92,24 +95,61 @@ pub(crate) async fn handle_client<S: AsyncRead + AsyncWrite + Unpin + Send>(
let mut auth_info = compute::AuthInfo::with_auth_keys(creds.keys);
auth_info.set_startup_params(params, params_compat);
let mut node;
let mut attempt = 0;
let connect = TcpMechanism {
locks: &config.connect_compute_locks,
};
let backend = auth::Backend::ControlPlane(cplane, creds.info);
// TODO: callback to pglb
let res = connect_auth::connect_to_compute_and_auth(
ctx,
config,
&backend,
auth_info,
connect_compute::TlsNegotiation::Postgres,
)
.await;
// NOTE: This is messy, but should hopefully be detangled with PGLB.
// We wanted to separate the concerns of **connect** to compute (a PGLB operation),
// from **authenticate** to compute (a NeonKeeper operation).
//
// This unfortunately removed retry handling for one error case where
// the compute was cached, and we connected, but the compute cache was actually stale
// and is associated with the wrong endpoint. We detect this when the **authentication** fails.
// As such, we retry once here if the `authenticate` function fails and the error is valid to retry.
loop {
attempt += 1;
let mut node = match res {
Ok(node) => node,
Err(e) => Err(client.throw_error(e, Some(ctx)).await)?,
};
// TODO: callback to pglb
let res = connect_to_compute(
ctx,
&connect,
&backend,
config.wake_compute_retry_config,
&config.connect_to_compute,
)
.await;
send_client_greeting(ctx, &config.greetings, client);
match res {
Ok(n) => node = n,
Err(e) => return Err(client.throw_error(e, Some(ctx)).await)?,
}
let auth::Backend::ControlPlane(cplane, user_info) = &backend else {
unreachable!("ensured above");
};
let res = auth_info.authenticate(ctx, &mut node).await;
match res {
Ok(()) => {
send_client_greeting(ctx, &config.greetings, client);
break;
}
Err(e) if attempt < 2 && e.should_retry_wake_compute() => {
tracing::warn!(error = ?e, "retrying wake compute");
#[allow(irrefutable_let_patterns)]
if let ControlPlaneClient::ProxyV1(cplane_proxy_v1) = &**cplane {
let key = user_info.endpoint_cache_key();
cplane_proxy_v1.caches.node_info.invalidate(&key);
}
}
Err(e) => Err(client.throw_error(e, Some(ctx)).await)?,
}
}
let auth::Backend::ControlPlane(_, user_info) = backend else {
unreachable!("ensured above");

View File

@@ -31,6 +31,18 @@ impl CouldRetry for io::Error {
}
}
impl CouldRetry for postgres_client::error::DbError {
fn could_retry(&self) -> bool {
use postgres_client::error::SqlState;
matches!(
self.code(),
&SqlState::CONNECTION_FAILURE
| &SqlState::CONNECTION_EXCEPTION
| &SqlState::CONNECTION_DOES_NOT_EXIST
| &SqlState::SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION,
)
}
}
impl ShouldRetryWakeCompute for postgres_client::error::DbError {
fn should_retry_wake_compute(&self) -> bool {
use postgres_client::error::SqlState;
@@ -61,6 +73,17 @@ impl ShouldRetryWakeCompute for postgres_client::error::DbError {
}
}
impl CouldRetry for postgres_client::Error {
fn could_retry(&self) -> bool {
if let Some(io_err) = self.source().and_then(|x| x.downcast_ref()) {
io::Error::could_retry(io_err)
} else if let Some(db_err) = self.source().and_then(|x| x.downcast_ref()) {
postgres_client::error::DbError::could_retry(db_err)
} else {
false
}
}
}
impl ShouldRetryWakeCompute for postgres_client::Error {
fn should_retry_wake_compute(&self) -> bool {
if let Some(db_err) = self.source().and_then(|x| x.downcast_ref()) {
@@ -79,8 +102,6 @@ impl CouldRetry for compute::ConnectionError {
compute::ConnectionError::TlsError(err) => err.could_retry(),
compute::ConnectionError::WakeComputeError(err) => err.could_retry(),
compute::ConnectionError::TooManyConnectionAttempts(_) => false,
#[cfg(test)]
compute::ConnectionError::TestError { retryable, .. } => *retryable,
}
}
}
@@ -89,8 +110,6 @@ impl ShouldRetryWakeCompute for compute::ConnectionError {
match self {
// the cache entry was not checked for validity
compute::ConnectionError::TooManyConnectionAttempts(_) => false,
#[cfg(test)]
compute::ConnectionError::TestError { wakeable, .. } => *wakeable,
_ => true,
}
}

View File

@@ -24,13 +24,13 @@ use crate::context::RequestContext;
use crate::control_plane::client::{ControlPlaneClient, TestControlPlaneClient};
use crate::control_plane::messages::{ControlPlaneErrorMessage, Details, MetricsAuxInfo, Status};
use crate::control_plane::{self, CachedNodeInfo, NodeInfo, NodeInfoCache};
use crate::error::ErrorKind;
use crate::error::{ErrorKind, ReportableError};
use crate::pglb::ERR_INSECURE_CONNECTION;
use crate::pglb::handshake::{HandshakeData, handshake};
use crate::pqproto::BeMessage;
use crate::proxy::NeonOptions;
use crate::proxy::connect_compute::{ConnectMechanism, connect_to_compute_inner};
use crate::proxy::retry::retry_after;
use crate::proxy::connect_compute::{ConnectMechanism, connect_to_compute};
use crate::proxy::retry::{ShouldRetryWakeCompute, retry_after};
use crate::stream::{PqStream, Stream};
use crate::tls::client_config::compute_client_config_with_certs;
use crate::tls::server_config::CertResolver;
@@ -430,36 +430,71 @@ impl TestConnectMechanism {
#[derive(Debug)]
struct TestConnection;
#[derive(Debug)]
struct TestConnectError {
retryable: bool,
wakeable: bool,
kind: crate::error::ErrorKind,
}
impl ReportableError for TestConnectError {
fn get_error_kind(&self) -> crate::error::ErrorKind {
self.kind
}
}
impl std::fmt::Display for TestConnectError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{self:?}")
}
}
impl std::error::Error for TestConnectError {}
impl CouldRetry for TestConnectError {
fn could_retry(&self) -> bool {
self.retryable
}
}
impl ShouldRetryWakeCompute for TestConnectError {
fn should_retry_wake_compute(&self) -> bool {
self.wakeable
}
}
#[async_trait]
impl ConnectMechanism for TestConnectMechanism {
type Connection = TestConnection;
type ConnectError = TestConnectError;
type Error = anyhow::Error;
async fn connect_once(
&self,
_ctx: &RequestContext,
_node_info: &control_plane::CachedNodeInfo,
_config: &ComputeConfig,
) -> Result<Self::Connection, compute::ConnectionError> {
) -> Result<Self::Connection, Self::ConnectError> {
let mut counter = self.counter.lock().unwrap();
let action = self.sequence[*counter];
*counter += 1;
match action {
ConnectAction::Connect => Ok(TestConnection),
ConnectAction::Retry => Err(compute::ConnectionError::TestError {
ConnectAction::Retry => Err(TestConnectError {
retryable: true,
wakeable: true,
kind: ErrorKind::Compute,
}),
ConnectAction::RetryNoWake => Err(compute::ConnectionError::TestError {
ConnectAction::RetryNoWake => Err(TestConnectError {
retryable: true,
wakeable: false,
kind: ErrorKind::Compute,
}),
ConnectAction::Fail => Err(compute::ConnectionError::TestError {
ConnectAction::Fail => Err(TestConnectError {
retryable: false,
wakeable: true,
kind: ErrorKind::Compute,
}),
ConnectAction::FailNoWake => Err(compute::ConnectionError::TestError {
ConnectAction::FailNoWake => Err(TestConnectError {
retryable: false,
wakeable: false,
kind: ErrorKind::Compute,
@@ -585,7 +620,7 @@ async fn connect_to_compute_success() {
let mechanism = TestConnectMechanism::new(vec![Wake, Connect]);
let user_info = helper_create_connect_info(&mechanism);
let config = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, config.retry, &config)
connect_to_compute(&ctx, &mechanism, &user_info, config.retry, &config)
.await
.unwrap();
mechanism.verify();
@@ -599,7 +634,7 @@ async fn connect_to_compute_retry() {
let mechanism = TestConnectMechanism::new(vec![Wake, Retry, Wake, Connect]);
let user_info = helper_create_connect_info(&mechanism);
let config = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, config.retry, &config)
connect_to_compute(&ctx, &mechanism, &user_info, config.retry, &config)
.await
.unwrap();
mechanism.verify();
@@ -614,7 +649,7 @@ async fn connect_to_compute_non_retry_1() {
let mechanism = TestConnectMechanism::new(vec![Wake, Retry, Wake, Fail]);
let user_info = helper_create_connect_info(&mechanism);
let config = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, config.retry, &config)
connect_to_compute(&ctx, &mechanism, &user_info, config.retry, &config)
.await
.unwrap_err();
mechanism.verify();
@@ -629,7 +664,7 @@ async fn connect_to_compute_non_retry_2() {
let mechanism = TestConnectMechanism::new(vec![Wake, Fail, Wake, Connect]);
let user_info = helper_create_connect_info(&mechanism);
let config = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, config.retry, &config)
connect_to_compute(&ctx, &mechanism, &user_info, config.retry, &config)
.await
.unwrap();
mechanism.verify();
@@ -651,7 +686,7 @@ async fn connect_to_compute_non_retry_3() {
backoff_factor: 2.0,
};
let config = config();
connect_to_compute_inner(
connect_to_compute(
&ctx,
&mechanism,
&user_info,
@@ -672,7 +707,7 @@ async fn wake_retry() {
let mechanism = TestConnectMechanism::new(vec![WakeRetry, Wake, Connect]);
let user_info = helper_create_connect_info(&mechanism);
let config = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, config.retry, &config)
connect_to_compute(&ctx, &mechanism, &user_info, config.retry, &config)
.await
.unwrap();
mechanism.verify();
@@ -687,7 +722,7 @@ async fn wake_non_retry() {
let mechanism = TestConnectMechanism::new(vec![WakeRetry, WakeFail]);
let user_info = helper_create_connect_info(&mechanism);
let config = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, config.retry, &config)
connect_to_compute(&ctx, &mechanism, &user_info, config.retry, &config)
.await
.unwrap_err();
mechanism.verify();
@@ -706,7 +741,7 @@ async fn fail_but_wake_invalidates_cache() {
let user = helper_create_connect_info(&mech);
let cfg = config();
connect_to_compute_inner(&ctx, &mech, &user, cfg.retry, &cfg)
connect_to_compute(&ctx, &mech, &user, cfg.retry, &cfg)
.await
.unwrap();
@@ -727,7 +762,7 @@ async fn fail_no_wake_skips_cache_invalidation() {
let user = helper_create_connect_info(&mech);
let cfg = config();
connect_to_compute_inner(&ctx, &mech, &user, cfg.retry, &cfg)
connect_to_compute(&ctx, &mech, &user, cfg.retry, &cfg)
.await
.unwrap();
@@ -748,7 +783,7 @@ async fn retry_but_wake_invalidates_cache() {
let user_info = helper_create_connect_info(&mechanism);
let cfg = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
connect_to_compute(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
.await
.unwrap();
mechanism.verify();
@@ -771,7 +806,7 @@ async fn retry_no_wake_skips_invalidation() {
let user_info = helper_create_connect_info(&mechanism);
let cfg = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
connect_to_compute(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
.await
.unwrap_err();
mechanism.verify();
@@ -794,7 +829,7 @@ async fn retry_no_wake_error_fast() {
let user_info = helper_create_connect_info(&mechanism);
let cfg = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
connect_to_compute(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
.await
.unwrap_err();
mechanism.verify();
@@ -817,7 +852,7 @@ async fn retry_cold_wake_skips_invalidation() {
let user_info = helper_create_connect_info(&mechanism);
let cfg = config();
connect_to_compute_inner(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
connect_to_compute(&ctx, &mechanism, &user_info, cfg.retry, &cfg)
.await
.unwrap();
mechanism.verify();

View File

@@ -1,11 +1,17 @@
use std::io;
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use ed25519_dalek::SigningKey;
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
use jose_jwk::jose_b64;
use postgres_client::maybe_tls_stream::MaybeTlsStream;
use postgres_client::config::SslMode;
use rand_core::OsRng;
use rustls::pki_types::{DnsName, ServerName};
use tokio::net::{TcpStream, lookup_host};
use tokio_rustls::TlsConnector;
use tracing::field::display;
use tracing::{debug, info};
@@ -15,22 +21,23 @@ use super::conn_pool_lib::{Client, ConnInfo, EndpointConnPool, GlobalConnPool};
use super::http_conn_pool::{self, HttpConnPool, LocalProxyClient, poll_http2_client};
use super::local_conn_pool::{self, EXT_NAME, EXT_SCHEMA, EXT_VERSION, LocalConnPool};
use crate::auth::backend::local::StaticAuthRules;
use crate::auth::backend::{ComputeCredentials, ComputeUserInfo};
use crate::auth::backend::{ComputeCredentialKeys, ComputeCredentials, ComputeUserInfo};
use crate::auth::{self, AuthError};
use crate::compute;
use crate::compute_ctl::{
ComputeCtlError, ExtensionInstallRequest, Privilege, SetRoleGrantsRequest,
};
use crate::config::ProxyConfig;
use crate::config::{ComputeConfig, ProxyConfig};
use crate::context::RequestContext;
use crate::control_plane::CachedNodeInfo;
use crate::control_plane::client::ApiLockError;
use crate::control_plane::errors::{GetAuthInfoError, WakeComputeError};
use crate::control_plane::locks::ApiLocks;
use crate::error::{ErrorKind, ReportableError, UserFacingError};
use crate::intern::EndpointIdInt;
use crate::pqproto::StartupMessageParams;
use crate::proxy::{connect_auth, connect_compute};
use crate::proxy::connect_compute::ConnectMechanism;
use crate::proxy::retry::{CouldRetry, ShouldRetryWakeCompute};
use crate::rate_limiter::EndpointRateLimiter;
use crate::types::{EndpointId, LOCAL_PROXY_SUFFIX};
use crate::types::{EndpointId, Host, LOCAL_PROXY_SUFFIX};
pub(crate) struct PoolingBackend {
pub(crate) http_conn_pool:
@@ -179,42 +186,20 @@ impl PoolingBackend {
tracing::Span::current().record("conn_id", display(conn_id));
info!(%conn_id, "pool: opening a new connection '{conn_info}'");
let backend = self.auth_backend.as_ref().map(|()| keys.info);
let mut params = StartupMessageParams::default();
params.insert("database", &conn_info.dbname);
params.insert("user", &conn_info.user_info.user);
let mut auth_info = compute::AuthInfo::with_auth_keys(keys.keys);
auth_info.set_startup_params(&params, true);
let node = connect_auth::connect_to_compute_and_auth(
crate::proxy::connect_compute::connect_to_compute(
ctx,
self.config,
&TokioMechanism {
conn_id,
conn_info,
pool: self.pool.clone(),
locks: &self.config.connect_compute_locks,
keys: keys.keys,
},
&backend,
auth_info,
connect_compute::TlsNegotiation::Postgres,
self.config.wake_compute_retry_config,
&self.config.connect_to_compute,
)
.await?;
let (client, connection) = postgres_client::connect::managed(
node.stream,
Some(node.socket_addr.ip()),
postgres_client::config::Host::Tcp(node.hostname.to_string()),
node.socket_addr.port(),
node.ssl_mode,
Some(self.config.connect_to_compute.timeout),
)
.await?;
Ok(poll_client(
self.pool.clone(),
ctx,
conn_info,
client,
connection,
conn_id,
node.aux,
))
.await
}
// Wake up the destination if needed
@@ -243,38 +228,19 @@ impl PoolingBackend {
)),
options: conn_info.user_info.options.clone(),
});
let node = connect_compute::connect_to_compute(
crate::proxy::connect_compute::connect_to_compute(
ctx,
self.config,
&HyperMechanism {
conn_id,
conn_info,
pool: self.http_conn_pool.clone(),
locks: &self.config.connect_compute_locks,
},
&backend,
connect_compute::TlsNegotiation::Direct,
self.config.wake_compute_retry_config,
&self.config.connect_to_compute,
)
.await?;
let stream = match node.stream.into_framed().into_inner() {
MaybeTlsStream::Raw(s) => Box::pin(s) as AsyncRW,
MaybeTlsStream::Tls(s) => Box::pin(s) as AsyncRW,
};
let (client, connection) = hyper::client::conn::http2::Builder::new(TokioExecutor::new())
.timer(TokioTimer::new())
.keep_alive_interval(Duration::from_secs(20))
.keep_alive_while_idle(true)
.keep_alive_timeout(Duration::from_secs(5))
.handshake(TokioIo::new(stream))
.await
.map_err(LocalProxyConnError::H2)?;
Ok(poll_http2_client(
self.http_conn_pool.clone(),
ctx,
&conn_info,
client,
connection,
conn_id,
node.aux.clone(),
))
.await
}
/// Connect to postgres over localhost.
@@ -414,8 +380,6 @@ fn create_random_jwk() -> (SigningKey, jose_jwk::Key) {
pub(crate) enum HttpConnError {
#[error("pooled connection closed at inconsistent state")]
ConnectionClosedAbruptly(#[from] tokio::sync::watch::error::SendError<uuid::Uuid>),
#[error("could not connect to compute")]
ConnectError(#[from] compute::ConnectionError),
#[error("could not connect to postgres in compute")]
PostgresConnectionError(#[from] postgres_client::Error),
#[error("could not connect to local-proxy in compute")]
@@ -435,19 +399,10 @@ pub(crate) enum HttpConnError {
TooManyConnectionAttempts(#[from] ApiLockError),
}
impl From<connect_auth::AuthError> for HttpConnError {
fn from(value: connect_auth::AuthError) -> Self {
match value {
connect_auth::AuthError::Auth(compute::PostgresError::Postgres(error)) => {
Self::PostgresConnectionError(error)
}
connect_auth::AuthError::Connect(error) => Self::ConnectError(error),
}
}
}
#[derive(Debug, thiserror::Error)]
pub(crate) enum LocalProxyConnError {
#[error("error with connection to local-proxy")]
Io(#[source] std::io::Error),
#[error("could not establish h2 connection")]
H2(#[from] hyper::Error),
}
@@ -455,7 +410,6 @@ pub(crate) enum LocalProxyConnError {
impl ReportableError for HttpConnError {
fn get_error_kind(&self) -> ErrorKind {
match self {
HttpConnError::ConnectError(_) => ErrorKind::Compute,
HttpConnError::ConnectionClosedAbruptly(_) => ErrorKind::Compute,
HttpConnError::PostgresConnectionError(p) => {
if p.as_db_error().is_some() {
@@ -480,7 +434,6 @@ impl ReportableError for HttpConnError {
impl UserFacingError for HttpConnError {
fn to_string_client(&self) -> String {
match self {
HttpConnError::ConnectError(p) => p.to_string_client(),
HttpConnError::ConnectionClosedAbruptly(_) => self.to_string(),
HttpConnError::PostgresConnectionError(p) => p.to_string(),
HttpConnError::LocalProxyConnectionError(p) => p.to_string(),
@@ -496,9 +449,36 @@ impl UserFacingError for HttpConnError {
}
}
impl CouldRetry for HttpConnError {
fn could_retry(&self) -> bool {
match self {
HttpConnError::PostgresConnectionError(e) => e.could_retry(),
HttpConnError::LocalProxyConnectionError(e) => e.could_retry(),
HttpConnError::ComputeCtl(_) => false,
HttpConnError::ConnectionClosedAbruptly(_) => false,
HttpConnError::JwtPayloadError(_) => false,
HttpConnError::GetAuthInfo(_) => false,
HttpConnError::AuthError(_) => false,
HttpConnError::WakeCompute(_) => false,
HttpConnError::TooManyConnectionAttempts(_) => false,
}
}
}
impl ShouldRetryWakeCompute for HttpConnError {
fn should_retry_wake_compute(&self) -> bool {
match self {
HttpConnError::PostgresConnectionError(e) => e.should_retry_wake_compute(),
// we never checked cache validity
HttpConnError::TooManyConnectionAttempts(_) => false,
_ => true,
}
}
}
impl ReportableError for LocalProxyConnError {
fn get_error_kind(&self) -> ErrorKind {
match self {
LocalProxyConnError::Io(_) => ErrorKind::Compute,
LocalProxyConnError::H2(_) => ErrorKind::Compute,
}
}
@@ -509,3 +489,215 @@ impl UserFacingError for LocalProxyConnError {
"Could not establish HTTP connection to the database".to_string()
}
}
impl CouldRetry for LocalProxyConnError {
fn could_retry(&self) -> bool {
match self {
LocalProxyConnError::Io(_) => false,
LocalProxyConnError::H2(_) => false,
}
}
}
impl ShouldRetryWakeCompute for LocalProxyConnError {
fn should_retry_wake_compute(&self) -> bool {
match self {
LocalProxyConnError::Io(_) => false,
LocalProxyConnError::H2(_) => false,
}
}
}
struct TokioMechanism {
pool: Arc<GlobalConnPool<postgres_client::Client, EndpointConnPool<postgres_client::Client>>>,
conn_info: ConnInfo,
conn_id: uuid::Uuid,
keys: ComputeCredentialKeys,
/// connect_to_compute concurrency lock
locks: &'static ApiLocks<Host>,
}
#[async_trait]
impl ConnectMechanism for TokioMechanism {
type Connection = Client<postgres_client::Client>;
type ConnectError = HttpConnError;
type Error = HttpConnError;
async fn connect_once(
&self,
ctx: &RequestContext,
node_info: &CachedNodeInfo,
compute_config: &ComputeConfig,
) -> Result<Self::Connection, Self::ConnectError> {
let permit = self.locks.get_permit(&node_info.conn_info.host).await?;
let mut config = node_info.conn_info.to_postgres_client_config();
let config = config
.user(&self.conn_info.user_info.user)
.dbname(&self.conn_info.dbname)
.connect_timeout(compute_config.timeout);
if let ComputeCredentialKeys::AuthKeys(auth_keys) = self.keys {
config.auth_keys(auth_keys);
}
let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Compute);
let res = config.connect(compute_config).await;
drop(pause);
let (client, connection) = permit.release_result(res)?;
tracing::Span::current().record("pid", tracing::field::display(client.get_process_id()));
tracing::Span::current().record(
"compute_id",
tracing::field::display(&node_info.aux.compute_id),
);
if let Some(query_id) = ctx.get_testodrome_id() {
info!("latency={}, query_id={}", ctx.get_proxy_latency(), query_id);
}
Ok(poll_client(
self.pool.clone(),
ctx,
self.conn_info.clone(),
client,
connection,
self.conn_id,
node_info.aux.clone(),
))
}
}
struct HyperMechanism {
pool: Arc<GlobalConnPool<LocalProxyClient, HttpConnPool<LocalProxyClient>>>,
conn_info: ConnInfo,
conn_id: uuid::Uuid,
/// connect_to_compute concurrency lock
locks: &'static ApiLocks<Host>,
}
#[async_trait]
impl ConnectMechanism for HyperMechanism {
type Connection = http_conn_pool::Client<LocalProxyClient>;
type ConnectError = HttpConnError;
type Error = HttpConnError;
async fn connect_once(
&self,
ctx: &RequestContext,
node_info: &CachedNodeInfo,
config: &ComputeConfig,
) -> Result<Self::Connection, Self::ConnectError> {
let host_addr = node_info.conn_info.host_addr;
let host = &node_info.conn_info.host;
let permit = self.locks.get_permit(host).await?;
let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Compute);
let tls = if node_info.conn_info.ssl_mode == SslMode::Disable {
None
} else {
Some(&config.tls)
};
let port = node_info.conn_info.port;
let res = connect_http2(host_addr, host, port, config.timeout, tls).await;
drop(pause);
let (client, connection) = permit.release_result(res)?;
tracing::Span::current().record(
"compute_id",
tracing::field::display(&node_info.aux.compute_id),
);
if let Some(query_id) = ctx.get_testodrome_id() {
info!("latency={}, query_id={}", ctx.get_proxy_latency(), query_id);
}
Ok(poll_http2_client(
self.pool.clone(),
ctx,
&self.conn_info,
client,
connection,
self.conn_id,
node_info.aux.clone(),
))
}
}
async fn connect_http2(
host_addr: Option<IpAddr>,
host: &str,
port: u16,
timeout: Duration,
tls: Option<&Arc<rustls::ClientConfig>>,
) -> Result<
(
http_conn_pool::LocalProxyClient,
http_conn_pool::LocalProxyConnection,
),
LocalProxyConnError,
> {
let addrs = match host_addr {
Some(addr) => vec![SocketAddr::new(addr, port)],
None => lookup_host((host, port))
.await
.map_err(LocalProxyConnError::Io)?
.collect(),
};
let mut last_err = None;
let mut addrs = addrs.into_iter();
let stream = loop {
let Some(addr) = addrs.next() else {
return Err(last_err.unwrap_or_else(|| {
LocalProxyConnError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
"could not resolve any addresses",
))
}));
};
match tokio::time::timeout(timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => {
stream.set_nodelay(true).map_err(LocalProxyConnError::Io)?;
break stream;
}
Ok(Err(e)) => {
last_err = Some(LocalProxyConnError::Io(e));
}
Err(e) => {
last_err = Some(LocalProxyConnError::Io(io::Error::new(
io::ErrorKind::TimedOut,
e,
)));
}
}
};
let stream = if let Some(tls) = tls {
let host = DnsName::try_from(host)
.map_err(io::Error::other)
.map_err(LocalProxyConnError::Io)?
.to_owned();
let stream = TlsConnector::from(tls.clone())
.connect(ServerName::DnsName(host), stream)
.await
.map_err(LocalProxyConnError::Io)?;
Box::pin(stream) as AsyncRW
} else {
Box::pin(stream) as AsyncRW
};
let (client, connection) = hyper::client::conn::http2::Builder::new(TokioExecutor::new())
.timer(TokioTimer::new())
.keep_alive_interval(Duration::from_secs(20))
.keep_alive_while_idle(true)
.keep_alive_timeout(Duration::from_secs(5))
.handshake(TokioIo::new(stream))
.await?;
Ok((client, connection))
}

View File

@@ -249,10 +249,6 @@ impl IntentState {
}
pub(crate) fn push_secondary(&mut self, scheduler: &mut Scheduler, new_secondary: NodeId) {
// Every assertion here should probably have a corresponding check in
// `validate_optimization` unless it is an invariant that should never be violated. Note
// that the lock is not held between planning optimizations and applying them so you have to
// assume any valid state transition of the intent state may have occurred
assert!(!self.secondary.contains(&new_secondary));
assert!(self.attached != Some(new_secondary));
scheduler.update_node_ref_counts(
@@ -1339,9 +1335,8 @@ impl TenantShard {
true
}
/// Check that the desired modifications to the intent state are compatible with the current
/// intent state. Note that the lock is not held between planning optimizations and applying
/// them so any valid state transition of the intent state may have occurred.
/// Check that the desired modifications to the intent state are compatible with
/// the current intent state
fn validate_optimization(&self, optimization: &ScheduleOptimization) -> bool {
match optimization.action {
ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment {
@@ -1357,9 +1352,6 @@ impl TenantShard {
}) => {
// It's legal to remove a secondary that is not present in the intent state
!self.intent.secondary.contains(&new_node_id)
// Ensure the secondary hasn't already been promoted to attached by a concurrent
// optimization/migration.
&& self.intent.attached != Some(new_node_id)
}
ScheduleOptimizationAction::CreateSecondary(new_node_id) => {
!self.intent.secondary.contains(&new_node_id)

View File

@@ -587,9 +587,7 @@ class NeonLocalCli(AbstractNeonCli):
]
extra_env_vars = env or {}
if basebackup_request_tries is not None:
extra_env_vars["NEON_COMPUTE_TESTING_BASEBACKUP_RETRIES"] = str(
basebackup_request_tries
)
extra_env_vars["NEON_COMPUTE_TESTING_BASEBACKUP_TRIES"] = str(basebackup_request_tries)
if remote_ext_base_url is not None:
args.extend(["--remote-ext-base-url", remote_ext_base_url])
@@ -625,7 +623,6 @@ class NeonLocalCli(AbstractNeonCli):
pageserver_id: int | None = None,
safekeepers: list[int] | None = None,
check_return_code=True,
timeout_sec: float | None = None,
) -> subprocess.CompletedProcess[str]:
args = ["endpoint", "reconfigure", endpoint_id]
if tenant_id is not None:
@@ -634,16 +631,7 @@ class NeonLocalCli(AbstractNeonCli):
args.extend(["--pageserver-id", str(pageserver_id)])
if safekeepers is not None:
args.extend(["--safekeepers", (",".join(map(str, safekeepers)))])
return self.raw_cli(args, check_return_code=check_return_code, timeout=timeout_sec)
def endpoint_refresh_configuration(
self,
endpoint_id: str,
) -> subprocess.CompletedProcess[str]:
args = ["endpoint", "refresh-configuration", endpoint_id]
res = self.raw_cli(args)
res.check_returncode()
return res
return self.raw_cli(args, check_return_code=check_return_code)
def endpoint_stop(
self,
@@ -669,22 +657,6 @@ class NeonLocalCli(AbstractNeonCli):
lsn: Lsn | None = None if lsn_str == "null" else Lsn(lsn_str)
return lsn, proc
def endpoint_update_pageservers(
self,
endpoint_id: str,
pageserver_id: int | None = None,
) -> subprocess.CompletedProcess[str]:
args = [
"endpoint",
"update-pageservers",
endpoint_id,
]
if pageserver_id is not None:
args.extend(["--pageserver-id", str(pageserver_id)])
res = self.raw_cli(args)
res.check_returncode()
return res
def mappings_map_branch(
self, name: str, tenant_id: TenantId, timeline_id: TimelineId
) -> subprocess.CompletedProcess[str]:

View File

@@ -4930,38 +4930,15 @@ class Endpoint(PgProtocol, LogUtils):
def is_running(self):
return self._running._value > 0
def reconfigure(
self,
pageserver_id: int | None = None,
safekeepers: list[int] | None = None,
timeout_sec: float = 120,
):
def reconfigure(self, pageserver_id: int | None = None, safekeepers: list[int] | None = None):
assert self.endpoint_id is not None
# If `safekeepers` is not None, they are remember them as active and use
# in the following commands.
if safekeepers is not None:
self.active_safekeepers = safekeepers
start_time = time.time()
while True:
try:
self.env.neon_cli.endpoint_reconfigure(
self.endpoint_id,
self.tenant_id,
pageserver_id,
self.active_safekeepers,
timeout_sec=timeout_sec,
)
return
except RuntimeError as e:
if time.time() - start_time > timeout_sec:
raise e
log.warning(f"Reconfigure failed with error: {e}. Retrying...")
time.sleep(5)
def refresh_configuration(self):
assert self.endpoint_id is not None
self.env.neon_cli.endpoint_refresh_configuration(self.endpoint_id)
self.env.neon_cli.endpoint_reconfigure(
self.endpoint_id, self.tenant_id, pageserver_id, self.active_safekeepers
)
def respec(self, **kwargs: Any) -> None:
"""Update the endpoint.json file used by control_plane."""
@@ -5009,10 +4986,6 @@ class Endpoint(PgProtocol, LogUtils):
log.debug("Updating compute config to: %s", json.dumps(config, indent=4))
json.dump(config, file, indent=4)
def update_pageservers_in_config(self, pageserver_id: int | None = None):
assert self.endpoint_id is not None
self.env.neon_cli.endpoint_update_pageservers(self.endpoint_id, pageserver_id)
def wait_for_migrations(self, wait_for: int = NUM_COMPUTE_MIGRATIONS) -> None:
"""
Wait for all compute migrations to be ran. Remember that migrations only

View File

@@ -78,9 +78,6 @@ class Workload:
"""
if self._endpoint is not None:
with ENDPOINT_LOCK:
# It's important that we update config.json before issuing the reconfigure request to make sure
# that PG-initiated spec refresh doesn't mess things up by reverting to the old spec.
self._endpoint.update_pageservers_in_config()
self._endpoint.reconfigure()
def endpoint(self, pageserver_id: int | None = None) -> Endpoint:
@@ -100,10 +97,10 @@ class Workload:
self._endpoint.start(pageserver_id=pageserver_id)
self._configured_pageserver = pageserver_id
else:
# It's important that we update config.json before issuing the reconfigure request to make sure
# that PG-initiated spec refresh doesn't mess things up by reverting to the old spec.
self._endpoint.update_pageservers_in_config(pageserver_id=pageserver_id)
self._endpoint.reconfigure(pageserver_id=pageserver_id)
if self._configured_pageserver != pageserver_id:
self._configured_pageserver = pageserver_id
self._endpoint.reconfigure(pageserver_id=pageserver_id)
self._endpoint_config = pageserver_id
connstring = self._endpoint.safe_psql(
"SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'"

View File

@@ -1,152 +0,0 @@
#!/usr/bin/env python3
"""
Generate TPS and latency charts from BenchBase TPC-C results CSV files.
This script reads a CSV file containing BenchBase results and generates two charts:
1. TPS (requests per second) over time
2. P95 and P99 latencies over time
Both charts are combined in a single SVG file.
"""
import argparse
import sys
from pathlib import Path
import matplotlib.pyplot as plt # type: ignore[import-not-found]
import pandas as pd # type: ignore[import-untyped]
def load_results_csv(csv_file_path):
"""Load BenchBase results CSV file into a pandas DataFrame."""
try:
df = pd.read_csv(csv_file_path)
# Validate required columns exist
required_columns = [
"Time (seconds)",
"Throughput (requests/second)",
"95th Percentile Latency (millisecond)",
"99th Percentile Latency (millisecond)",
]
missing_columns = [col for col in required_columns if col not in df.columns]
if missing_columns:
print(f"Error: Missing required columns: {missing_columns}")
sys.exit(1)
return df
except FileNotFoundError:
print(f"Error: CSV file not found: {csv_file_path}")
sys.exit(1)
except pd.errors.EmptyDataError:
print(f"Error: CSV file is empty: {csv_file_path}")
sys.exit(1)
except Exception as e:
print(f"Error reading CSV file: {e}")
sys.exit(1)
def generate_charts(df, input_filename, output_svg_path, title_suffix=None):
"""Generate combined TPS and latency charts and save as SVG."""
# Get the filename without extension for chart titles
file_label = Path(input_filename).stem
# Build title ending with optional suffix
if title_suffix:
title_ending = f"{title_suffix} - {file_label}"
else:
title_ending = file_label
# Create figure with two subplots
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10))
# Chart 1: Time vs TPS
ax1.plot(
df["Time (seconds)"],
df["Throughput (requests/second)"],
linewidth=1,
color="blue",
alpha=0.7,
)
ax1.set_xlabel("Time (seconds)")
ax1.set_ylabel("TPS (Requests Per Second)")
ax1.set_title(f"Benchbase TPC-C Like Throughput (TPS) - {title_ending}")
ax1.grid(True, alpha=0.3)
ax1.set_xlim(0, df["Time (seconds)"].max())
# Chart 2: Time vs P95 and P99 Latencies
ax2.plot(
df["Time (seconds)"],
df["95th Percentile Latency (millisecond)"],
linewidth=1,
color="orange",
alpha=0.7,
label="Latency P95",
)
ax2.plot(
df["Time (seconds)"],
df["99th Percentile Latency (millisecond)"],
linewidth=1,
color="red",
alpha=0.7,
label="Latency P99",
)
ax2.set_xlabel("Time (seconds)")
ax2.set_ylabel("Latency (ms)")
ax2.set_title(f"Benchbase TPC-C Like Latency - {title_ending}")
ax2.grid(True, alpha=0.3)
ax2.set_xlim(0, df["Time (seconds)"].max())
ax2.legend()
plt.tight_layout()
# Save as SVG
try:
plt.savefig(output_svg_path, format="svg", dpi=300, bbox_inches="tight")
print(f"Charts saved to: {output_svg_path}")
except Exception as e:
print(f"Error saving SVG file: {e}")
sys.exit(1)
def main():
"""Main function to parse arguments and generate charts."""
parser = argparse.ArgumentParser(
description="Generate TPS and latency charts from BenchBase TPC-C results CSV"
)
parser.add_argument(
"--input-csv", type=str, required=True, help="Path to the input CSV results file"
)
parser.add_argument(
"--output-svg", type=str, required=True, help="Path for the output SVG chart file"
)
parser.add_argument(
"--title-suffix",
type=str,
required=False,
help="Optional suffix to add to chart titles (e.g., 'Warmup', 'Benchmark Phase')",
)
args = parser.parse_args()
# Validate input file exists
if not Path(args.input_csv).exists():
print(f"Error: Input CSV file does not exist: {args.input_csv}")
sys.exit(1)
# Create output directory if it doesn't exist
output_path = Path(args.output_svg)
output_path.parent.mkdir(parents=True, exist_ok=True)
# Load data and generate charts
df = load_results_csv(args.input_csv)
generate_charts(df, args.input_csv, args.output_svg, args.title_suffix)
print(f"Successfully generated charts from {len(df)} data points")
if __name__ == "__main__":
main()

View File

@@ -1,339 +0,0 @@
import argparse
import html
import math
import os
import sys
from pathlib import Path
CONFIGS_DIR = Path("../configs")
SCRIPTS_DIR = Path("../scripts")
# Constants
## TODO increase times after testing
WARMUP_TIME_SECONDS = 1200 # 20 minutes
BENCHMARK_TIME_SECONDS = 3600 # 1 hour
RAMP_STEP_TIME_SECONDS = 300 # 5 minutes
BASE_TERMINALS = 130
TERMINALS_PER_WAREHOUSE = 0.2
OPTIMAL_RATE_FACTOR = 0.7 # 70% of max rate
BATCH_SIZE = 1000
LOADER_THREADS = 4
TRANSACTION_WEIGHTS = "45,43,4,4,4" # NewOrder, Payment, OrderStatus, Delivery, StockLevel
# Ramp-up rate multipliers
RAMP_RATE_FACTORS = [1.5, 1.1, 0.9, 0.7, 0.6, 0.4, 0.6, 0.7, 0.9, 1.1]
# Templates for XML configs
WARMUP_XML = """<?xml version="1.0"?>
<parameters>
<type>POSTGRES</type>
<driver>org.postgresql.Driver</driver>
<url>jdbc:postgresql://{hostname}/neondb?sslmode=require&amp;ApplicationName=tpcc&amp;reWriteBatchedInserts=true</url>
<username>neondb_owner</username>
<password>{password}</password>
<reconnectOnConnectionFailure>true</reconnectOnConnectionFailure>
<isolation>TRANSACTION_READ_COMMITTED</isolation>
<batchsize>{batch_size}</batchsize>
<scalefactor>{warehouses}</scalefactor>
<loaderThreads>0</loaderThreads>
<terminals>{terminals}</terminals>
<works>
<work>
<time>{warmup_time}</time>
<weights>{transaction_weights}</weights>
<rate>unlimited</rate>
<arrival>POISSON</arrival>
<distribution>ZIPFIAN</distribution>
</work>
</works>
<transactiontypes>
<transactiontype><name>NewOrder</name></transactiontype>
<transactiontype><name>Payment</name></transactiontype>
<transactiontype><name>OrderStatus</name></transactiontype>
<transactiontype><name>Delivery</name></transactiontype>
<transactiontype><name>StockLevel</name></transactiontype>
</transactiontypes>
</parameters>
"""
MAX_RATE_XML = """<?xml version="1.0"?>
<parameters>
<type>POSTGRES</type>
<driver>org.postgresql.Driver</driver>
<url>jdbc:postgresql://{hostname}/neondb?sslmode=require&amp;ApplicationName=tpcc&amp;reWriteBatchedInserts=true</url>
<username>neondb_owner</username>
<password>{password}</password>
<reconnectOnConnectionFailure>true</reconnectOnConnectionFailure>
<isolation>TRANSACTION_READ_COMMITTED</isolation>
<batchsize>{batch_size}</batchsize>
<scalefactor>{warehouses}</scalefactor>
<loaderThreads>0</loaderThreads>
<terminals>{terminals}</terminals>
<works>
<work>
<time>{benchmark_time}</time>
<weights>{transaction_weights}</weights>
<rate>unlimited</rate>
<arrival>POISSON</arrival>
<distribution>ZIPFIAN</distribution>
</work>
</works>
<transactiontypes>
<transactiontype><name>NewOrder</name></transactiontype>
<transactiontype><name>Payment</name></transactiontype>
<transactiontype><name>OrderStatus</name></transactiontype>
<transactiontype><name>Delivery</name></transactiontype>
<transactiontype><name>StockLevel</name></transactiontype>
</transactiontypes>
</parameters>
"""
OPT_RATE_XML = """<?xml version="1.0"?>
<parameters>
<type>POSTGRES</type>
<driver>org.postgresql.Driver</driver>
<url>jdbc:postgresql://{hostname}/neondb?sslmode=require&amp;ApplicationName=tpcc&amp;reWriteBatchedInserts=true</url>
<username>neondb_owner</username>
<password>{password}</password>
<reconnectOnConnectionFailure>true</reconnectOnConnectionFailure>
<isolation>TRANSACTION_READ_COMMITTED</isolation>
<batchsize>{batch_size}</batchsize>
<scalefactor>{warehouses}</scalefactor>
<loaderThreads>0</loaderThreads>
<terminals>{terminals}</terminals>
<works>
<work>
<time>{benchmark_time}</time>
<rate>{opt_rate}</rate>
<weights>{transaction_weights}</weights>
<arrival>POISSON</arrival>
<distribution>ZIPFIAN</distribution>
</work>
</works>
<transactiontypes>
<transactiontype><name>NewOrder</name></transactiontype>
<transactiontype><name>Payment</name></transactiontype>
<transactiontype><name>OrderStatus</name></transactiontype>
<transactiontype><name>Delivery</name></transactiontype>
<transactiontype><name>StockLevel</name></transactiontype>
</transactiontypes>
</parameters>
"""
RAMP_UP_XML = """<?xml version="1.0"?>
<parameters>
<type>POSTGRES</type>
<driver>org.postgresql.Driver</driver>
<url>jdbc:postgresql://{hostname}/neondb?sslmode=require&amp;ApplicationName=tpcc&amp;reWriteBatchedInserts=true</url>
<username>neondb_owner</username>
<password>{password}</password>
<reconnectOnConnectionFailure>true</reconnectOnConnectionFailure>
<isolation>TRANSACTION_READ_COMMITTED</isolation>
<batchsize>{batch_size}</batchsize>
<scalefactor>{warehouses}</scalefactor>
<loaderThreads>0</loaderThreads>
<terminals>{terminals}</terminals>
<works>
{works}
</works>
<transactiontypes>
<transactiontype><name>NewOrder</name></transactiontype>
<transactiontype><name>Payment</name></transactiontype>
<transactiontype><name>OrderStatus</name></transactiontype>
<transactiontype><name>Delivery</name></transactiontype>
<transactiontype><name>StockLevel</name></transactiontype>
</transactiontypes>
</parameters>
"""
WORK_TEMPLATE = f""" <work>\n <time>{RAMP_STEP_TIME_SECONDS}</time>\n <rate>{{rate}}</rate>\n <weights>{TRANSACTION_WEIGHTS}</weights>\n <arrival>POISSON</arrival>\n <distribution>ZIPFIAN</distribution>\n </work>\n"""
# Templates for shell scripts
EXECUTE_SCRIPT = """# Create results directories
mkdir -p results_warmup
mkdir -p results_{suffix}
chmod 777 results_warmup results_{suffix}
# Run warmup phase
docker run --network=host --rm \
-v $(pwd)/configs:/configs \
-v $(pwd)/results_warmup:/results \
{docker_image}\
-b tpcc \
-c /configs/execute_{warehouses}_warehouses_warmup.xml \
-d /results \
--create=false --load=false --execute=true
# Run benchmark phase
docker run --network=host --rm \
-v $(pwd)/configs:/configs \
-v $(pwd)/results_{suffix}:/results \
{docker_image}\
-b tpcc \
-c /configs/execute_{warehouses}_warehouses_{suffix}.xml \
-d /results \
--create=false --load=false --execute=true\n"""
LOAD_XML = """<?xml version="1.0"?>
<parameters>
<type>POSTGRES</type>
<driver>org.postgresql.Driver</driver>
<url>jdbc:postgresql://{hostname}/neondb?sslmode=require&amp;ApplicationName=tpcc&amp;reWriteBatchedInserts=true</url>
<username>neondb_owner</username>
<password>{password}</password>
<reconnectOnConnectionFailure>true</reconnectOnConnectionFailure>
<isolation>TRANSACTION_READ_COMMITTED</isolation>
<batchsize>{batch_size}</batchsize>
<scalefactor>{warehouses}</scalefactor>
<loaderThreads>{loader_threads}</loaderThreads>
</parameters>
"""
LOAD_SCRIPT = """# Create results directory for loading
mkdir -p results_load
chmod 777 results_load
docker run --network=host --rm \
-v $(pwd)/configs:/configs \
-v $(pwd)/results_load:/results \
{docker_image}\
-b tpcc \
-c /configs/load_{warehouses}_warehouses.xml \
-d /results \
--create=true --load=true --execute=false\n"""
def write_file(path, content):
path.parent.mkdir(parents=True, exist_ok=True)
try:
with open(path, "w") as f:
f.write(content)
except OSError as e:
print(f"Error writing {path}: {e}")
sys.exit(1)
# If it's a shell script, set executable permission
if str(path).endswith(".sh"):
os.chmod(path, 0o755)
def escape_xml_password(password):
"""Escape XML special characters in password."""
return html.escape(password, quote=True)
def get_docker_arch_tag(runner_arch):
"""Map GitHub Actions runner.arch to Docker image architecture tag."""
arch_mapping = {"X64": "amd64", "ARM64": "arm64"}
return arch_mapping.get(runner_arch, "amd64") # Default to amd64
def main():
parser = argparse.ArgumentParser(description="Generate BenchBase workload configs and scripts.")
parser.add_argument("--warehouses", type=int, required=True, help="Number of warehouses")
parser.add_argument("--max-rate", type=int, required=True, help="Max rate (TPS)")
parser.add_argument("--hostname", type=str, required=True, help="Database hostname")
parser.add_argument("--password", type=str, required=True, help="Database password")
parser.add_argument(
"--runner-arch", type=str, required=True, help="GitHub Actions runner architecture"
)
args = parser.parse_args()
warehouses = args.warehouses
max_rate = args.max_rate
hostname = args.hostname
password = args.password
runner_arch = args.runner_arch
# Escape password for safe XML insertion
escaped_password = escape_xml_password(password)
# Get the appropriate Docker architecture tag
docker_arch = get_docker_arch_tag(runner_arch)
docker_image = f"ghcr.io/neondatabase-labs/benchbase-postgres:latest-{docker_arch}"
opt_rate = math.ceil(max_rate * OPTIMAL_RATE_FACTOR)
# Calculate terminals as next rounded integer of 40% of warehouses
terminals = math.ceil(BASE_TERMINALS + warehouses * TERMINALS_PER_WAREHOUSE)
ramp_rates = [math.ceil(max_rate * factor) for factor in RAMP_RATE_FACTORS]
# Write configs
write_file(
CONFIGS_DIR / f"execute_{warehouses}_warehouses_warmup.xml",
WARMUP_XML.format(
warehouses=warehouses,
hostname=hostname,
password=escaped_password,
terminals=terminals,
batch_size=BATCH_SIZE,
warmup_time=WARMUP_TIME_SECONDS,
transaction_weights=TRANSACTION_WEIGHTS,
),
)
write_file(
CONFIGS_DIR / f"execute_{warehouses}_warehouses_max_rate.xml",
MAX_RATE_XML.format(
warehouses=warehouses,
hostname=hostname,
password=escaped_password,
terminals=terminals,
batch_size=BATCH_SIZE,
benchmark_time=BENCHMARK_TIME_SECONDS,
transaction_weights=TRANSACTION_WEIGHTS,
),
)
write_file(
CONFIGS_DIR / f"execute_{warehouses}_warehouses_opt_rate.xml",
OPT_RATE_XML.format(
warehouses=warehouses,
opt_rate=opt_rate,
hostname=hostname,
password=escaped_password,
terminals=terminals,
batch_size=BATCH_SIZE,
benchmark_time=BENCHMARK_TIME_SECONDS,
transaction_weights=TRANSACTION_WEIGHTS,
),
)
ramp_works = "".join([WORK_TEMPLATE.format(rate=rate) for rate in ramp_rates])
write_file(
CONFIGS_DIR / f"execute_{warehouses}_warehouses_ramp_up.xml",
RAMP_UP_XML.format(
warehouses=warehouses,
works=ramp_works,
hostname=hostname,
password=escaped_password,
terminals=terminals,
batch_size=BATCH_SIZE,
),
)
# Loader config
write_file(
CONFIGS_DIR / f"load_{warehouses}_warehouses.xml",
LOAD_XML.format(
warehouses=warehouses,
hostname=hostname,
password=escaped_password,
batch_size=BATCH_SIZE,
loader_threads=LOADER_THREADS,
),
)
# Write scripts
for suffix in ["max_rate", "opt_rate", "ramp_up"]:
script = EXECUTE_SCRIPT.format(
warehouses=warehouses, suffix=suffix, docker_image=docker_image
)
write_file(SCRIPTS_DIR / f"execute_{warehouses}_warehouses_{suffix}.sh", script)
# Loader script
write_file(
SCRIPTS_DIR / f"load_{warehouses}_warehouses.sh",
LOAD_SCRIPT.format(warehouses=warehouses, docker_image=docker_image),
)
print(f"Generated configs and scripts for {warehouses} warehouses and max rate {max_rate}.")
if __name__ == "__main__":
main()

View File

@@ -1,591 +0,0 @@
#!/usr/bin/env python3
# ruff: noqa
# we exclude the file from ruff because on the github runner we have python 3.9 and ruff
# is running with newer python 3.12 which suggests changes incompatible with python 3.9
"""
Upload BenchBase TPC-C results from summary.json and results.csv files to perf_test_results database.
This script extracts metrics from BenchBase *.summary.json and *.results.csv files and uploads them
to a PostgreSQL database table for performance tracking and analysis.
"""
import argparse
import json
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
import pandas as pd # type: ignore[import-untyped]
import psycopg2
def load_summary_json(json_file_path):
"""Load summary.json file and return parsed data."""
try:
with open(json_file_path) as f:
return json.load(f)
except FileNotFoundError:
print(f"Error: Summary JSON file not found: {json_file_path}")
sys.exit(1)
except json.JSONDecodeError as e:
print(f"Error: Invalid JSON in file {json_file_path}: {e}")
sys.exit(1)
except Exception as e:
print(f"Error loading JSON file {json_file_path}: {e}")
sys.exit(1)
def get_metric_info(metric_name):
"""Get metric unit and report type for a given metric name."""
metrics_config = {
"Throughput": {"unit": "req/s", "report_type": "higher_is_better"},
"Goodput": {"unit": "req/s", "report_type": "higher_is_better"},
"Measured Requests": {"unit": "requests", "report_type": "higher_is_better"},
"95th Percentile Latency": {"unit": "µs", "report_type": "lower_is_better"},
"Maximum Latency": {"unit": "µs", "report_type": "lower_is_better"},
"Median Latency": {"unit": "µs", "report_type": "lower_is_better"},
"Minimum Latency": {"unit": "µs", "report_type": "lower_is_better"},
"25th Percentile Latency": {"unit": "µs", "report_type": "lower_is_better"},
"90th Percentile Latency": {"unit": "µs", "report_type": "lower_is_better"},
"99th Percentile Latency": {"unit": "µs", "report_type": "lower_is_better"},
"75th Percentile Latency": {"unit": "µs", "report_type": "lower_is_better"},
"Average Latency": {"unit": "µs", "report_type": "lower_is_better"},
}
return metrics_config.get(metric_name, {"unit": "", "report_type": "higher_is_better"})
def extract_metrics(summary_data):
"""Extract relevant metrics from summary JSON data."""
metrics = []
# Direct top-level metrics
direct_metrics = {
"Throughput (requests/second)": "Throughput",
"Goodput (requests/second)": "Goodput",
"Measured Requests": "Measured Requests",
}
for json_key, clean_name in direct_metrics.items():
if json_key in summary_data:
metrics.append((clean_name, summary_data[json_key]))
# Latency metrics from nested "Latency Distribution" object
if "Latency Distribution" in summary_data:
latency_data = summary_data["Latency Distribution"]
latency_metrics = {
"95th Percentile Latency (microseconds)": "95th Percentile Latency",
"Maximum Latency (microseconds)": "Maximum Latency",
"Median Latency (microseconds)": "Median Latency",
"Minimum Latency (microseconds)": "Minimum Latency",
"25th Percentile Latency (microseconds)": "25th Percentile Latency",
"90th Percentile Latency (microseconds)": "90th Percentile Latency",
"99th Percentile Latency (microseconds)": "99th Percentile Latency",
"75th Percentile Latency (microseconds)": "75th Percentile Latency",
"Average Latency (microseconds)": "Average Latency",
}
for json_key, clean_name in latency_metrics.items():
if json_key in latency_data:
metrics.append((clean_name, latency_data[json_key]))
return metrics
def build_labels(summary_data, project_id):
"""Build labels JSON object from summary data and project info."""
labels = {}
# Extract required label keys from summary data
label_keys = [
"DBMS Type",
"DBMS Version",
"Benchmark Type",
"Final State",
"isolation",
"scalefactor",
"terminals",
]
for key in label_keys:
if key in summary_data:
labels[key] = summary_data[key]
# Add project_id from workflow
labels["project_id"] = project_id
return labels
def build_suit_name(scalefactor, terminals, run_type, min_cu, max_cu):
"""Build the suit name according to specification."""
return f"benchbase-tpc-c-{scalefactor}-{terminals}-{run_type}-{min_cu}-{max_cu}"
def convert_timestamp_to_utc(timestamp_ms):
"""Convert millisecond timestamp to PostgreSQL-compatible UTC timestamp."""
try:
dt = datetime.fromtimestamp(timestamp_ms / 1000.0, tz=timezone.utc)
return dt.isoformat()
except (ValueError, TypeError) as e:
print(f"Warning: Could not convert timestamp {timestamp_ms}: {e}")
return datetime.now(timezone.utc).isoformat()
def insert_metrics(conn, metrics_data):
"""Insert metrics data into the perf_test_results table."""
insert_query = """
INSERT INTO perf_test_results
(suit, revision, platform, metric_name, metric_value, metric_unit,
metric_report_type, recorded_at_timestamp, labels)
VALUES (%(suit)s, %(revision)s, %(platform)s, %(metric_name)s, %(metric_value)s,
%(metric_unit)s, %(metric_report_type)s, %(recorded_at_timestamp)s, %(labels)s)
"""
try:
with conn.cursor() as cursor:
cursor.executemany(insert_query, metrics_data)
conn.commit()
print(f"Successfully inserted {len(metrics_data)} metrics into perf_test_results")
# Log some sample data for verification
if metrics_data:
print(
f"Sample metric: {metrics_data[0]['metric_name']} = {metrics_data[0]['metric_value']} {metrics_data[0]['metric_unit']}"
)
except Exception as e:
print(f"Error inserting metrics into database: {e}")
sys.exit(1)
def create_benchbase_results_details_table(conn):
"""Create benchbase_results_details table if it doesn't exist."""
create_table_query = """
CREATE TABLE IF NOT EXISTS benchbase_results_details (
id BIGSERIAL PRIMARY KEY,
suit TEXT,
revision CHAR(40),
platform TEXT,
recorded_at_timestamp TIMESTAMP WITH TIME ZONE,
requests_per_second NUMERIC,
average_latency_ms NUMERIC,
minimum_latency_ms NUMERIC,
p25_latency_ms NUMERIC,
median_latency_ms NUMERIC,
p75_latency_ms NUMERIC,
p90_latency_ms NUMERIC,
p95_latency_ms NUMERIC,
p99_latency_ms NUMERIC,
maximum_latency_ms NUMERIC
);
CREATE INDEX IF NOT EXISTS benchbase_results_details_recorded_at_timestamp_idx
ON benchbase_results_details USING BRIN (recorded_at_timestamp);
CREATE INDEX IF NOT EXISTS benchbase_results_details_suit_idx
ON benchbase_results_details USING BTREE (suit text_pattern_ops);
"""
try:
with conn.cursor() as cursor:
cursor.execute(create_table_query)
conn.commit()
print("Successfully created/verified benchbase_results_details table")
except Exception as e:
print(f"Error creating benchbase_results_details table: {e}")
sys.exit(1)
def process_csv_results(csv_file_path, start_timestamp_ms, suit, revision, platform):
"""Process CSV results and return data for database insertion."""
try:
# Read CSV file
df = pd.read_csv(csv_file_path)
# Validate required columns exist
required_columns = [
"Time (seconds)",
"Throughput (requests/second)",
"Average Latency (millisecond)",
"Minimum Latency (millisecond)",
"25th Percentile Latency (millisecond)",
"Median Latency (millisecond)",
"75th Percentile Latency (millisecond)",
"90th Percentile Latency (millisecond)",
"95th Percentile Latency (millisecond)",
"99th Percentile Latency (millisecond)",
"Maximum Latency (millisecond)",
]
missing_columns = [col for col in required_columns if col not in df.columns]
if missing_columns:
print(f"Error: Missing required columns in CSV: {missing_columns}")
return []
csv_data = []
for _, row in df.iterrows():
# Calculate timestamp: start_timestamp_ms + (time_seconds * 1000)
time_seconds = row["Time (seconds)"]
row_timestamp_ms = start_timestamp_ms + (time_seconds * 1000)
# Convert to UTC timestamp
row_timestamp = datetime.fromtimestamp(
row_timestamp_ms / 1000.0, tz=timezone.utc
).isoformat()
csv_row = {
"suit": suit,
"revision": revision,
"platform": platform,
"recorded_at_timestamp": row_timestamp,
"requests_per_second": float(row["Throughput (requests/second)"]),
"average_latency_ms": float(row["Average Latency (millisecond)"]),
"minimum_latency_ms": float(row["Minimum Latency (millisecond)"]),
"p25_latency_ms": float(row["25th Percentile Latency (millisecond)"]),
"median_latency_ms": float(row["Median Latency (millisecond)"]),
"p75_latency_ms": float(row["75th Percentile Latency (millisecond)"]),
"p90_latency_ms": float(row["90th Percentile Latency (millisecond)"]),
"p95_latency_ms": float(row["95th Percentile Latency (millisecond)"]),
"p99_latency_ms": float(row["99th Percentile Latency (millisecond)"]),
"maximum_latency_ms": float(row["Maximum Latency (millisecond)"]),
}
csv_data.append(csv_row)
print(f"Processed {len(csv_data)} rows from CSV file")
return csv_data
except FileNotFoundError:
print(f"Error: CSV file not found: {csv_file_path}")
return []
except Exception as e:
print(f"Error processing CSV file {csv_file_path}: {e}")
return []
def insert_csv_results(conn, csv_data):
"""Insert CSV results into benchbase_results_details table."""
if not csv_data:
print("No CSV data to insert")
return
insert_query = """
INSERT INTO benchbase_results_details
(suit, revision, platform, recorded_at_timestamp, requests_per_second,
average_latency_ms, minimum_latency_ms, p25_latency_ms, median_latency_ms,
p75_latency_ms, p90_latency_ms, p95_latency_ms, p99_latency_ms, maximum_latency_ms)
VALUES (%(suit)s, %(revision)s, %(platform)s, %(recorded_at_timestamp)s, %(requests_per_second)s,
%(average_latency_ms)s, %(minimum_latency_ms)s, %(p25_latency_ms)s, %(median_latency_ms)s,
%(p75_latency_ms)s, %(p90_latency_ms)s, %(p95_latency_ms)s, %(p99_latency_ms)s, %(maximum_latency_ms)s)
"""
try:
with conn.cursor() as cursor:
cursor.executemany(insert_query, csv_data)
conn.commit()
print(
f"Successfully inserted {len(csv_data)} detailed results into benchbase_results_details"
)
# Log some sample data for verification
sample = csv_data[0]
print(
f"Sample detail: {sample['requests_per_second']} req/s at {sample['recorded_at_timestamp']}"
)
except Exception as e:
print(f"Error inserting CSV results into database: {e}")
sys.exit(1)
def parse_load_log(log_file_path, scalefactor):
"""Parse load log file and extract load metrics."""
try:
with open(log_file_path) as f:
log_content = f.read()
# Regex patterns to match the timestamp lines
loading_pattern = r"\[INFO \] (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),\d{3}.*Loading data into TPCC database"
finished_pattern = r"\[INFO \] (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),\d{3}.*Finished loading data into TPCC database"
loading_match = re.search(loading_pattern, log_content)
finished_match = re.search(finished_pattern, log_content)
if not loading_match or not finished_match:
print(f"Warning: Could not find loading timestamps in log file {log_file_path}")
return None
# Parse timestamps
loading_time = datetime.strptime(loading_match.group(1), "%Y-%m-%d %H:%M:%S")
finished_time = datetime.strptime(finished_match.group(1), "%Y-%m-%d %H:%M:%S")
# Calculate duration in seconds
duration_seconds = (finished_time - loading_time).total_seconds()
# Calculate throughput: scalefactor/warehouses: 10 warehouses is approx. 1 GB of data
load_throughput = (scalefactor * 1024 / 10.0) / duration_seconds
# Convert end time to UTC timestamp for database
finished_time_utc = finished_time.replace(tzinfo=timezone.utc).isoformat()
print(f"Load metrics: Duration={duration_seconds}s, Throughput={load_throughput:.2f} MB/s")
return {
"duration_seconds": duration_seconds,
"throughput_mb_per_sec": load_throughput,
"end_timestamp": finished_time_utc,
}
except FileNotFoundError:
print(f"Warning: Load log file not found: {log_file_path}")
return None
except Exception as e:
print(f"Error parsing load log file {log_file_path}: {e}")
return None
def insert_load_metrics(conn, load_metrics, suit, revision, platform, labels_json):
"""Insert load metrics into perf_test_results table."""
if not load_metrics:
print("No load metrics to insert")
return
load_metrics_data = [
{
"suit": suit,
"revision": revision,
"platform": platform,
"metric_name": "load_duration_seconds",
"metric_value": load_metrics["duration_seconds"],
"metric_unit": "seconds",
"metric_report_type": "lower_is_better",
"recorded_at_timestamp": load_metrics["end_timestamp"],
"labels": labels_json,
},
{
"suit": suit,
"revision": revision,
"platform": platform,
"metric_name": "load_throughput",
"metric_value": load_metrics["throughput_mb_per_sec"],
"metric_unit": "MB/second",
"metric_report_type": "higher_is_better",
"recorded_at_timestamp": load_metrics["end_timestamp"],
"labels": labels_json,
},
]
insert_query = """
INSERT INTO perf_test_results
(suit, revision, platform, metric_name, metric_value, metric_unit,
metric_report_type, recorded_at_timestamp, labels)
VALUES (%(suit)s, %(revision)s, %(platform)s, %(metric_name)s, %(metric_value)s,
%(metric_unit)s, %(metric_report_type)s, %(recorded_at_timestamp)s, %(labels)s)
"""
try:
with conn.cursor() as cursor:
cursor.executemany(insert_query, load_metrics_data)
conn.commit()
print(f"Successfully inserted {len(load_metrics_data)} load metrics into perf_test_results")
except Exception as e:
print(f"Error inserting load metrics into database: {e}")
sys.exit(1)
def main():
"""Main function to parse arguments and upload results."""
parser = argparse.ArgumentParser(
description="Upload BenchBase TPC-C results to perf_test_results database"
)
parser.add_argument(
"--summary-json", type=str, required=False, help="Path to the summary.json file"
)
parser.add_argument(
"--run-type",
type=str,
required=True,
choices=["warmup", "opt-rate", "ramp-up", "load"],
help="Type of benchmark run",
)
parser.add_argument("--min-cu", type=float, required=True, help="Minimum compute units")
parser.add_argument("--max-cu", type=float, required=True, help="Maximum compute units")
parser.add_argument("--project-id", type=str, required=True, help="Neon project ID")
parser.add_argument(
"--revision", type=str, required=True, help="Git commit hash (40 characters)"
)
parser.add_argument(
"--connection-string", type=str, required=True, help="PostgreSQL connection string"
)
parser.add_argument(
"--results-csv",
type=str,
required=False,
help="Path to the results.csv file for detailed metrics upload",
)
parser.add_argument(
"--load-log",
type=str,
required=False,
help="Path to the load log file for load phase metrics",
)
parser.add_argument(
"--warehouses",
type=int,
required=False,
help="Number of warehouses (scalefactor) for load metrics calculation",
)
args = parser.parse_args()
# Validate inputs
if args.summary_json and not Path(args.summary_json).exists():
print(f"Error: Summary JSON file does not exist: {args.summary_json}")
sys.exit(1)
if not args.summary_json and not args.load_log:
print("Error: Either summary JSON or load log file must be provided")
sys.exit(1)
if len(args.revision) != 40:
print(f"Warning: Revision should be 40 characters, got {len(args.revision)}")
# Load and process summary data if provided
summary_data = None
metrics = []
if args.summary_json:
summary_data = load_summary_json(args.summary_json)
metrics = extract_metrics(summary_data)
if not metrics:
print("Warning: No metrics found in summary JSON")
# Build common data for all metrics
if summary_data:
scalefactor = summary_data.get("scalefactor", "unknown")
terminals = summary_data.get("terminals", "unknown")
labels = build_labels(summary_data, args.project_id)
else:
# For load-only processing, use warehouses argument as scalefactor
scalefactor = args.warehouses if args.warehouses else "unknown"
terminals = "unknown"
labels = {"project_id": args.project_id}
suit = build_suit_name(scalefactor, terminals, args.run_type, args.min_cu, args.max_cu)
platform = f"prod-us-east-2-{args.project_id}"
# Convert timestamp - only needed for summary metrics and CSV processing
current_timestamp_ms = None
start_timestamp_ms = None
recorded_at = None
if summary_data:
current_timestamp_ms = summary_data.get("Current Timestamp (milliseconds)")
start_timestamp_ms = summary_data.get("Start timestamp (milliseconds)")
if current_timestamp_ms:
recorded_at = convert_timestamp_to_utc(current_timestamp_ms)
else:
print("Warning: No timestamp found in JSON, using current time")
recorded_at = datetime.now(timezone.utc).isoformat()
if not start_timestamp_ms:
print("Warning: No start timestamp found in JSON, CSV upload may be incorrect")
start_timestamp_ms = (
current_timestamp_ms or datetime.now(timezone.utc).timestamp() * 1000
)
# Print Grafana dashboard link for cross-service endpoint debugging
if start_timestamp_ms and current_timestamp_ms:
grafana_url = (
f"https://neonprod.grafana.net/d/cdya0okb81zwga/cross-service-endpoint-debugging"
f"?orgId=1&from={int(start_timestamp_ms)}&to={int(current_timestamp_ms)}"
f"&timezone=utc&var-env=prod&var-input_project_id={args.project_id}"
)
print(f'Cross service endpoint dashboard for "{args.run_type}" phase: {grafana_url}')
# Prepare metrics data for database insertion (only if we have summary metrics)
metrics_data = []
if metrics and recorded_at:
for metric_name, metric_value in metrics:
metric_info = get_metric_info(metric_name)
row = {
"suit": suit,
"revision": args.revision,
"platform": platform,
"metric_name": metric_name,
"metric_value": float(metric_value), # Ensure numeric type
"metric_unit": metric_info["unit"],
"metric_report_type": metric_info["report_type"],
"recorded_at_timestamp": recorded_at,
"labels": json.dumps(labels), # Convert to JSON string for JSONB column
}
metrics_data.append(row)
print(f"Prepared {len(metrics_data)} summary metrics for upload to database")
print(f"Suit: {suit}")
print(f"Platform: {platform}")
# Connect to database and insert metrics
try:
conn = psycopg2.connect(args.connection_string)
# Insert summary metrics into perf_test_results (if any)
if metrics_data:
insert_metrics(conn, metrics_data)
else:
print("No summary metrics to upload")
# Process and insert detailed CSV results if provided
if args.results_csv:
print(f"Processing detailed CSV results from: {args.results_csv}")
# Create table if it doesn't exist
create_benchbase_results_details_table(conn)
# Process CSV data
csv_data = process_csv_results(
args.results_csv, start_timestamp_ms, suit, args.revision, platform
)
# Insert CSV data
if csv_data:
insert_csv_results(conn, csv_data)
else:
print("No CSV data to upload")
else:
print("No CSV file provided, skipping detailed results upload")
# Process and insert load metrics if provided
if args.load_log:
print(f"Processing load metrics from: {args.load_log}")
# Parse load log and extract metrics
load_metrics = parse_load_log(args.load_log, scalefactor)
# Insert load metrics
if load_metrics:
insert_load_metrics(
conn, load_metrics, suit, args.revision, platform, json.dumps(labels)
)
else:
print("No load metrics to upload")
else:
print("No load log file provided, skipping load metrics upload")
conn.close()
print("Database upload completed successfully")
except psycopg2.Error as e:
print(f"Database connection/query error: {e}")
sys.exit(1)
except Exception as e:
print(f"Unexpected error: {e}")
sys.exit(1)
if __name__ == "__main__":
main()

View File

@@ -26,7 +26,7 @@ def test_compute_pageserver_connection_stress(neon_env_builder: NeonEnvBuilder):
# Enable failpoint before starting everything else up so that we exercise the retry
# on fetching basebackup
pageserver_http = env.pageserver.http_client()
pageserver_http.configure_failpoints(("simulated-bad-compute-connection", "20%return(15)"))
pageserver_http.configure_failpoints(("simulated-bad-compute-connection", "50%return(15)"))
env.create_branch("test_compute_pageserver_connection_stress")
endpoint = env.endpoints.create_start("test_compute_pageserver_connection_stress")

View File

@@ -3,35 +3,14 @@ from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING
import pytest
from fixtures.log_helper import log
from fixtures.neon_fixtures import NeonEnvBuilder
from fixtures.remote_storage import RemoteStorageKind
if TYPE_CHECKING:
from fixtures.neon_fixtures import Endpoint, NeonEnvBuilder
from fixtures.neon_fixtures import NeonEnvBuilder
def reconfigure_endpoint(endpoint: Endpoint, pageserver_id: int, use_explicit_reconfigure: bool):
# It's important that we always update config.json before issuing any reconfigure requests
# to make sure that PG-initiated config refresh doesn't mess things up by reverting to the old config.
endpoint.update_pageservers_in_config(pageserver_id=pageserver_id)
# PG will automatically refresh its configuration if it detects connectivity issues with pageservers.
# We also allow the test to explicitly request a reconfigure so that the test can be sure that the
# endpoint is running with the latest configuration.
#
# Note that explicit reconfiguration is not required for the system to function or for this test to pass.
# It is kept for reference as this is how this test used to work before the capability of initiating
# configuration refreshes was added to compute nodes.
if use_explicit_reconfigure:
endpoint.reconfigure(pageserver_id=pageserver_id)
@pytest.mark.parametrize("use_explicit_reconfigure_for_failover", [False, True])
def test_change_pageserver(
neon_env_builder: NeonEnvBuilder, use_explicit_reconfigure_for_failover: bool
):
def test_change_pageserver(neon_env_builder: NeonEnvBuilder):
"""
A relatively low level test of reconfiguring a compute's pageserver at runtime. Usually this
is all done via the storage controller, but this test will disable the storage controller's compute
@@ -93,10 +72,7 @@ def test_change_pageserver(
execute("SELECT count(*) FROM foo")
assert fetchone() == (100000,)
# Reconfigure the endpoint to use the alt pageserver. We issue an explicit reconfigure request here
# regardless of test mode as this is testing the externally driven reconfiguration scenario, not the
# compute-initiated reconfiguration scenario upon detecting failures.
reconfigure_endpoint(endpoint, pageserver_id=alt_pageserver_id, use_explicit_reconfigure=True)
endpoint.reconfigure(pageserver_id=alt_pageserver_id)
# Verify that the neon.pageserver_connstring GUC is set to the correct thing
execute("SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'")
@@ -124,12 +100,6 @@ def test_change_pageserver(
env.storage_controller.node_configure(env.pageservers[1].id, {"availability": "Offline"})
env.storage_controller.reconcile_until_idle()
reconfigure_endpoint(
endpoint,
pageserver_id=env.pageservers[0].id,
use_explicit_reconfigure=use_explicit_reconfigure_for_failover,
)
endpoint.reconfigure(pageserver_id=env.pageservers[0].id)
execute("SELECT count(*) FROM foo")
@@ -146,11 +116,7 @@ def test_change_pageserver(
await asyncio.sleep(
1
) # Sleep for 1 second just to make sure we actually started our count(*) query
reconfigure_endpoint(
endpoint,
pageserver_id=env.pageservers[1].id,
use_explicit_reconfigure=use_explicit_reconfigure_for_failover,
)
endpoint.reconfigure(pageserver_id=env.pageservers[1].id)
def execute_count():
execute("SELECT count(*) FROM FOO")

View File

@@ -1,369 +0,0 @@
from __future__ import annotations
import json
import os
import shutil
import subprocess
import threading
import time
from http.server import BaseHTTPRequestHandler, HTTPServer
from typing import TYPE_CHECKING
import requests
from fixtures.log_helper import log
from typing_extensions import override
if TYPE_CHECKING:
from typing import Any
from fixtures.common_types import TenantId, TimelineId
from fixtures.neon_fixtures import NeonEnv
from fixtures.port_distributor import PortDistributor
def launch_compute_ctl(
env: NeonEnv,
endpoint_name: str,
external_http_port: int,
internal_http_port: int,
pg_port: int,
control_plane_port: int,
) -> subprocess.Popen[str]:
"""
Helper function to launch compute_ctl process with common configuration.
Returns the Popen process object.
"""
# Create endpoint directory structure following the standard pattern
endpoint_path = env.repo_dir / "endpoints" / endpoint_name
# Clean up any existing endpoint directory to avoid conflicts
if endpoint_path.exists():
shutil.rmtree(endpoint_path)
endpoint_path.mkdir(mode=0o755, parents=True, exist_ok=True)
# pgdata path - compute_ctl will create this directory during basebackup
pgdata_path = endpoint_path / "pgdata"
# Create log file in endpoint directory
log_file = endpoint_path / "compute.log"
log_handle = open(log_file, "w")
# Start compute_ctl pointing to our control plane
compute_ctl_path = env.neon_binpath / "compute_ctl"
connstr = f"postgresql://cloud_admin@localhost:{pg_port}/postgres"
# Find postgres binary path
pg_bin_path = env.pg_distrib_dir / env.pg_version.v_prefixed / "bin" / "postgres"
pg_lib_path = env.pg_distrib_dir / env.pg_version.v_prefixed / "lib"
env_vars = {
"INSTANCE_ID": "lakebase-instance-id",
"LD_LIBRARY_PATH": str(pg_lib_path), # Linux, etc.
"DYLD_LIBRARY_PATH": str(pg_lib_path), # macOS
}
cmd = [
str(compute_ctl_path),
"--external-http-port",
str(external_http_port),
"--internal-http-port",
str(internal_http_port),
"--pgdata",
str(pgdata_path),
"--connstr",
connstr,
"--pgbin",
str(pg_bin_path),
"--compute-id",
endpoint_name, # Use endpoint_name as compute-id
"--control-plane-uri",
f"http://127.0.0.1:{control_plane_port}",
"--lakebase-mode",
"true",
]
print(f"Launching compute_ctl with command: {cmd}")
# Start compute_ctl
process = subprocess.Popen(
cmd,
env=env_vars,
stdout=log_handle,
stderr=subprocess.STDOUT, # Combine stderr with stdout
text=True,
)
return process
def wait_for_compute_status(
compute_process: subprocess.Popen[str],
http_port: int,
expected_status: str,
timeout_seconds: int = 10,
) -> None:
"""
Wait for compute_ctl to reach the expected status.
Raises an exception if timeout is reached or process exits unexpectedly.
"""
start_time = time.time()
while time.time() - start_time < timeout_seconds:
try:
# Try to connect to the HTTP endpoint
response = requests.get(f"http://localhost:{http_port}/status", timeout=0.5)
if response.status_code == 200:
status_json = response.json()
# Check if it's in expected status
if status_json.get("status") == expected_status:
return
except (requests.ConnectionError, requests.Timeout):
pass
# Check if process has exited
if compute_process.poll() is not None:
raise Exception(
f"compute_ctl exited unexpectedly with code {compute_process.returncode}."
)
time.sleep(0.5)
# Timeout reached
compute_process.terminate()
raise Exception(
f"compute_ctl failed to reach {expected_status} status within {timeout_seconds} seconds."
)
class EmptySpecHandler(BaseHTTPRequestHandler):
"""HTTP handler that returns an Empty compute spec response"""
def do_GET(self):
if self.path.startswith("/compute/api/v2/computes/") and self.path.endswith("/spec"):
# Return empty status which will put compute in Empty state
response: dict[str, Any] = {
"status": "empty",
"spec": None,
"compute_ctl_config": {"jwks": {"keys": []}},
}
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(response).encode())
else:
self.send_error(404)
@override
def log_message(self, format: str, *args: Any):
# Suppress request logging
pass
def test_compute_terminate_empty(neon_simple_env: NeonEnv, port_distributor: PortDistributor):
"""
Test that terminating a compute in Empty status works correctly.
This tests the bug fix where terminating an Empty compute would hang
waiting for a non-existent postgres process to terminate.
"""
env = neon_simple_env
# Get ports for our test
control_plane_port = port_distributor.get_port()
external_http_port = port_distributor.get_port()
internal_http_port = port_distributor.get_port()
pg_port = port_distributor.get_port()
# Start a simple HTTP server that will serve the Empty spec
server = HTTPServer(("127.0.0.1", control_plane_port), EmptySpecHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
compute_process = None
try:
# Start compute_ctl with ephemeral tenant ID
compute_process = launch_compute_ctl(
env,
"test-empty-compute",
external_http_port,
internal_http_port,
pg_port,
control_plane_port,
)
# Wait for compute_ctl to start and report "empty" status
wait_for_compute_status(compute_process, external_http_port, "empty")
# Now send terminate request
response = requests.post(f"http://localhost:{external_http_port}/terminate")
# Verify that the termination request sends back a 200 OK response and is not abruptly terminated.
assert response.status_code == 200, (
f"Expected 200 OK, got {response.status_code}: {response.text}"
)
# Wait for compute_ctl to exit
exit_code = compute_process.wait(timeout=10)
assert exit_code == 0, f"compute_ctl exited with non-zero code: {exit_code}"
finally:
# Clean up
server.shutdown()
if compute_process and compute_process.poll() is None:
compute_process.terminate()
compute_process.wait()
class SwitchableConfigHandler(BaseHTTPRequestHandler):
"""HTTP handler that can switch between normal compute configs and compute configs without specs"""
return_empty_spec: bool = False
tenant_id: TenantId | None = None
timeline_id: TimelineId | None = None
pageserver_port: int | None = None
safekeeper_connstrs: list[str] | None = None
def do_GET(self):
if self.path.startswith("/compute/api/v2/computes/") and self.path.endswith("/spec"):
if self.return_empty_spec:
# Return empty status
response: dict[str, object | None] = {
"status": "empty",
"spec": None,
"compute_ctl_config": {
"jwks": {"keys": []},
},
}
else:
# Return normal attached spec
response = {
"status": "attached",
"spec": {
"format_version": 1.0,
"cluster": {
"roles": [],
"databases": [],
"postgresql_conf": "shared_preload_libraries='neon'",
},
"tenant_id": str(self.tenant_id) if self.tenant_id else "",
"timeline_id": str(self.timeline_id) if self.timeline_id else "",
"pageserver_connstring": f"postgres://no_user@localhost:{self.pageserver_port}"
if self.pageserver_port
else "",
"safekeeper_connstrings": self.safekeeper_connstrs or [],
"mode": "Primary",
"skip_pg_catalog_updates": True,
"reconfigure_concurrency": 1,
"suspend_timeout_seconds": -1,
},
"compute_ctl_config": {
"jwks": {"keys": []},
},
}
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(response).encode())
else:
self.send_error(404)
@override
def log_message(self, format: str, *args: Any):
# Suppress request logging
pass
def test_compute_empty_spec_during_refresh_configuration(
neon_simple_env: NeonEnv, port_distributor: PortDistributor
):
"""
Test that compute exits when it receives an empty spec during refresh configuration state.
This test:
1. Start compute with a normal spec
2. Change the spec handler to return empty spec
3. Trigger some condition to force compute to refresh configuration
4. Verify that compute_ctl exits
"""
env = neon_simple_env
# Get ports for our test
control_plane_port = port_distributor.get_port()
external_http_port = port_distributor.get_port()
internal_http_port = port_distributor.get_port()
pg_port = port_distributor.get_port()
# Set up handler class variables
SwitchableConfigHandler.tenant_id = env.initial_tenant
SwitchableConfigHandler.timeline_id = env.initial_timeline
SwitchableConfigHandler.pageserver_port = env.pageserver.service_port.pg
# Convert comma-separated string to list
safekeeper_connstrs = env.get_safekeeper_connstrs()
if safekeeper_connstrs:
SwitchableConfigHandler.safekeeper_connstrs = safekeeper_connstrs.split(",")
else:
SwitchableConfigHandler.safekeeper_connstrs = []
SwitchableConfigHandler.return_empty_spec = False # Start with normal spec
# Start HTTP server with switchable spec handler
server = HTTPServer(("127.0.0.1", control_plane_port), SwitchableConfigHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
compute_process = None
try:
# Start compute_ctl with tenant and timeline IDs
# Use a unique endpoint name to avoid conflicts
endpoint_name = f"test-refresh-compute-{os.getpid()}"
compute_process = launch_compute_ctl(
env,
endpoint_name,
external_http_port,
internal_http_port,
pg_port,
control_plane_port,
)
# Wait for compute_ctl to start and report "running" status
wait_for_compute_status(compute_process, external_http_port, "running", timeout_seconds=30)
log.info("Compute is running. Now returning empty spec and trigger configuration refresh.")
# Switch spec fetch handler to return empty spec
SwitchableConfigHandler.return_empty_spec = True
# Trigger a configuration refresh
try:
requests.post(f"http://localhost:{internal_http_port}/refresh_configuration")
except requests.RequestException as e:
log.info(f"Call to /refresh_configuration failed: {e}")
log.info(
"Ignoring the error, assuming that compute_ctl is already refreshing or has exited"
)
# Wait for compute_ctl to exit (it should exit when it gets an empty spec during refresh)
exit_start_time = time.time()
while time.time() - exit_start_time < 30:
if compute_process.poll() is not None:
# Process exited
break
time.sleep(0.5)
# Verify that compute_ctl exited
exit_code = compute_process.poll()
if exit_code is None:
compute_process.terminate()
raise Exception("compute_ctl did not exit after receiving empty spec.")
# The exit code might not be 0 in this case since it's an unexpected termination
# but we mainly care that it did exit
assert exit_code is not None, "compute_ctl should have exited"
finally:
# Clean up
server.shutdown()
if compute_process and compute_process.poll() is None:
compute_process.terminate()
compute_process.wait()

View File

@@ -1,137 +0,0 @@
import json
import shutil
from fixtures.common_types import TenantShardId
from fixtures.log_helper import log
from fixtures.metrics import parse_metrics
from fixtures.neon_fixtures import Endpoint, NeonEnvBuilder, NeonPageserver
from requests.exceptions import ConnectionError
# Helper function to attempt reconfiguration of the compute to point to a new pageserver. Note that in these tests,
# we don't expect the reconfiguration attempts to go through, as we will be pointing the compute at a "wrong" pageserver.
def _attempt_reconfiguration(endpoint: Endpoint, new_pageserver_id: int, timeout_sec: float):
try:
endpoint.reconfigure(pageserver_id=new_pageserver_id, timeout_sec=timeout_sec)
except Exception as e:
log.info(f"reconfiguration failed with exception {e}")
pass
def read_misrouted_metric_value(pageserver: NeonPageserver) -> float:
return (
pageserver.http_client()
.get_metrics()
.query_one("pageserver_misrouted_pagestream_requests_total")
.value
)
def read_request_error_metric_value(endpoint: Endpoint) -> float:
return (
parse_metrics(endpoint.http_client().metrics())
.query_one("pg_cctl_pagestream_request_errors_total")
.value
)
def test_misrouted_to_secondary(
neon_env_builder: NeonEnvBuilder,
):
"""
Tests that the following metrics are incremented when compute tries to talk to a secondary pageserver:
- On pageserver receiving the request: pageserver_misrouted_pagestream_requests_total
- On compute: pg_cctl_pagestream_request_errors_total
"""
neon_env_builder.num_pageservers = 2
env = neon_env_builder.init_configs()
env.broker.start()
env.storage_controller.start()
for ps in env.pageservers:
ps.start()
for sk in env.safekeepers:
sk.start()
# Create a tenant that has one primary and one secondary. Due to primary/secondary placement constraints,
# the primary and secondary pageservers will be different.
tenant_id, _ = env.create_tenant(shard_count=1, placement_policy=json.dumps({"Attached": 1}))
endpoint = env.endpoints.create(
"main", tenant_id=tenant_id, config_lines=["neon.lakebase_mode = true"]
)
endpoint.respec(skip_pg_catalog_updates=False)
endpoint.start()
# Get the primary pageserver serving the zero shard of the tenant, and detach it from the primary pageserver.
# This test operation configures tenant directly on the pageserver/does not go through the storage controller,
# so the compute does not get any notifications and will keep pointing at the detached pageserver.
tenant_zero_shard = TenantShardId(tenant_id, shard_number=0, shard_count=1)
primary_ps = env.get_tenant_pageserver(tenant_zero_shard)
secondary_ps = (
env.pageservers[1] if primary_ps.id == env.pageservers[0].id else env.pageservers[0]
)
# Now try to point the compute at the pageserver that is acting as secondary for the tenant. Test that the metrics
# on both compute_ctl and the pageserver register the misrouted requests following the reconfiguration attempt.
assert read_misrouted_metric_value(secondary_ps) == 0
assert read_request_error_metric_value(endpoint) == 0
_attempt_reconfiguration(endpoint, new_pageserver_id=secondary_ps.id, timeout_sec=2.0)
assert read_misrouted_metric_value(secondary_ps) > 0
try:
assert read_request_error_metric_value(endpoint) > 0
except ConnectionError:
# When configuring PG to use misconfigured pageserver, PG will cancel the query after certain number of failed
# reconfigure attempts. This will cause compute_ctl to exit.
log.info("Cannot connect to PG, ignoring")
pass
def test_misrouted_to_ps_not_hosting_tenant(
neon_env_builder: NeonEnvBuilder,
):
"""
Tests that the following metrics are incremented when compute tries to talk to a pageserver that does not host the tenant:
- On pageserver receiving the request: pageserver_misrouted_pagestream_requests_total
- On compute: pg_cctl_pagestream_request_errors_total
"""
neon_env_builder.num_pageservers = 2
env = neon_env_builder.init_configs()
env.broker.start()
env.storage_controller.start(handle_ps_local_disk_loss=False)
for ps in env.pageservers:
ps.start()
for sk in env.safekeepers:
sk.start()
tenant_id, _ = env.create_tenant(shard_count=1)
endpoint = env.endpoints.create(
"main", tenant_id=tenant_id, config_lines=["neon.lakebase_mode = true"]
)
endpoint.respec(skip_pg_catalog_updates=False)
endpoint.start()
tenant_ps_id = env.get_tenant_pageserver(
TenantShardId(tenant_id, shard_number=0, shard_count=1)
).id
non_hosting_ps = (
env.pageservers[1] if tenant_ps_id == env.pageservers[0].id else env.pageservers[0]
)
# Clear the disk of the non-hosting PS to make sure that it indeed doesn't have any information about the tenant.
non_hosting_ps.stop(immediate=True)
shutil.rmtree(non_hosting_ps.tenant_dir())
non_hosting_ps.start()
# Now try to point the compute to the non-hosting pageserver. Test that the metrics
# on both compute_ctl and the pageserver register the misrouted requests following the reconfiguration attempt.
assert read_misrouted_metric_value(non_hosting_ps) == 0
assert read_request_error_metric_value(endpoint) == 0
_attempt_reconfiguration(endpoint, new_pageserver_id=non_hosting_ps.id, timeout_sec=2.0)
assert read_misrouted_metric_value(non_hosting_ps) > 0
try:
assert read_request_error_metric_value(endpoint) > 0
except ConnectionError:
# When configuring PG to use misconfigured pageserver, PG will cancel the query after certain number of failed
# reconfigure attempts. This will cause compute_ctl to exit.
log.info("Cannot connect to PG, ignoring")
pass

View File

@@ -133,9 +133,6 @@ def test_hot_standby_gc(neon_env_builder: NeonEnvBuilder, pause_apply: bool):
tenant_conf = {
# set PITR interval to be small, so we can do GC
"pitr_interval": "0 s",
# we want to control gc and checkpoint frequency precisely
"gc_period": "0s",
"compaction_period": "0s",
}
env = neon_env_builder.init_start(initial_tenant_conf=tenant_conf)
timeline_id = env.initial_timeline
@@ -189,23 +186,6 @@ def test_hot_standby_gc(neon_env_builder: NeonEnvBuilder, pause_apply: bool):
client = pageserver.http_client()
client.timeline_checkpoint(tenant_shard_id, timeline_id)
client.timeline_compact(tenant_shard_id, timeline_id)
# Wait for standby horizon to get propagated.
# This shouldn't be necessary, but the current mechanism for
# standby_horizon propagation is imperfect. Detailed
# description in https://databricks.atlassian.net/browse/LKB-2499
while True:
val = client.get_metric_value(
"pageserver_standby_horizon",
{
"tenant_id": str(tenant_shard_id.tenant_id),
"shard_id": str(tenant_shard_id.shard_index),
"timeline_id": str(timeline_id),
},
)
log.info("waiting for next standby_horizon push from safekeeper, {val=}")
if val != 0:
break
time.sleep(0.1)
client.timeline_gc(tenant_shard_id, timeline_id, 0)
# Re-execute the query. The GetPage requests that this

View File

@@ -124,6 +124,7 @@ def test_readonly_node(neon_simple_env: NeonEnv):
)
@pytest.mark.repeat(100)
def test_readonly_node_gc(neon_env_builder: NeonEnvBuilder):
"""
Test static endpoint is protected from GC by acquiring and renewing lsn leases.

View File

@@ -1751,15 +1751,14 @@ def test_back_pressure_per_shard(neon_env_builder: NeonEnvBuilder):
"max_replication_apply_lag = 0",
"max_replication_flush_lag = 15MB",
"neon.max_cluster_size = 10GB",
"neon.lakebase_mode = true",
],
)
endpoint.respec(skip_pg_catalog_updates=False)
endpoint.start()
# generate 20MB of data
# generate 10MB of data
endpoint.safe_psql(
"CREATE TABLE usertable AS SELECT s AS KEY, repeat('a', 1000) as VALUE from generate_series(1, 20000) s;"
"CREATE TABLE usertable AS SELECT s AS KEY, repeat('a', 1000) as VALUE from generate_series(1, 10000) s;"
)
res = endpoint.safe_psql("SELECT neon.backpressure_throttling_time() as throttling_time")[0]
assert res[0] == 0, f"throttling_time should be 0, but got {res[0]}"