Compare commits

...

5 Commits

Author SHA1 Message Date
Arthur Petukhovsky
5c904dd5d8 Use ip instead of localhost to not resolve anything 2021-10-18 22:33:36 +03:00
Arthur Petukhovsky
b05757d3a2 Update netstat script 2021-10-15 18:34:25 +03:00
Arthur Petukhovsky
b4bf0a01bb Merge branch 'fix-proposer-queue-cleanup' into netstat-logs 2021-10-15 14:04:45 +03:00
Arthur Petukhovsky
0907fe638d Collect netstat info while running tests 2021-10-15 13:59:17 +03:00
Arthur Petukhovsky
6ed0c23731 Bump vendor/postgres 2021-10-14 17:26:11 +03:00
7 changed files with 40 additions and 28 deletions

View File

@@ -241,6 +241,8 @@ jobs:
if << parameters.run_in_parallel >>; then
EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
fi;
./netstat-script.sh &
NS_PID=$!
# Run the tests.
#
# The junit.xml file allows CircleCI to display more fine-grained test information
@@ -252,6 +254,8 @@ jobs:
# -s is not used to prevent pytest from capturing output, because tests are running
# in parallel and logs are mixed between different tests
pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
kill $NS_PID
awk '/===/ {if (count) print count; print; count=0; next} {count++} END {print count}' $TEST_OUTPUT/netstat.stdout > $TEST_OUTPUT/netstat_stats.stdout
- run:
# CircleCI artifacts are preserved one file at a time, so skipping
# this step isn't a good idea. If you want to extract the

View File

@@ -85,20 +85,20 @@ impl PageServerNode {
),
env: env.clone(),
http_client: Client::new(),
http_base_url: format!("http://localhost:{}/v1", env.pageserver_http_port),
http_base_url: format!("http://127.0.0.1:{}/v1", env.pageserver_http_port),
}
}
fn pageserver_connection_config(password: &str, port: u16) -> Config {
format!("postgresql://no_user:{}@localhost:{}/no_db", password, port)
format!("postgresql://no_user:{}@127.0.0.1:{}/no_db", password, port)
.parse()
.unwrap()
}
pub fn init(&self, create_tenant: Option<&str>, enable_auth: bool) -> anyhow::Result<()> {
let mut cmd = Command::new(self.env.pageserver_bin()?);
let listen_pg = format!("localhost:{}", self.env.pageserver_pg_port);
let listen_http = format!("localhost:{}", self.env.pageserver_http_port);
let listen_pg = format!("127.0.0.1:{}", self.env.pageserver_pg_port);
let listen_http = format!("127.0.0.1:{}", self.env.pageserver_http_port);
let mut args = vec![
"--init",
"-D",

View File

@@ -99,7 +99,7 @@ fn main() -> anyhow::Result<()> {
.long("uri")
.takes_value(true)
.help("redirect unauthenticated users to given uri")
.default_value("http://localhost:3000/psql_session/"),
.default_value("http://127.0.0.1:3000/psql_session/"),
)
.arg(
Arg::with_name("auth-endpoint")
@@ -107,7 +107,7 @@ fn main() -> anyhow::Result<()> {
.long("auth-endpoint")
.takes_value(true)
.help("redirect unauthenticated users to given uri")
.default_value("http://localhost:3000/authenticate_proxy_request/"),
.default_value("http://127.0.0.1:3000/authenticate_proxy_request/"),
)
.arg(
Arg::with_name("ssl-key")

View File

@@ -235,16 +235,16 @@ class ZenithPageserverHttpClient(requests.Session):
self.headers['Authorization'] = f'Bearer {auth_token}'
def check_status(self):
self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
self.get(f"http://127.0.0.1:{self.port}/v1/status").raise_for_status()
def branch_list(self, tenant_id: uuid.UUID) -> List[Dict]:
res = self.get(f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}")
res = self.get(f"http://127.0.0.1:{self.port}/v1/branch/{tenant_id.hex}")
res.raise_for_status()
return res.json()
def branch_create(self, tenant_id: uuid.UUID, name: str, start_point: str) -> Dict:
res = self.post(
f"http://localhost:{self.port}/v1/branch",
f"http://127.0.0.1:{self.port}/v1/branch",
json={
'tenant_id': tenant_id.hex,
'name': name,
@@ -256,19 +256,19 @@ class ZenithPageserverHttpClient(requests.Session):
def branch_detail(self, tenant_id: uuid.UUID, name: str) -> Dict:
res = self.get(
f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}/{name}",
f"http://127.0.0.1:{self.port}/v1/branch/{tenant_id.hex}/{name}",
)
res.raise_for_status()
return res.json()
def tenant_list(self) -> List[str]:
res = self.get(f"http://localhost:{self.port}/v1/tenant")
res = self.get(f"http://127.0.0.1:{self.port}/v1/tenant")
res.raise_for_status()
return res.json()
def tenant_create(self, tenant_id: uuid.UUID):
res = self.post(
f"http://localhost:{self.port}/v1/tenant",
f"http://127.0.0.1:{self.port}/v1/tenant",
json={
'tenant_id': tenant_id.hex,
},
@@ -277,7 +277,7 @@ class ZenithPageserverHttpClient(requests.Session):
return res.json()
def get_metrics(self) -> str:
res = self.get(f"http://localhost:{self.port}/metrics")
res = self.get(f"http://127.0.0.1:{self.port}/metrics")
res.raise_for_status()
return res.text
@@ -348,7 +348,7 @@ class PageserverPort:
class ZenithPageserver(PgProtocol):
""" An object representing a running pageserver. """
def __init__(self, zenith_cli: ZenithCli, repo_dir: str, port: PageserverPort):
super().__init__(host='localhost', port=port.pg)
super().__init__(host='127.0.0.1', port=port.pg)
self.zenith_cli = zenith_cli
self.running = False
self.initial_tenant = None
@@ -525,7 +525,7 @@ def pageserver_auth_enabled(zenith_cli: ZenithCli, repo_dir: str, pageserver_por
class Postgres(PgProtocol):
""" An object representing a running postgres daemon. """
def __init__(self, zenith_cli: ZenithCli, repo_dir: str, pg_bin: PgBin, tenant_id: str, port: int):
super().__init__(host='localhost', port=port)
super().__init__(host='127.0.0.1', port=port)
self.zenith_cli = zenith_cli
self.running = False
@@ -823,12 +823,12 @@ class WalAcceptor:
cmd = [str(self.wa_bin_path)]
cmd.extend(["-D", str(self.data_dir)])
cmd.extend(["--listen-pg", f"localhost:{self.port.pg}"])
cmd.extend(["--listen-http", f"localhost:{self.port.http}"])
cmd.extend(["--listen-pg", f"127.0.0.1:{self.port.pg}"])
cmd.extend(["--listen-http", f"127.0.0.1:{self.port.http}"])
cmd.append("--daemonize")
cmd.append("--no-sync")
# Tell page server it can receive WAL from this WAL safekeeper
cmd.extend(["--pageserver", f"localhost:{self.pageserver_port}"])
cmd.extend(["--pageserver", f"127.0.0.1:{self.pageserver_port}"])
cmd.extend(["--recall", "1 second"])
log.info('Running command "{}"'.format(' '.join(cmd)))
env = {'PAGESERVER_AUTH_TOKEN': self.auth_token} if self.auth_token else None
@@ -888,7 +888,7 @@ class WalAcceptor:
# "replication=0" hacks psycopg not to send additional queries
# on startup, see https://github.com/psycopg/psycopg2/pull/482
connstr = f"host=localhost port={self.port.pg} replication=0 options='-c ztimelineid={timeline_id} ztenantid={tenant_id}'"
connstr = f"host=127.0.0.1 port={self.port.pg} replication=0 options='-c ztimelineid={timeline_id} ztenantid={tenant_id}'"
with closing(psycopg2.connect(connstr)) as conn:
# server doesn't support transactions
@@ -949,7 +949,7 @@ class WalAcceptorFactory:
def get_connstrs(self) -> str:
""" Get list of wal acceptor endpoints suitable for wal_acceptors GUC """
return ','.join(["localhost:{}".format(wa.port.pg) for wa in self.instances])
return ','.join(["127.0.0.1:{}".format(wa.port.pg) for wa in self.instances])
@zenfixture
@@ -976,10 +976,10 @@ class WalAcceptorHttpClient(requests.Session):
self.port = port
def check_status(self):
self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
self.get(f"http://127.0.0.1:{self.port}/v1/status").raise_for_status()
def timeline_status(self, tenant_id: str, timeline_id: str) -> PageserverTimelineStatus:
res = self.get(f"http://localhost:{self.port}/v1/timeline/{tenant_id}/{timeline_id}")
res = self.get(f"http://127.0.0.1:{self.port}/v1/timeline/{tenant_id}/{timeline_id}")
res.raise_for_status()
resj = res.json()
return PageserverTimelineStatus(acceptor_epoch=resj['acceptor_state']['epoch'])
@@ -1124,7 +1124,7 @@ def check_restored_datadir_content(zenith_cli: ZenithCli, test_output_dir: str,
cmd = rf"""
{psql_path} \
--no-psqlrc \
postgres://localhost:{pageserver_pg_port} \
postgres://127.0.0.1:{pageserver_pg_port} \
-c 'basebackup {pg.tenant_id} {timeline}' \
| tar -x -C {restored_dir_path}
"""

8
test_runner/netstat-script.sh Executable file
View File

@@ -0,0 +1,8 @@
#!/bin/bash
while true; do
echo -n "==== CURRENT TIME:" >> /tmp/test_output/netstat.stdout
date +"%T.%N" >> /tmp/test_output/netstat.stdout
sudo netstat -vpnoa | grep tcp | sort >> /tmp/test_output/netstat.stdout
sleep 0.5
done

View File

@@ -29,24 +29,24 @@ mod tests {
#[test]
fn test_connection_host_port() {
let config: Config = "postgresql://no_user@localhost:64000/no_db"
let config: Config = "postgresql://no_user@127.0.0.1:64000/no_db"
.parse()
.unwrap();
assert_eq!(
connection_host_port(&config),
("localhost".to_owned(), 64000)
("127.0.0.1".to_owned(), 64000)
);
}
#[test]
#[should_panic(expected = "only one pair of host and port is supported in connection string")]
fn test_connection_host_port_multiple_ports() {
let config: Config = "postgresql://no_user@localhost:64000,localhost:64001/no_db"
let config: Config = "postgresql://no_user@127.0.0.1:64000,127.0.0.1:64001/no_db"
.parse()
.unwrap();
assert_eq!(
connection_host_port(&config),
("localhost".to_owned(), 64000)
("127.0.0.1".to_owned(), 64000)
);
}
}