mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
fabeff822f
## Problem We want to regularly verify the performance of pgvector HNSW parallel index builds and parallel similarity search using HNSW indexes. The first release that considerably improved the index-build parallelism was pgvector 0.7.0 and we want to make sure that we do not regress by our neon compute VM settings (swap, memory over commit, pg conf etc.) ## Summary of changes Prepare a Neon project with 1 million openAI vector embeddings (vector size 1536). Run HNSW indexing operations in the regression test for the various distance metrics. Run similarity queries using pgbench with 100 concurrent clients. I have also added the relevant metrics to the grafana dashboards pgbench and olape --------- Co-authored-by: Alexander Bayandin <alexander@neon.tech>
100 lines
2.3 KiB
TOML
100 lines
2.3 KiB
TOML
[tool.poetry]
|
|
name = "neon"
|
|
version = "0.1.0"
|
|
description = ""
|
|
authors = []
|
|
|
|
[tool.poetry.dependencies]
|
|
python = "^3.9"
|
|
pytest = "^7.4.4"
|
|
psycopg2-binary = "^2.9.6"
|
|
typing-extensions = "^4.6.1"
|
|
PyJWT = {version = "^2.1.0", extras = ["crypto"]}
|
|
requests = "^2.32.0"
|
|
pytest-xdist = "^3.3.1"
|
|
asyncpg = "^0.29.0"
|
|
aiopg = "^1.4.0"
|
|
Jinja2 = "^3.1.4"
|
|
types-requests = "^2.31.0.0"
|
|
types-psycopg2 = "^2.9.21.10"
|
|
boto3 = "^1.34.11"
|
|
boto3-stubs = {extras = ["s3"], version = "^1.26.16"}
|
|
moto = {extras = ["server"], version = "^5.0.6"}
|
|
backoff = "^2.2.1"
|
|
pytest-lazy-fixture = "^0.6.3"
|
|
prometheus-client = "^0.14.1"
|
|
pytest-timeout = "^2.1.0"
|
|
Werkzeug = "^3.0.3"
|
|
pytest-order = "^1.1.0"
|
|
allure-pytest = "^2.13.2"
|
|
pytest-asyncio = "^0.21.0"
|
|
toml = "^0.10.2"
|
|
psutil = "^5.9.4"
|
|
types-psutil = "^5.9.5.12"
|
|
types-toml = "^0.10.8.6"
|
|
pytest-httpserver = "^1.0.8"
|
|
aiohttp = "3.9.4"
|
|
pytest-rerunfailures = "^13.0"
|
|
types-pytest-lazy-fixture = "^0.6.3.3"
|
|
pytest-split = "^0.8.1"
|
|
zstandard = "^0.21.0"
|
|
httpx = {extras = ["http2"], version = "^0.26.0"}
|
|
pytest-repeat = "^0.9.3"
|
|
websockets = "^12.0"
|
|
|
|
[tool.poetry.group.dev.dependencies]
|
|
mypy = "==1.3.0"
|
|
ruff = "^0.2.2"
|
|
|
|
[build-system]
|
|
requires = ["poetry-core>=1.0.0"]
|
|
build-backend = "poetry.core.masonry.api"
|
|
|
|
[tool.mypy]
|
|
exclude = [
|
|
"^vendor/",
|
|
"^target/",
|
|
"test_runner/performance/pgvector/loaddata.py",
|
|
]
|
|
check_untyped_defs = true
|
|
# Help mypy find imports when running against list of individual files.
|
|
# Without this line it would behave differently when executed on the entire project.
|
|
mypy_path = "$MYPY_CONFIG_FILE_DIR:$MYPY_CONFIG_FILE_DIR/test_runner"
|
|
|
|
disallow_incomplete_defs = false
|
|
disallow_untyped_calls = false
|
|
disallow_untyped_decorators = false
|
|
disallow_untyped_defs = false
|
|
strict = true
|
|
|
|
[[tool.mypy.overrides]]
|
|
module = [
|
|
"asyncpg.*",
|
|
"pg8000.*",
|
|
"allure.*",
|
|
"allure_commons.*",
|
|
"allure_pytest.*",
|
|
]
|
|
ignore_missing_imports = true
|
|
|
|
[tool.ruff]
|
|
target-version = "py39"
|
|
extend-exclude = [
|
|
"vendor/",
|
|
"target/",
|
|
]
|
|
line-length = 100 # this setting is rather guidance, it won't fail if it can't make the shorter
|
|
|
|
[tool.ruff.lint]
|
|
ignore = [
|
|
"E501", # Line too long, we don't want to be too strict about it
|
|
]
|
|
select = [
|
|
"E", # pycodestyle
|
|
"F", # Pyflakes
|
|
"I", # isort
|
|
"W", # pycodestyle
|
|
"B", # bugbear
|
|
"UP032", # f-string
|
|
]
|