mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-04 02:42:57 +00:00
Compare commits
1 Commits
xuanwo/cod
...
codex/upda
| Author | SHA1 | Date | |
|---|---|---|---|
|
e1d56b9c18 |
297
.github/workflows/codex-autofix-ci.yml
vendored
297
.github/workflows/codex-autofix-ci.yml
vendored
@@ -1,297 +0,0 @@
|
||||
name: Codex Autofix CI
|
||||
|
||||
on:
|
||||
check_suite:
|
||||
types:
|
||||
- completed
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
actions: read
|
||||
|
||||
concurrency:
|
||||
group: codex-autofix-${{ github.event.check_suite.head_branch }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
autofix:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Resolve PR and failing required checks
|
||||
id: ctx
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SHA: ${{ github.event.check_suite.head_sha }}
|
||||
HEAD_BRANCH: ${{ github.event.check_suite.head_branch }}
|
||||
MAX_ATTEMPTS: "3"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "Repository: $REPO"
|
||||
echo "head_branch: $HEAD_BRANCH"
|
||||
echo "head_sha: $SHA"
|
||||
|
||||
if [[ "$HEAD_BRANCH" != codex/update-lance-* ]]; then
|
||||
echo "Skip: branch '$HEAD_BRANCH' does not match codex/update-lance-*"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
prs_json="$(gh api -H "Accept: application/vnd.github+json" "repos/$REPO/commits/$SHA/pulls")"
|
||||
pr_json="$(echo "$prs_json" | jq -c '[.[] | select(.state=="open")] | .[0]')"
|
||||
if [[ -z "$pr_json" || "$pr_json" == "null" ]]; then
|
||||
echo "Skip: no open PR found for sha $SHA"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
pr_number="$(echo "$pr_json" | jq -r '.number')"
|
||||
head_ref="$(echo "$pr_json" | jq -r '.head.ref')"
|
||||
head_repo="$(echo "$pr_json" | jq -r '.head.repo.full_name')"
|
||||
pr_head_sha="$(echo "$pr_json" | jq -r '.head.sha')"
|
||||
|
||||
if [[ "$head_repo" != "$REPO" ]]; then
|
||||
echo "Skip: cross-repo PR ($head_repo != $REPO)"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$pr_head_sha" != "$SHA" ]]; then
|
||||
echo "Skip: stale check_suite event (pr head sha $pr_head_sha != event sha $SHA)"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
set +e
|
||||
checks_json="$(gh pr checks "$pr_number" --required --repo "$REPO" --json name,state,bucket,link,workflow)"
|
||||
checks_rc=$?
|
||||
set -e
|
||||
|
||||
if [[ "$checks_rc" -eq 8 ]]; then
|
||||
echo "Skip: required checks still pending"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$checks_rc" -ne 0 ]]; then
|
||||
echo "Skip: failed to query required checks (exit=$checks_rc)"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
fail_count="$(echo "$checks_json" | jq '[.[] | select(.bucket=="fail")] | length')"
|
||||
if [[ "$fail_count" -eq 0 ]]; then
|
||||
echo "Skip: no failing required checks"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
comments_json="$(gh api "repos/$REPO/issues/$pr_number/comments" --paginate)"
|
||||
stopped_count="$(echo "$comments_json" | jq '[.[].body | select(test("<!-- codex-autofix stopped -->"))] | length')"
|
||||
if [[ "$stopped_count" -gt 0 ]]; then
|
||||
echo "Skip: codex-autofix already stopped for this PR"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
prior_attempts="$(echo "$comments_json" | jq '[.[].body | select(test("<!-- codex-autofix attempt:"))] | length')"
|
||||
attempt="$((prior_attempts + 1))"
|
||||
|
||||
if [[ "$attempt" -gt "$MAX_ATTEMPTS" ]]; then
|
||||
run_url="${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}"
|
||||
comment_file="$(mktemp /tmp/codex-autofix-comment.XXXXXX.md)"
|
||||
{
|
||||
printf '%s\n' '<!-- codex-autofix stopped -->'
|
||||
printf '<!-- codex-autofix attempt: %s -->\n' "$attempt"
|
||||
printf 'Codex autofix stopped: reached max attempts (%s).\n\n' "$MAX_ATTEMPTS"
|
||||
printf -- '- Run: %s\n' "$run_url"
|
||||
printf -- '- head_sha: `%s`\n' "$SHA"
|
||||
printf -- '- head_ref: `%s`\n' "$head_ref"
|
||||
} >"$comment_file"
|
||||
gh pr comment "$pr_number" --repo "$REPO" --body-file "$comment_file"
|
||||
echo "needs_fix=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
evidence_file="$(mktemp /tmp/codex-autofix-evidence.XXXXXX.txt)"
|
||||
run_url="${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}"
|
||||
|
||||
{
|
||||
echo "PR: #$pr_number"
|
||||
echo "head_ref: $head_ref"
|
||||
echo "head_sha: $SHA"
|
||||
echo "Run: $run_url"
|
||||
echo ""
|
||||
echo "Failing required checks:"
|
||||
echo "$checks_json" | jq -r '.[] | select(.bucket=="fail") | "- \(.name) (\(.workflow // "unknown")): \(.link // "n/a")"'
|
||||
echo ""
|
||||
} > "$evidence_file"
|
||||
|
||||
while IFS= read -r row; do
|
||||
name="$(echo "$row" | jq -r '.name')"
|
||||
link="$(echo "$row" | jq -r '.link // empty')"
|
||||
workflow="$(echo "$row" | jq -r '.workflow // "unknown"')"
|
||||
|
||||
{
|
||||
echo "================================================================================"
|
||||
echo "CHECK: $name"
|
||||
echo "WORKFLOW: $workflow"
|
||||
echo "LINK: ${link:-n/a}"
|
||||
} >> "$evidence_file"
|
||||
|
||||
run_id=""
|
||||
if [[ -n "$link" ]]; then
|
||||
run_id="$(echo "$link" | sed -n 's#.*actions/runs/\\([0-9][0-9]*\\).*#\\1#p' | head -n 1 || true)"
|
||||
fi
|
||||
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "LOGS: unavailable (no run id found in link)" >> "$evidence_file"
|
||||
echo "" >> "$evidence_file"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "LOGS: gh run view $run_id --log-failed (tail -c 20000)" >> "$evidence_file"
|
||||
set +e
|
||||
gh run view "$run_id" --repo "$REPO" --log-failed 2>/dev/null | tail -c 20000 >> "$evidence_file"
|
||||
echo "" >> "$evidence_file"
|
||||
set -e
|
||||
done < <(echo "$checks_json" | jq -c '.[] | select(.bucket=="fail")')
|
||||
|
||||
comment_file="$(mktemp /tmp/codex-autofix-comment.XXXXXX.md)"
|
||||
{
|
||||
printf '<!-- codex-autofix attempt: %s -->\n' "$attempt"
|
||||
printf 'Starting Codex autofix attempt %s.\n\n' "$attempt"
|
||||
printf -- '- Run: %s\n' "$run_url"
|
||||
printf -- '- head_sha: `%s`\n' "$SHA"
|
||||
printf -- '- head_ref: `%s`\n' "$head_ref"
|
||||
printf -- '- Failing required checks: %s\n' "$fail_count"
|
||||
} >"$comment_file"
|
||||
gh pr comment "$pr_number" --repo "$REPO" --body-file "$comment_file"
|
||||
|
||||
{
|
||||
echo "needs_fix=true"
|
||||
echo "pr_number=$pr_number"
|
||||
echo "head_ref=$head_ref"
|
||||
echo "attempt=$attempt"
|
||||
echo "evidence_file=$evidence_file"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout PR branch
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ steps.ctx.outputs.head_ref }}
|
||||
token: ${{ secrets.ROBOT_TOKEN }}
|
||||
persist-credentials: true
|
||||
|
||||
- name: Configure git
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
run: |
|
||||
git config user.name "lancedb automation"
|
||||
git config user.email "robot@lancedb.com"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
- name: Install Codex CLI
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
run: npm install -g @openai/codex
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: stable
|
||||
components: clippy, rustfmt
|
||||
|
||||
- name: Install system dependencies
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y protobuf-compiler libssl-dev
|
||||
|
||||
- name: Run Codex to fix failing CI
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
env:
|
||||
PR_NUMBER: ${{ steps.ctx.outputs.pr_number }}
|
||||
HEAD_REF: ${{ steps.ctx.outputs.head_ref }}
|
||||
ATTEMPT: ${{ steps.ctx.outputs.attempt }}
|
||||
EVIDENCE_FILE: ${{ steps.ctx.outputs.evidence_file }}
|
||||
GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
|
||||
GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
|
||||
OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
prompt_file="/tmp/codex-prompt.txt"
|
||||
{
|
||||
printf 'You are running inside the lancedb repository on a GitHub Actions runner.\n'
|
||||
printf 'Your task is to fix failing required CI checks for pull request #%s on branch %s.\n\n' "$PR_NUMBER" "$HEAD_REF"
|
||||
printf 'Goal:\n'
|
||||
printf -- '- Make the smallest change necessary so that all required checks pass.\n\n'
|
||||
printf 'Evidence (generated from GitHub checks and logs):\n'
|
||||
printf '---\n'
|
||||
cat "${EVIDENCE_FILE}"
|
||||
printf '\n---\n\n'
|
||||
printf 'Follow these steps exactly:\n'
|
||||
printf '1. Identify the root cause from the evidence and repository state.\n'
|
||||
printf '2. Make changes to fix the failures.\n'
|
||||
printf '3. Run the relevant local commands that correspond to the failing checks until they succeed.\n'
|
||||
printf ' If unsure, start with:\n'
|
||||
printf ' - cargo fmt --all -- --check\n'
|
||||
printf ' - cargo clippy --profile ci --workspace --tests --all-features -- -D warnings\n'
|
||||
printf '4. Ensure the repository is clean except for intentional changes (git status --short, git diff).\n'
|
||||
printf '5. Create a commit with message "fix: codex autofix (attempt %s)".\n' "$ATTEMPT"
|
||||
printf '6. Push to origin branch "%s" (use --force-with-lease only if required).\n' "$HEAD_REF"
|
||||
printf '7. Print the commands you ran and their results, plus git status --short and git log -1 --oneline.\n\n'
|
||||
printf 'Constraints:\n'
|
||||
printf -- '- Do not create a new pull request.\n'
|
||||
printf -- '- Do not merge.\n'
|
||||
printf -- '- Avoid modifying GitHub workflow files unless strictly required to fix CI for this PR.\n'
|
||||
} >"$prompt_file"
|
||||
|
||||
printenv OPENAI_API_KEY | codex login --with-api-key
|
||||
codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat "$prompt_file")"
|
||||
|
||||
- name: Ensure branch is pushed
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
env:
|
||||
HEAD_REF: ${{ steps.ctx.outputs.head_ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet && git diff --cached --quiet; then
|
||||
echo "Working tree clean."
|
||||
else
|
||||
git add -A
|
||||
git commit -m "fix: codex autofix (post-run)" || true
|
||||
fi
|
||||
git push origin "HEAD:${HEAD_REF}" --force-with-lease
|
||||
|
||||
- name: Comment result
|
||||
if: steps.ctx.outputs.needs_fix == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ steps.ctx.outputs.pr_number }}
|
||||
ATTEMPT: ${{ steps.ctx.outputs.attempt }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
run_url="${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}"
|
||||
sha="$(git rev-parse HEAD)"
|
||||
summary="$(git log -1 --oneline || true)"
|
||||
status="$(git status --short || true)"
|
||||
comment_file="$(mktemp /tmp/codex-autofix-comment.XXXXXX.md)"
|
||||
{
|
||||
printf 'Codex autofix attempt %s finished.\n\n' "$ATTEMPT"
|
||||
printf -- '- Run: %s\n' "$run_url"
|
||||
printf -- '- head_sha: `%s`\n' "$sha"
|
||||
printf -- '- Last commit: %s\n\n' "$summary"
|
||||
printf '```\n%s\n```\n' "$status"
|
||||
} >"$comment_file"
|
||||
gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "$comment_file"
|
||||
76
Cargo.lock
generated
76
Cargo.lock
generated
@@ -3141,8 +3141,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.2",
|
||||
@@ -4478,8 +4478,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4544,13 +4544,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-cast",
|
||||
"arrow-data",
|
||||
"arrow-ord",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"bytes",
|
||||
@@ -4563,8 +4564,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"paste",
|
||||
@@ -4573,8 +4574,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4587,6 +4588,7 @@ dependencies = [
|
||||
"datafusion-sql",
|
||||
"deepsize",
|
||||
"futures",
|
||||
"itertools 0.13.0",
|
||||
"lance-arrow",
|
||||
"libc",
|
||||
"log",
|
||||
@@ -4610,8 +4612,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4641,8 +4643,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4653,14 +4655,15 @@ dependencies = [
|
||||
"half",
|
||||
"hex",
|
||||
"rand 0.9.2",
|
||||
"rand_distr 0.5.1",
|
||||
"rand_xoshiro",
|
||||
"random_word 0.5.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4697,8 +4700,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4730,8 +4733,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-types",
|
||||
@@ -4742,8 +4745,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4789,6 +4792,7 @@ dependencies = [
|
||||
"prost-types",
|
||||
"rand 0.9.2",
|
||||
"rand_distr 0.5.1",
|
||||
"rangemap",
|
||||
"rayon",
|
||||
"roaring",
|
||||
"serde",
|
||||
@@ -4804,8 +4808,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4845,8 +4849,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4862,8 +4866,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -4875,8 +4879,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -4905,9 +4909,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-reqwest-client"
|
||||
version = "0.0.18"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19"
|
||||
checksum = "00a21b43fe2a373896727b97927adedd2683d2907683f294f62cf8815fbf6a01"
|
||||
dependencies = [
|
||||
"reqwest",
|
||||
"serde",
|
||||
@@ -4918,8 +4922,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4958,8 +4962,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
version = "2.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.4#4c71be26ad92c6f0beea27022e1a489c6fc1f8bc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
|
||||
28
Cargo.toml
28
Cargo.toml
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.78.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=2.0.0-beta.4", default-features = false, "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=2.0.0-beta.4", default-features = false, "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=2.0.0-beta.4", default-features = false, "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=2.0.0-beta.4", "tag" = "v2.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "56.2", optional = false }
|
||||
|
||||
@@ -325,12 +325,12 @@ impl Connection {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let py = self_.py();
|
||||
future_into_py(py, async move {
|
||||
use lance_namespace::models::{create_namespace_request, CreateNamespaceRequest};
|
||||
let mode_enum = mode.and_then(|m| match m.to_lowercase().as_str() {
|
||||
"create" => Some(create_namespace_request::Mode::Create),
|
||||
"exist_ok" => Some(create_namespace_request::Mode::ExistOk),
|
||||
"overwrite" => Some(create_namespace_request::Mode::Overwrite),
|
||||
_ => None,
|
||||
use lance_namespace::models::CreateNamespaceRequest;
|
||||
let mode_enum = mode.map(|m| match m.to_lowercase().as_str() {
|
||||
"create" => "Create".to_string(),
|
||||
"exist_ok" => "ExistOk".to_string(),
|
||||
"overwrite" => "Overwrite".to_string(),
|
||||
_ => m,
|
||||
});
|
||||
let request = CreateNamespaceRequest {
|
||||
id: if namespace.is_empty() {
|
||||
@@ -360,16 +360,16 @@ impl Connection {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let py = self_.py();
|
||||
future_into_py(py, async move {
|
||||
use lance_namespace::models::{drop_namespace_request, DropNamespaceRequest};
|
||||
let mode_enum = mode.and_then(|m| match m.to_uppercase().as_str() {
|
||||
"SKIP" => Some(drop_namespace_request::Mode::Skip),
|
||||
"FAIL" => Some(drop_namespace_request::Mode::Fail),
|
||||
_ => None,
|
||||
use lance_namespace::models::DropNamespaceRequest;
|
||||
let mode_enum = mode.map(|m| match m.to_uppercase().as_str() {
|
||||
"SKIP" => "Skip".to_string(),
|
||||
"FAIL" => "Fail".to_string(),
|
||||
_ => m,
|
||||
});
|
||||
let behavior_enum = behavior.and_then(|b| match b.to_uppercase().as_str() {
|
||||
"RESTRICT" => Some(drop_namespace_request::Behavior::Restrict),
|
||||
"CASCADE" => Some(drop_namespace_request::Behavior::Cascade),
|
||||
_ => None,
|
||||
let behavior_enum = behavior.map(|b| match b.to_uppercase().as_str() {
|
||||
"RESTRICT" => "Restrict".to_string(),
|
||||
"CASCADE" => "Cascade".to_string(),
|
||||
_ => b,
|
||||
});
|
||||
let request = DropNamespaceRequest {
|
||||
id: if namespace.is_empty() {
|
||||
|
||||
@@ -154,6 +154,7 @@ impl Database for LanceNamespaceDatabase {
|
||||
let describe_request = DescribeTableRequest {
|
||||
id: Some(table_id.clone()),
|
||||
version: None,
|
||||
with_table_uri: None,
|
||||
};
|
||||
|
||||
let describe_result = self.namespace.describe_table(describe_request).await;
|
||||
|
||||
@@ -42,8 +42,8 @@ use lance_index::DatasetIndexExt;
|
||||
use lance_index::IndexType;
|
||||
use lance_io::object_store::LanceNamespaceStorageOptionsProvider;
|
||||
use lance_namespace::models::{
|
||||
QueryTableRequest as NsQueryTableRequest, QueryTableRequestFullTextQuery,
|
||||
QueryTableRequestVector, StringFtsQuery,
|
||||
QueryTableRequest as NsQueryTableRequest, QueryTableRequestColumns,
|
||||
QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery,
|
||||
};
|
||||
use lance_namespace::LanceNamespace;
|
||||
use lance_table::format::Manifest;
|
||||
@@ -2348,7 +2348,10 @@ impl NativeTable {
|
||||
// Convert select to columns list
|
||||
let columns = match &vq.base.select {
|
||||
Select::All => None,
|
||||
Select::Columns(cols) => Some(cols.clone()),
|
||||
Select::Columns(cols) => Some(Box::new(QueryTableRequestColumns {
|
||||
column_names: Some(cols.clone()),
|
||||
column_aliases: None,
|
||||
})),
|
||||
Select::Dynamic(_) => {
|
||||
return Err(Error::NotSupported {
|
||||
message:
|
||||
@@ -2421,7 +2424,10 @@ impl NativeTable {
|
||||
|
||||
let columns = match &q.select {
|
||||
Select::All => None,
|
||||
Select::Columns(cols) => Some(cols.clone()),
|
||||
Select::Columns(cols) => Some(Box::new(QueryTableRequestColumns {
|
||||
column_names: Some(cols.clone()),
|
||||
column_aliases: None,
|
||||
})),
|
||||
Select::Dynamic(_) => {
|
||||
return Err(Error::NotSupported {
|
||||
message: "Dynamic columns are not supported for server-side query"
|
||||
@@ -5146,7 +5152,13 @@ mod tests {
|
||||
assert_eq!(ns_request.k, 10);
|
||||
assert_eq!(ns_request.offset, Some(5));
|
||||
assert_eq!(ns_request.filter, Some("id > 0".to_string()));
|
||||
assert_eq!(ns_request.columns, Some(vec!["id".to_string()]));
|
||||
assert_eq!(
|
||||
ns_request
|
||||
.columns
|
||||
.as_ref()
|
||||
.and_then(|c| c.column_names.clone()),
|
||||
Some(vec!["id".to_string()])
|
||||
);
|
||||
assert_eq!(ns_request.vector_column, Some("vector".to_string()));
|
||||
assert_eq!(ns_request.distance_type, Some("l2".to_string()));
|
||||
assert!(ns_request.vector.single_vector.is_some());
|
||||
@@ -5187,7 +5199,13 @@ mod tests {
|
||||
assert_eq!(ns_request.k, 20);
|
||||
assert_eq!(ns_request.offset, Some(5));
|
||||
assert_eq!(ns_request.filter, Some("id > 5".to_string()));
|
||||
assert_eq!(ns_request.columns, Some(vec!["id".to_string()]));
|
||||
assert_eq!(
|
||||
ns_request
|
||||
.columns
|
||||
.as_ref()
|
||||
.and_then(|c| c.column_names.clone()),
|
||||
Some(vec!["id".to_string()])
|
||||
);
|
||||
assert_eq!(ns_request.with_row_id, Some(true));
|
||||
assert_eq!(ns_request.bypass_vector_index, Some(true));
|
||||
assert!(ns_request.vector_column.is_none()); // No vector column for plain queries
|
||||
|
||||
@@ -100,7 +100,8 @@ impl DatasetRef {
|
||||
let should_checkout = match &target_ref {
|
||||
refs::Ref::Version(_, Some(target_ver)) => version != target_ver,
|
||||
refs::Ref::Version(_, None) => true, // No specific version, always checkout
|
||||
refs::Ref::Tag(_) => true, // Always checkout for tags
|
||||
refs::Ref::VersionNumber(target_ver) => version != target_ver,
|
||||
refs::Ref::Tag(_) => true, // Always checkout for tags
|
||||
};
|
||||
|
||||
if should_checkout {
|
||||
|
||||
Reference in New Issue
Block a user