feat: create_index returns a Job handle (#3742)

IndexBuilder::execute now returns a Job with wait and cancel methods.
Local tables build the index synchronously and return an already-done
job. Remote tables read the job id the server returns from create_index
and track it through the /v1/jobs API: wait polls describe until the job
reaches a terminal state and cancel posts a cancellation. Servers that
return no job id yield a done job, so behavior against older servers is
unchanged. The job id is not exposed on the handle.

The Python and TypeScript bindings keep their current signatures and
discard the handle; exposing Job there is left to follow-ups.

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

python/python/tests/test_index.py

@@ -84,6 +84,15 @@ async def binary_table(db_async):
     )
 
 
+@pytest.mark.asyncio
+async def test_create_index_async_returns_done_job(some_table: AsyncTable):
+    job = await some_table.create_index_async("id", config=BTree())
+    assert job.id is None
+    await job.wait()
+    assert len(await some_table.list_indices()) == 1
+    await job.cancel()
+
+
 @pytest.mark.asyncio
 async def test_create_scalar_index(some_table: AsyncTable):
     # Can create

python/python/tests/test_remote_db.py

@@ -812,6 +812,121 @@ def test_table_create_indices():
         table.drop_index("custom_fts_idx")
 
 
+def test_remote_create_index_async_returns_job():
+    from lancedb.index import BTree
+
+    describe_calls = []
+
+    def handler(request):
+        content_len = int(request.headers.get("Content-Length", 0))
+        body = request.rfile.read(content_len) if content_len > 0 else b""
+        if request.path == "/v1/table/test/create_index/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b'{"job_id": "job-1"}')
+        elif request.path == "/v1/jobs/describe":
+            assert json.loads(body)["job_id"] == "job-1"
+            describe_calls.append(1)
+            state = "IN_PROGRESS" if len(describe_calls) == 1 else "DONE"
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(dict(job_id="job-1", job_state=state)).encode()
+            )
+        elif request.path == "/v1/jobs/cancel":
+            assert json.loads(body)["job_id"] == "job-1"
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    dict(
+                        version=1,
+                        schema=dict(
+                            fields=[
+                                dict(name="id", type={"type": "int64"}, nullable=False),
+                            ]
+                        ),
+                    )
+                ).encode()
+            )
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}])
+        job = table.create_index_async("id", config=BTree())
+        assert job.id == "job-1"
+        job.wait(timeout=timedelta(seconds=30))
+        assert len(describe_calls) == 2
+        job.cancel()
+
+
+def test_remote_job_wait_raises_on_failure():
+    from lancedb.exceptions import JobFailedError
+    from lancedb.index import BTree
+
+    def handler(request):
+        content_len = int(request.headers.get("Content-Length", 0))
+        body = request.rfile.read(content_len) if content_len > 0 else b""
+        if request.path == "/v1/table/test/create_index/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b'{"job_id": "job-2"}')
+        elif request.path == "/v1/jobs/describe":
+            assert json.loads(body)["job_id"] == "job-2"
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(dict(job_id="job-2", job_state="FAILED")).encode()
+            )
+        elif request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    dict(
+                        version=1,
+                        schema=dict(
+                            fields=[
+                                dict(name="id", type={"type": "int64"}, nullable=False),
+                            ]
+                        ),
+                    )
+                ).encode()
+            )
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}])
+        job = table.create_index_async("id", config=BTree())
+        with pytest.raises(JobFailedError, match="job-2"):
+            job.wait()
+
+
 def test_remote_create_index_new_api():
     received_requests = []
 

python/python/tests/test_table.py

@@ -1402,6 +1402,15 @@ async def test_async_open_table_with_branch_version(tmp_path):
     assert await pinned.count_rows() == 4  # writable again
 
 
+def test_create_index_async_returns_done_job(mem_db: DBConnection):
+    table = mem_db.create_table("job_test", [{"id": i} for i in range(10)])
+    job = table.create_index_async("id", config=BTree())
+    assert job.id is None
+    job.wait()
+    assert len(table.list_indices()) == 1
+    job.cancel()
+
+
 @patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_method(mock_create_index, mem_db: DBConnection):
     table = mem_db.create_table(