chore: update lance dependency to v6.0.0-beta.7 (#3334)

## Summary
- Update Lance Rust dependencies to `6.0.0-beta.7` using
`ci/set_lance_version.py`.
- Update Java `lance-core.version` to `6.0.0-beta.7`.
- Align Arrow/DataFusion/PyO3 dependency versions and apply required
compatibility fixes for the Lance upgrade.

Triggering tag:
[v6.0.0-beta.7](https://github.com/lance-format/lance/releases/tag/v6.0.0-beta.7)

## Verification
- `cargo clippy --workspace --tests --all-features -- -D warnings`
- `cargo fmt --all`

python/src/permutation.rs

@@ -25,12 +25,12 @@ use pyo3_async_runtimes::tokio::future_into_py;
 
 fn table_from_py<'a>(table: Bound<'a, PyAny>) -> PyResult<Bound<'a, Table>> {
     if table.hasattr("_inner")? {
-        Ok(table.getattr("_inner")?.downcast_into::<Table>()?)
+        Ok(table.getattr("_inner")?.cast_into::<Table>()?)
     } else if table.hasattr("_table")? {
         Ok(table
             .getattr("_table")?
             .getattr("_inner")?
-            .downcast_into::<Table>()?)
+            .cast_into::<Table>()?)
     } else {
         Err(PyRuntimeError::new_err(
             "Provided table does not appear to be a Table or RemoteTable instance",
@@ -90,9 +90,9 @@ impl PyAsyncPermutationBuilder {
             database
                 .getattr("_conn")?
                 .getattr("_inner")?
-                .downcast_into::<Connection>()?
+                .cast_into::<Connection>()?
         } else {
-            database.getattr("_inner")?.downcast_into::<Connection>()?
+            database.getattr("_inner")?.cast_into::<Connection>()?
         };
         let database = conn.borrow().database()?;
         slf.modify(|builder| builder.persist(database, table_name))
@@ -243,7 +243,7 @@ impl PyPermutationReader {
         let Some(selection) = selection else {
             return Ok(Select::All);
         };
-        let selection = selection.downcast_into::<PyDict>()?;
+        let selection = selection.cast_into::<PyDict>()?;
         let selection = selection
             .iter()
             .map(|(key, value)| {