feat(remote): announce opaque page tokens when listing tables

`/v1/namespace/{ns}/table/list` also serves `table_names`, whose
`page_token` is a table name to resume after, so a server cannot tell
which contract a listing walk is in from an absent token. Send a
`!cursor/` marker on the first page of `list_tables` to say this walk
round-trips opaque tokens, which lets the server page the object store
directly instead of enumerating names.

A server that predates the marker reads it as a name; `!` sorts below any
practical table name, so the listing still starts at the beginning.

rust/lancedb/src/remote/db.rs

@@ -683,9 +683,17 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
         if let Some(limit) = request.limit {
             req = req.query(&[("limit", limit)]);
         }
-        if let Some(ref page_token) = request.page_token {
-            req = req.query(&[("page_token", page_token)]);
-        }
+        // `/v1/namespace/{ns}/table/list` also serves `table_names`, whose `page_token` is a table
+        // name to resume after. Sending this marker on the first page tells the server this walk
+        // round-trips opaque tokens, so it can page the object store directly instead. A server
+        // that predates the marker reads it as a name and, since `!` sorts below any practical
+        // table name, still starts at the beginning of the listing.
+        const OPAQUE_PAGE_TOKEN_MARKER: &str = "!cursor/";
+        let page_token = request
+            .page_token
+            .as_deref()
+            .unwrap_or(OPAQUE_PAGE_TOKEN_MARKER);
+        req = req.query(&[("page_token", page_token)]);
 
         let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
         let rsp = self.client.check_response(&request_id, rsp).await?;
@@ -1666,6 +1674,40 @@ mod tests {
         assert_eq!(names, vec!["table1", "table2"]);
     }
 
+    #[tokio::test]
+    async fn test_list_tables_announces_opaque_page_tokens() {
+        let conn = Connection::new_with_handler(|request| {
+            assert_eq!(request.url().query(), Some("page_token=%21cursor%2F"));
+
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"tables": ["table1"], "page_token": "!cursor/table1"}"#)
+                .unwrap()
+        });
+        let page = conn.list_tables().execute().await.unwrap();
+        assert_eq!(page.tables, vec!["table1"]);
+        assert_eq!(page.page_token.as_deref(), Some("!cursor/table1"));
+    }
+
+    #[tokio::test]
+    async fn test_list_tables_resumes_with_the_server_token() {
+        let conn = Connection::new_with_handler(|request| {
+            assert_eq!(request.url().query(), Some("page_token=%21cursor%2Ftable1"));
+
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"tables": ["table2"]}"#)
+                .unwrap()
+        });
+        let page = conn
+            .list_tables()
+            .page_token("!cursor/table1")
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(page.tables, vec!["table2"]);
+    }
+
     #[tokio::test]
     async fn test_table_names_with_nested_namespace() {
         // When namespace is vec!["ns1", "ns2"], should use /v1/namespace/ns1$ns2/table/list