rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-01-03 18:32:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: include manifest files in mirrow store (#589)

Browse Source
This commit is contained in:
Rob Meng
2023-10-21 12:21:41 -04:00
committed by GitHub
parent 14e8e48de2
commit ce19fedb08
2 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
node/src/integration_test/test.ts
View File

@@ -65,8 +65,8 @@ describe('LanceDB Mirrored Store Integration test', function () {
const mirroredPath = path.join(dir, `${tableName}.lance`)
fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
if (err != null) throw err
// there should be two dirs
assert.equal(files.length, 2)
// there should be three dirs
assert.equal(files.length, 3)
assert.isTrue(files[0].isDirectory())
assert.isTrue(files[1].isDirectory())
@@ -76,6 +76,12 @@ describe('LanceDB Mirrored Store Integration test', function () {
assert.isTrue(files[0].name.endsWith('.txn'))
})
fs.readdir(path.join(mirroredPath, '_versions'), { withFileTypes: true }, (err, files) => {
if (err != null) throw err
assert.equal(files.length, 1)
assert.isTrue(files[0].name.endsWith('.manifest'))
})
fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
if (err != null) throw err
assert.equal(files.length, 1)
@@ -88,8 +94,8 @@ describe('LanceDB Mirrored Store Integration test', function () {
fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
if (err != null) throw err
// there should be two dirs
assert.equal(files.length, 3)
// there should be four dirs
assert.equal(files.length, 4)
assert.isTrue(files[0].isDirectory())
assert.isTrue(files[1].isDirectory())
assert.isTrue(files[2].isDirectory())
@@ -128,12 +134,13 @@ describe('LanceDB Mirrored Store Integration test', function () {
fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
if (err != null) throw err
// there should be two dirs
assert.equal(files.length, 4)
// there should be five dirs
assert.equal(files.length, 5)
assert.isTrue(files[0].isDirectory())
assert.isTrue(files[1].isDirectory())
assert.isTrue(files[2].isDirectory())
assert.isTrue(files[3].isDirectory())
assert.isTrue(files[4].isDirectory())
// Three TXs now
fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {

13
rust/vectordb/src/io/object_store.rs
View File

@@ -57,7 +57,7 @@ trait PrimaryOnly {
impl PrimaryOnly for Path {
fn primary_only(&self) -> bool {
self.to_string().contains("manifest")
self.filename().unwrap_or("") == "_latest.manifest"
}
}
@@ -118,8 +118,10 @@ impl ObjectStore for MirroringObjectStore {
self.primary.head(location).await
}
// garbage collection on secondary will happen async from other means
async fn delete(&self, location: &Path) -> Result<()> {
if !location.primary_only() {
self.secondary.delete(location).await?;
}
self.primary.delete(location).await
}
@@ -132,7 +134,7 @@ impl ObjectStore for MirroringObjectStore {
}
async fn copy(&self, from: &Path, to: &Path) -> Result<()> {
if from.primary_only() {
if to.primary_only() {
self.primary.copy(from, to).await
} else {
self.secondary.copy(from, to).await?;
@@ -142,6 +144,9 @@ impl ObjectStore for MirroringObjectStore {
}
async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> {
if !to.primary_only() {
self.secondary.copy(from, to).await?;
}
self.primary.copy_if_not_exists(from, to).await
}
}
@@ -379,7 +384,7 @@ mod test {
let primary_f = primary_elem.unwrap().unwrap();
// hit manifest, skip, _versions contains all the manifest and should not exist on secondary
let primary_raw_path = primary_f.file_name().to_str().unwrap();
if primary_raw_path.contains("manifest") || primary_raw_path.contains("_versions") {
if primary_raw_path.contains("_latest.manifest") {
primary_elem = primary_iter.next();
continue;
}