From 72afa06b7a99026c197f084a40e12ccf65ded2fe Mon Sep 17 00:00:00 2001 From: gsilvestrin Date: Mon, 17 Jul 2023 08:48:24 -0700 Subject: [PATCH] feat(node): Add Windows support (#294) --- .github/workflows/npm-publish.yml | 33 +++++++++++++++++++++++++ .github/workflows/rust.yml | 21 ++++++++++++++++ ci/build_windows_artifacts.ps1 | 41 +++++++++++++++++++++++++++++++ node/package-lock.json | 6 +++-- node/package.json | 11 ++++++--- rust/vectordb/src/database.rs | 11 ++++++--- rust/vectordb/src/table.rs | 1 + 7 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 ci/build_windows_artifacts.ps1 diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 0690c8d8..3ea41228 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -116,6 +116,39 @@ jobs: path: | node/dist/vectordb-linux*.tgz + node-windows: + runs-on: windows-2022 + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + target: [x86_64-pc-windows-msvc] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Protoc v21.12 + working-directory: C:\ + run: | + New-Item -Path 'C:\protoc' -ItemType Directory + Set-Location C:\protoc + Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip + 7z x protoc.zip + Add-Content $env:GITHUB_PATH "C:\protoc\bin" + shell: powershell + - name: Install npm dependencies + run: | + cd node + npm ci + - name: Build Windows native node modules + run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }} + - name: Upload Windows Artifacts + uses: actions/upload-artifact@v3 + with: + name: windows-native + path: | + node/dist/vectordb-win32*.tgz + release: needs: [node, node-macos, node-linux] runs-on: ubuntu-latest diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a2e1b34e..c768337c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -66,3 +66,24 @@ jobs: run: cargo build --all-features - name: Run tests run: cargo test --all-features + windows: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - name: Install Protoc v21.12 + working-directory: C:\ + run: | + New-Item -Path 'C:\protoc' -ItemType Directory + Set-Location C:\protoc + Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip + 7z x protoc.zip + Add-Content $env:GITHUB_PATH "C:\protoc\bin" + shell: powershell + - name: Run tests + run: | + $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT + cargo build + cargo test diff --git a/ci/build_windows_artifacts.ps1 b/ci/build_windows_artifacts.ps1 new file mode 100644 index 00000000..039d4b95 --- /dev/null +++ b/ci/build_windows_artifacts.ps1 @@ -0,0 +1,41 @@ +# Builds the Windows artifacts (node binaries). +# Usage: .\ci\build_windows_artifacts.ps1 [target] +# Targets supported: +# - x86_64-pc-windows-msvc +# - i686-pc-windows-msvc + +function Prebuild-Rust { + param ( + [string]$target + ) + + # Building here for the sake of easier debugging. + Push-Location -Path "rust/ffi/node" + Write-Host "Building rust library for $target" + $env:RUST_BACKTRACE=1 + cargo build --release --target $target + Pop-Location +} + +function Build-NodeBinaries { + param ( + [string]$target + ) + + Push-Location -Path "node" + Write-Host "Building node library for $target" + npm run build-release -- --target $target + npm run pack-build -- --target $target + Pop-Location +} + +$targets = $args[0] +if (-not $targets) { + $targets = "x86_64-pc-windows-msvc" +} + +Write-Host "Building artifacts for targets: $targets" +foreach ($target in $targets) { + Prebuild-Rust $target + Build-NodeBinaries $target +} diff --git a/node/package-lock.json b/node/package-lock.json index e81040d6..768084be 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -14,7 +14,8 @@ "license": "Apache-2.0", "os": [ "darwin", - "linux" + "linux", + "win32" ], "dependencies": { "@apache-arrow/ts": "^12.0.0", @@ -52,7 +53,8 @@ "vectordb-darwin-arm64": "0.1.13", "vectordb-darwin-x64": "0.1.13", "vectordb-linux-arm64-gnu": "0.1.13", - "vectordb-linux-x64-gnu": "0.1.13" + "vectordb-linux-x64-gnu": "0.1.13", + "vectordb-win32-x64-msvc": "0.1.13" } }, "node_modules/@apache-arrow/ts": { diff --git a/node/package.json b/node/package.json index f9c67be2..f2056dce 100644 --- a/node/package.json +++ b/node/package.json @@ -8,7 +8,7 @@ "tsc": "tsc -b", "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json", "build-release": "npm run build -- --release", - "test": "npm run tsc; mocha -recursive dist/test", + "test": "npm run tsc && mocha -recursive dist/test", "lint": "eslint src --ext .js,.ts", "clean": "rm -rf node_modules *.node dist/", "pack-build": "neon pack-build", @@ -60,7 +60,8 @@ }, "os": [ "darwin", - "linux" + "linux", + "win32" ], "cpu": [ "x64", @@ -71,13 +72,15 @@ "x86_64-apple-darwin": "vectordb-darwin-x64", "aarch64-apple-darwin": "vectordb-darwin-arm64", "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu", - "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu" + "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu", + "x86_64-pc-windows-msvc": "vectordb-win32-x64-msvc" } }, "optionalDependencies": { "vectordb-darwin-arm64": "0.1.13", "vectordb-darwin-x64": "0.1.13", + "vectordb-linux-arm64-gnu": "0.1.13", "vectordb-linux-x64-gnu": "0.1.13", - "vectordb-linux-arm64-gnu": "0.1.13" + "vectordb-win32-x64-msvc": "0.1.13" } } diff --git a/rust/vectordb/src/database.rs b/rust/vectordb/src/database.rs index 4b0f161c..9200071c 100644 --- a/rust/vectordb/src/database.rs +++ b/rust/vectordb/src/database.rs @@ -27,6 +27,7 @@ pub struct Database { object_store: ObjectStore, pub(crate) uri: String, + pub(crate) base_path: object_store::path::Path, } const LANCE_EXTENSION: &str = "lance"; @@ -43,12 +44,13 @@ impl Database { /// /// * A [Database] object. pub async fn connect(uri: &str) -> Result { - let (object_store, _) = ObjectStore::from_uri(uri).await?; + let (object_store, base_path) = ObjectStore::from_uri(uri).await?; if object_store.is_local() { Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?; } Ok(Database { uri: uri.to_string(), + base_path, object_store, }) } @@ -70,7 +72,7 @@ impl Database { pub async fn table_names(&self) -> Result> { let f = self .object_store - .read_dir(self.uri.as_str()) + .read_dir(self.base_path.clone()) .await? .iter() .map(|fname| Path::new(fname)) @@ -141,8 +143,9 @@ impl Database { /// # Arguments /// * `name` - The name of the table. pub async fn drop_table(&self, name: &str) -> Result<()> { - let dir_name = format!("{}/{}.{}", self.uri, name, LANCE_EXTENSION); - self.object_store.remove_dir_all(dir_name).await?; + let dir_name = format!("{}.{}", name, LANCE_EXTENSION); + let full_path = self.base_path.child(dir_name.clone()); + self.object_store.remove_dir_all(full_path).await?; Ok(()) } } diff --git a/rust/vectordb/src/table.rs b/rust/vectordb/src/table.rs index b4641826..ab6e2e08 100644 --- a/rust/vectordb/src/table.rs +++ b/rust/vectordb/src/table.rs @@ -285,6 +285,7 @@ mod tests { } #[test] + #[cfg(not(windows))] fn test_object_store_path() { use std::path::Path as StdPath; let p = StdPath::new("s3://bucket/path/to/file");