From 76a4654424fa3239f0970cdf0914f9b4dfff6a1c Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Fri, 24 Jan 2025 11:53:58 +0100 Subject: [PATCH] need another patch foro pg_duckdb which has its own cached copy of httpfs_filesystem --- compute/compute-node.Dockerfile | 2 + compute/patches/pg_duckdb-0-2-0.patch | 61 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 compute/patches/pg_duckdb-0-2-0.patch diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 86fc77b37c..7cd165c92a 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1118,6 +1118,7 @@ FROM build-deps AS pg-duckdb-pg-build ARG PG_VERSION COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY compute/patches/duckdb-v1-1-3.patch /duckdb-v1-1-3.patch +COPY compute/patches/pg_duckdb-0-2-0.patch /pg_duckdb-0-2-0.patch ENV PATH="/usr/local/pgsql/bin/:$PATH" @@ -1133,6 +1134,7 @@ RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git cd third_party/duckdb && \ patch -p1 < /duckdb-v1-1-3.patch && \ cd ../.. && \ + patch -p1 < /pg_duckdb-0-2-0.patch && \ make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ file=/usr/local/pgsql/share/extension/pg_duckdb--0.1.0--0.2.0.sql && \ diff --git a/compute/patches/pg_duckdb-0-2-0.patch b/compute/patches/pg_duckdb-0-2-0.patch new file mode 100644 index 0000000000..ab7e6ef8d3 --- /dev/null +++ b/compute/patches/pg_duckdb-0-2-0.patch @@ -0,0 +1,61 @@ +diff --git a/third_party/cached_httpfs/http_file_cache.cpp b/third_party/cached_httpfs/http_file_cache.cpp +index 813db57..c6e4c3a 100644 +--- a/third_party/cached_httpfs/http_file_cache.cpp ++++ b/third_party/cached_httpfs/http_file_cache.cpp +@@ -4,20 +4,22 @@ namespace duckdb { + + CachedFile::CachedFile(const string &cache_dir, FileSystem &fs, const string &key, bool cache_file) : cache_directory(cache_dir), fs(fs) { + file_name = cache_dir + "/" + key; +- ++ FileSystem::TrustedContext trusted1(fs); + GetDirectoryCacheLock(cache_dir); + + FileOpenFlags flags = + FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS | FileLockType::READ_LOCK; ++ FileSystem::TrustedContext trusted2(fs); + handle = fs.OpenFile(file_name, flags); + if (handle) { + initialized = true; + size = handle->GetFileSize(); + } else if (cache_file) { + flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE | FileLockType::WRITE_LOCK; ++ FileSystem::TrustedContext trusted3(fs); + handle = fs.OpenFile(file_name, flags); + } +- ++ FileSystem::TrustedContext trusted4(fs); + ReleaseDirectoryCacheLock(); + } + +@@ -30,6 +32,7 @@ void CachedFile::GetDirectoryCacheLock(const string &cache_dir) { + FileOpenFlags flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE | + FileFlags::FILE_FLAGS_EXCLUSIVE_CREATE | FileFlags::FILE_FLAGS_NULL_IF_EXISTS | + FileLockType::WRITE_LOCK; ++ FileSystem::TrustedContext trusted(fs); + directory_lock_handle = fs.OpenFile(lock_file, flags); + if (directory_lock_handle == nullptr) { + flags = FileFlags::FILE_FLAGS_WRITE | FileLockType::WRITE_LOCK; +@@ -52,6 +55,7 @@ void CachedFileHandle::WriteMetadata(const string &cache_key, const string &remo + D_ASSERT(!file->initialized); + string metadata_file_name = file->cache_directory + "/" + cache_key + ".meta"; + FileOpenFlags flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE | FileLockType::WRITE_LOCK; ++ FileSystem::TrustedContext trusted(file->fs); + auto handle = file->fs.OpenFile(metadata_file_name, flags); + auto cached_file_timestamp = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + string metadata_info = cache_key + "," + remote_path + "," + std::to_string(total_size) + "," + std::to_string(cached_file_timestamp); +@@ -66,6 +70,7 @@ void CachedFileHandle::SetInitialized(idx_t total_size) { + file->initialized = true; + file->handle->Close(); + FileOpenFlags flags = FileFlags::FILE_FLAGS_READ | FileLockType::READ_LOCK; ++ FileSystem::TrustedContext trusted(file->fs); + file->handle = file->fs.OpenFile(file->file_name, flags); + } + +@@ -93,6 +98,7 @@ void CachedFileHandle::Read(void *buffer, idx_t length, idx_t offset) { + //! Get cache entry, create if not exists only if caching is enabled + shared_ptr HTTPFileCache::GetCachedFile(const string &cache_dir, const string &key, bool cache_file) { + lock_guard lock(cached_files_mutex); ++ FileSystem::TrustedContext trusted(db->GetFileSystem()); + auto it = cached_files.find(key); + if (it != cached_files.end()) { + return it->second;