From b32c6a8798e357b9d766409459b78d617c78f45c Mon Sep 17 00:00:00 2001
From: Vlad Lazar <vlad@neon.tech>
Date: Tue, 19 Nov 2024 19:40:40 +0100
Subject: [PATCH] wal_decoder: generate protobuf types

---
 Cargo.lock                                   |  3 ++
 libs/wal_decoder/Cargo.toml                  |  5 +++
 libs/wal_decoder/build.rs                    | 11 +++++
 libs/wal_decoder/proto/interpreted_wal.proto | 43 ++++++++++++++++++++
 libs/wal_decoder/src/models.rs               | 11 +++++
 5 files changed, 73 insertions(+)
 create mode 100644 libs/wal_decoder/build.rs
 create mode 100644 libs/wal_decoder/proto/interpreted_wal.proto

diff --git a/Cargo.lock b/Cargo.lock
index 56986cbb5e..d828761331 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7123,8 +7123,11 @@ dependencies = [
  "bytes",
  "pageserver_api",
  "postgres_ffi",
+ "prost",
  "serde",
  "thiserror",
+ "tonic",
+ "tonic-build",
  "tracing",
  "utils",
  "workspace_hack",
diff --git a/libs/wal_decoder/Cargo.toml b/libs/wal_decoder/Cargo.toml
index 02b9f72ed5..96e1295434 100644
--- a/libs/wal_decoder/Cargo.toml
+++ b/libs/wal_decoder/Cargo.toml
@@ -11,9 +11,14 @@ testing = ["pageserver_api/testing"]
 anyhow.workspace = true
 bytes.workspace = true
 pageserver_api.workspace = true
+prost.workspace = true
 postgres_ffi.workspace = true
 serde.workspace = true
 thiserror.workspace = true
+tonic.workspace = true
 tracing.workspace = true
 utils.workspace = true
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
+
+[build-dependencies]
+tonic-build.workspace = true
diff --git a/libs/wal_decoder/build.rs b/libs/wal_decoder/build.rs
new file mode 100644
index 0000000000..d5b7ad02ad
--- /dev/null
+++ b/libs/wal_decoder/build.rs
@@ -0,0 +1,11 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Generate rust code from .proto protobuf.
+    //
+    // Note: we previously tried to use deterministic location at proto/ for
+    // easy location, but apparently interference with cachepot sometimes fails
+    // the build then. Anyway, per cargo docs build script shouldn't output to
+    // anywhere but $OUT_DIR.
+    tonic_build::compile_protos("proto/interpreted_wal.proto")
+        .unwrap_or_else(|e| panic!("failed to compile protos {:?}", e));
+    Ok(())
+}
diff --git a/libs/wal_decoder/proto/interpreted_wal.proto b/libs/wal_decoder/proto/interpreted_wal.proto
new file mode 100644
index 0000000000..0393392c1a
--- /dev/null
+++ b/libs/wal_decoder/proto/interpreted_wal.proto
@@ -0,0 +1,43 @@
+syntax = "proto3";
+
+package interpreted_wal;
+
+message InterpretedWalRecords {
+  repeated InterpretedWalRecord records = 1;
+  optional uint64 next_record_lsn = 2;
+}
+
+message InterpretedWalRecord {
+  optional bytes metadata_record = 1;
+  SerializedValueBatch batch = 2;
+  uint64 next_record_lsn = 3;
+  bool flush_uncommitted = 4;
+  uint32 xid = 5;
+}
+
+message SerializedValueBatch {
+  bytes raw = 1;
+  repeated ValueMeta metadata = 2;
+  uint64 max_lsn = 3;
+  uint64 len = 4;
+}
+
+enum ValueMetaType {
+  Serialized = 0;
+  Observed = 1;
+}
+
+message ValueMeta {
+  ValueMetaType type = 1;
+  CompactKey key = 2;
+  uint64 lsn = 3;
+  optional uint64 batch_offset = 4;
+  optional uint64 len = 5;
+  optional bool will_init = 6;
+}
+
+message CompactKey {
+  int64 high = 1;
+  int64 low = 2;
+}
+
diff --git a/libs/wal_decoder/src/models.rs b/libs/wal_decoder/src/models.rs
index c11ef49495..50bfb4a3d0 100644
--- a/libs/wal_decoder/src/models.rs
+++ b/libs/wal_decoder/src/models.rs
@@ -37,6 +37,17 @@ use utils::lsn::Lsn;
 
 use crate::serialized_batch::SerializedValueBatch;
 
+// Code generated by protobuf.
+pub mod proto {
+    // Tonic does derives as `#[derive(Clone, PartialEq, ::prost::Message)]`
+    // we don't use these types for anything but broker data transmission,
+    // so it's ok to ignore this one.
+    #![allow(clippy::derive_partial_eq_without_eq)]
+    // The generated ValueMeta has a `len` method generate for its `len` field.
+    #![allow(clippy::len_without_is_empty)]
+    tonic::include_proto!("interpreted_wal");
+}
+
 #[derive(Serialize, Deserialize)]
 pub enum FlushUncommittedRecords {
     Yes,