diff --git a/src/indexer/segment_serializer.rs b/src/indexer/segment_serializer.rs
index 7cb5bf3f5..dab5095df 100644
--- a/src/indexer/segment_serializer.rs
+++ b/src/indexer/segment_serializer.rs
@@ -24,12 +24,25 @@ impl SegmentSerializer {
         // In the merge case this is not necessary because we can kmerge the already sorted
         // segments
         let remapping_required = segment.index().settings().sort_by_field.is_some() && !is_in_merge;
-        let store_component = if remapping_required {
-            SegmentComponent::TempStore
+        let settings = segment.index().settings().clone();
+        let store_writer = if remapping_required {
+            let store_write = segment.open_write(SegmentComponent::TempStore)?;
+            StoreWriter::new(
+                store_write,
+                crate::store::Compressor::None,
+                0, // we want random access on the docs, so we choose a minimal block size. Every
+                // doc will get its own block.
+                settings.docstore_compress_dedicated_thread,
+            )?
         } else {
-            SegmentComponent::Store
+            let store_write = segment.open_write(SegmentComponent::Store)?;
+            StoreWriter::new(
+                store_write,
+                settings.docstore_compression,
+                settings.docstore_blocksize,
+                settings.docstore_compress_dedicated_thread,
+            )?
         };
-        let store_write = segment.open_write(store_component)?;
 
         let fast_field_write = segment.open_write(SegmentComponent::FastFields)?;
         let fast_field_serializer = CompositeFastFieldSerializer::from_write(fast_field_write)?;
@@ -38,13 +51,6 @@ impl SegmentSerializer {
         let fieldnorms_serializer = FieldNormsSerializer::from_write(fieldnorms_write)?;
 
         let postings_serializer = InvertedIndexSerializer::open(&mut segment)?;
-        let settings = segment.index().settings();
-        let store_writer = StoreWriter::new(
-            store_write,
-            settings.docstore_compression,
-            settings.docstore_blocksize,
-            settings.docstore_compress_dedicated_thread,
-        )?;
         Ok(SegmentSerializer {
             segment,
             store_writer,
diff --git a/src/indexer/segment_writer.rs b/src/indexer/segment_writer.rs
index a3479b94e..349f0326e 100644
--- a/src/indexer/segment_writer.rs
+++ b/src/indexer/segment_writer.rs
@@ -393,7 +393,8 @@ fn remap_and_write(
             serializer
                 .segment()
                 .open_read(SegmentComponent::TempStore)?,
-            50,
+            1, /* The docstore is configured to have one doc per block, and each doc is accessed
+                * only once: we don't need caching. */
         )?;
         for old_doc_id in doc_id_map.iter_old_doc_ids() {
             let doc_bytes = store_read.get_document_bytes(old_doc_id)?;