From ede330792d8c3696c2604c6badc88de2447ab7f6 Mon Sep 17 00:00:00 2001
From: sr_zhao <sr_zhao@trip.com>
Date: Tue, 31 Dec 2024 12:16:56 +0800
Subject: [PATCH] reduce latency spike caused by rordb load by fdatasync sst
 files incrementally.

---
 src/config.c           |  1 +
 src/ctrip_swap_rordb.c | 20 +++++++++++++++++++-
 src/server.h           |  1 +
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index 7910eb98d97..731b60ad5c6 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2893,6 +2893,7 @@ standardConfig configs[] = {
     createBoolConfig("replica-announced", NULL, MODIFIABLE_CONFIG, server.replica_announced, 1, NULL, NULL),
     createBoolConfig("slave-repl-all", NULL, MODIFIABLE_CONFIG, server.repl_slave_repl_all, 0, NULL, NULL),
     createBoolConfig("swap-debug-trace-latency", NULL, MODIFIABLE_CONFIG, server.swap_debug_trace_latency, 0, NULL, NULL),
+    createBoolConfig("swap-rordb-load-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.swap_rordb_load_incremental_fsync, 1, NULL, NULL),
     createBoolConfig("swap-cuckoo-filter-enabled", NULL, MODIFIABLE_CONFIG, server.swap_cuckoo_filter_enabled, 1, NULL, updateSwapCuckooFilterEnabled),
     createBoolConfig("swap-absent-cache-enabled", NULL, MODIFIABLE_CONFIG, server.swap_absent_cache_enabled, 1, NULL, updateSwapAbsentCacheEnabled),
     createBoolConfig("swap-absent-cache-include-subkey", NULL, MODIFIABLE_CONFIG, server.swap_absent_cache_include_subkey, 1, NULL, NULL),
diff --git a/src/ctrip_swap_rordb.c b/src/ctrip_swap_rordb.c
index f4d4651135a..1e04db94fd7 100644
--- a/src/ctrip_swap_rordb.c
+++ b/src/ctrip_swap_rordb.c
@@ -204,7 +204,7 @@ static int rordbLoadSSTFile(rio *rdb, char* path) {
     FILE *fp = NULL;
     char *filepath = NULL, *buffer = NULL;
     size_t buflen = RORDB_SST_READ_BUF_LEN, readlen = 0,
-           filesize, toread, fplen;
+           filesize, toread, fplen, fsynclen = 0;
 
     filename = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL);
     if (filename == NULL) {
@@ -243,6 +243,24 @@ static int rordbLoadSSTFile(rio *rdb, char* path) {
                     strerror(errno),errno);
             goto err;
         }
+
+        if (server.swap_rordb_load_incremental_fsync &&
+                readlen - fsynclen >= REDIS_AUTOSYNC_BYTES) {
+            fflush(fp);
+            if (redis_fsync(fileno(fp)) == -1) {
+                serverLog(LL_WARNING,"[rordb] fsync file failed: %s(%d)",
+                        strerror(errno),errno);
+            }
+            fsynclen = readlen;
+        }
+    }
+
+    if (server.swap_rordb_load_incremental_fsync) {
+        fflush(fp);
+        if (redis_fsync(fileno(fp)) == -1) {
+            serverLog(LL_WARNING,"[rordb] fsync file failed: %s(%d)",
+                    strerror(errno),errno);
+        }
     }
 
     serverLog(LL_VERBOSE, "[rordb] load sst file(%s) ok.", filepath);
diff --git a/src/server.h b/src/server.h
index 7e1733a1ec8..dc0922d9aef 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1825,6 +1825,7 @@ struct redisServer {
     int swap_debug_bgsave_metalen_addition;
     int swap_debug_compaction_filter_delay_micro;
     int swap_debug_rdb_key_save_delay_micro;
+    int swap_rordb_load_incremental_fsync;
 
     /* repl swap */
     int repl_workers;   /* num of repl worker clients */