[wip] Feature/gtid memory mode

.github/workflows/ci.yml

@@ -1,10 +1,10 @@
 name: CI
 
-on: 
+on:
   push:
-    branches: [ xredis_2_ror,xredis_2_ror_v1 ]
+    branches: [ xredis_2_ror ]
   pull_request:
-    branches: [ xredis_2_ror,xredis_2_ror_v1 ]
+    branches: [ xredis_2_ror ]
 
 jobs:
    unit:
@@ -33,7 +33,7 @@ jobs:
       run: |
           sudo apt-get update
           sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6
-          make unit-test -j8
+          make SWAP=1 unit-test -j8
    mem:
     strategy:
       matrix:
@@ -89,9 +89,9 @@ jobs:
       run: |
           sudo apt-get update
           sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6
-          make -j8
+          make SWAP=1 -j8
     - name: make test
-      run: make test-disk
+      run: make SWAP=1 test
    swap-asan:
     strategy:
       matrix:
@@ -118,8 +118,8 @@ jobs:
       run: |
           sudo apt-get update
           sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6
-          make SANITIZER=address -j8
+          make SWAP=1 SANTIZER=address -j8
     - name: make test
-      run: make test-asan-disk
+      run: make SWAP=1 test
 
 

.github/workflows/daily.yml

@@ -13,7 +13,7 @@ on:
     inputs:
       use_repo:
         description: 'repo owner and name'
-        default: 'ctripcorp/xredis'
+        default: 'ctripcorp/Redis-On-Rocks'
       use_git_ref:
         description: 'git branch or sha to use'
         default: 'xredis_2_ror'
@@ -22,7 +22,7 @@ jobs:
 
   test-ubuntu-jemalloc:
     runs-on: ubuntu-22.04
-    if: github.repository == 'ctripcorp/xredis'
+    if: github.repository == 'ctripcorp/Redis-On-Rocks'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -47,13 +47,13 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.6
-        ./runtest --swap-mode disk --accurate --verbose --dump-logs
+        ./runtest --accurate --verbose --dump-logs
     - name: unittest
       run: ./src/redis-server test all
 
   test-ubuntu-libc-malloc:
     runs-on: ubuntu-22.04
-    if: github.repository == 'ctripcorp/xredis'
+    if: github.repository == 'ctripcorp/Redis-On-Rocks'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -78,11 +78,11 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.6
-        ./runtest --swap-mode disk --accurate --verbose --dump-logs
+        ./runtest --accurate --verbose --dump-logs
 
   test-sanitizer-address:
     runs-on: ubuntu-22.04
-    if: github.repository == 'ctripcorp/xredis'
+    if: github.repository == 'ctripcorp/Redis-On-Rocks'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -107,13 +107,13 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.6 tclx -y
-        ./runtest --swap-mode disk --tags -nosanitizer --accurate --verbose --dump-logs
+        ./runtest ---tags -nosanitizer -accurate --verbose --dump-logs
     - name: unittest
       run: ./src/redis-server test all
 
   test-sanitizer-undefined:
     runs-on: ubuntu-22.04
-    if: github.repository == 'ctripcorp/xredis'
+    if: github.repository == 'ctripcorp/Redis-On-Rocks'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -138,69 +138,7 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.6 tclx -y
-        ./runtest --swap-mode disk --tags -nosanitizer --accurate --verbose --dump-logs
+        ./runtest --tags -nosanitizer --accurate --verbose --dump-logs
     - name: unittest
       run: ./src/redis-server test all
 
-  test-ubuntu-io-threads:
-    runs-on: ubuntu-22.04
-    if: github.repository == 'ctripcorp/xredis'
-    timeout-minutes: 14400
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-    - name: get rocksdb sha
-      id: rocksdb-sha
-      run: |
-        echo "::set-output name=sha::$(cd deps/rocksdb && git rev-parse HEAD)"
-      shell: bash
-    - name: cache rocksdb
-      id: cache-rocksdb
-      uses: actions/cache@v3
-      with:
-        path: deps/rocksdb
-        key: ${{ runner.os }}-rocksdb-${{ steps.rocksdb-sha.outputs.sha }}
-    - name: make
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6
-        make -j8
-    - name: test
-      run: |
-        sudo apt-get install tcl8.6 tcl-tls
-        ./runtest --swap-mode disk --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs
-
-  test-valgrind:
-    runs-on: ubuntu-22.04
-    if: github.repository == 'ctripcorp/xredis'
-    timeout-minutes: 14400
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-    - name: get rocksdb sha
-      id: rocksdb-sha
-      run: |
-        echo "::set-output name=sha::$(cd deps/rocksdb && git rev-parse HEAD)"
-      shell: bash
-    - name: cache rocksdb
-      id: cache-rocksdb
-      uses: actions/cache@v3
-      with:
-        path: deps/rocksdb
-        key: ${{ runner.os }}-rocksdb-${{ steps.rocksdb-sha.outputs.sha }}
-    - name: make
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6
-        make valgrind REDIS_CFLAGS='-DREDIS_TEST' -j8
-    - name: test
-      run: |
-        sudo apt-get install tcl8.6 valgrind -y
-        ./runtest --swap-mode disk --valgrind --verbose --clients 1 --tags -large-memory --dump-logs
-    - name: unittest
-      run: |
-        valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all
-        if grep -q 0x err.txt; then cat err.txt; exit 1; fi
-

.gitignore

@@ -50,3 +50,8 @@ repack/*
 tests/tmp.tcl
 
 src/data.rocks/*
+now.c
+orig.c
+utils/orig_cpped.c
+utils/now_cpped.c
+utils/tmp.c

redis.conf

@@ -2552,10 +2552,6 @@ rocksdb.ratelimiter.rate_per_sec 1024mb
 #
 gtid-enabled no
 
-# Gtid gaps could consume much memory, set this to limit uuid gap memory.
-# default 1mb.
-# gtid-uuid-gap-max-memory 1mb
-
 # If master & slave gtid gap is less than max-gap, xsync will continue.
 # gtid-xsync-max-gap 10000
 

src/Makefile

@@ -292,6 +292,10 @@ else
     endef
 endif
 
+ifdef SWAP
+	FINAL_CFLAGS+=-DENABLE_SWAP=1
+endif
+
 REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)
 REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS)
 REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL)
@@ -311,7 +315,11 @@ endif
 
 REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
 REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o ctrip.o ctrip_swap.o ctrip_swap_adlist.o ctrip_lru_cache.o ctrip_swap_async.o ctrip_swap_batch.o ctrip_swap_cmd.o ctrip_swap_data.o ctrip_swap_debug.o ctrip_swap_evict.o ctrip_swap_exec.o ctrip_swap_expire.o ctrip_swap_hash.o ctrip_swap_set.o ctrip_swap_list.o ctrip_swap_iter.o ctrip_swap_zset.o ctrip_swap_meta.o ctrip_swap_object.o ctrip_swap_rdb.o ctrip_swap_repl.o ctrip_swap_rio.o ctrip_swap_rocks.o ctrip_swap_stat.o ctrip_swap_sync.o ctrip_swap_thread.o ctrip_swap_util.o ctrip_swap_lock.o ctrip_swap_string.o ctrip_swap_bitmap.o ctrip_swap_compact.o  ctrip_swap_slowlog.o ctrip_swap_blocked.o xredis_gtid.o ctrip_cuckoo_hash.o ctrip_cuckoo_filter.o ctrip_swap_filter.o ctrip_absent_cache.o ctrip_swap_load.o ctrip_swap_dirty.o ctrip_swap_persist.o ctrip_roaring_bitmap.o ctrip_swap_rordb.o ctrip_wtdigest.o
+REDIS_SWAP_OBJ=ctrip_swap.o ctrip_swap_adlist.o ctrip_lru_cache.o ctrip_swap_async.o ctrip_swap_batch.o ctrip_swap_cmd.o ctrip_swap_data.o ctrip_swap_debug.o ctrip_swap_evict.o ctrip_swap_exec.o ctrip_swap_expire.o ctrip_swap_hash.o ctrip_swap_set.o ctrip_swap_list.o ctrip_swap_iter.o ctrip_swap_zset.o ctrip_swap_meta.o ctrip_swap_object.o ctrip_swap_rdb.o ctrip_swap_repl.o ctrip_swap_rio.o ctrip_swap_rocks.o ctrip_swap_stat.o ctrip_swap_sync.o ctrip_swap_thread.o ctrip_swap_util.o ctrip_swap_lock.o ctrip_swap_string.o ctrip_swap_bitmap.o ctrip_swap_compact.o  ctrip_swap_slowlog.o ctrip_swap_blocked.o ctrip_cuckoo_hash.o ctrip_cuckoo_filter.o ctrip_swap_filter.o ctrip_absent_cache.o ctrip_swap_load.o ctrip_swap_dirty.o ctrip_swap_persist.o ctrip_roaring_bitmap.o ctrip_swap_rordb.o ctrip_wtdigest.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o ctrip.o xredis_gtid.o
+ifdef SWAP
+REDIS_SERVER_OBJ+= $(REDIS_SWAP_OBJ)
+endif
 REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o
 REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
@@ -420,10 +428,7 @@ distclean: clean
 .PHONY: distclean
 
 test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME)
-	@(cd ..; ./runtest --swap-mode memory)
-
-test-disk: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME)
-	@(cd ..; ./runtest --swap-mode disk)
+	@(cd ..; ./runtest)
 
 test-asan-disk: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME)
 	@(cd ..; ./runtest --swap-mode disk --tags -nosanitizer)
@@ -475,7 +480,9 @@ helgrind:
 unit-test:
 	$(MAKE) CFLAGS="-DREDIS_TEST" OPTIMIZATION="-O0"
 	@(./$(REDIS_SERVER_NAME) test gtid)
+ifdef SWAP
 	@(./$(REDIS_SERVER_NAME) test swap)
+endif
 
 src/help.h:
 	@../utils/generate-command-help.rb > help.h

src/aof.c

@@ -1525,7 +1525,11 @@ int rewriteAppendOnlyFileRio(rio *aof) {
             if ((key_count++ & 1023) == 0) {
                 long long now = mstime();
                 if (now - updated_time >= 1000) {
+#ifdef ENABLE_SWAP
                     sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, 0, "AOF rewrite");
+#else
+                    sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite");
+#endif
                     updated_time = now;
                 }
             }
@@ -1572,7 +1576,11 @@ int rewriteAppendOnlyFile(char *filename) {
 
     if (server.aof_use_rdb_preamble) {
         int error;
+#ifdef ENABLE_SWAP
         if (rdbSaveRio(&aof,&error,RDBFLAGS_AOF_PREAMBLE,NULL,0) == C_ERR) {
+#else
+        if (rdbSaveRio(&aof,&error,RDBFLAGS_AOF_PREAMBLE,NULL) == C_ERR) {
+#endif
             errno = error;
             goto werr;
         }
@@ -1643,7 +1651,11 @@ int rewriteAppendOnlyFile(char *filename) {
         /* Update COW info */
         long long now = mstime();
         if (now - cow_updated_time >= 1000) {
+#ifdef ENABLE_SWAP
             sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, 0, "AOF rewrite");
+#else
+            sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite");
+#endif
             cow_updated_time = now;
         }
     }
@@ -1766,30 +1778,26 @@ void aofClosePipes(void) {
  */
 int rewriteAppendOnlyFileBackground(void) {
     pid_t childpid;
-    swapForkRocksdbCtx *sfrctx = NULL;
 
     if (hasActiveChildProcess()) return C_ERR;
     if (aofCreatePipes() != C_OK) return C_ERR;
-
-    if (server.swap_mode != SWAP_MODE_MEMORY) {
-        sfrctx = swapForkRocksdbCtxCreate(SWAP_FORK_ROCKSDB_TYPE_SNAPSHOT);
-        if (swapForkRocksdbBefore(sfrctx)) {
-            swapForkRocksdbCtxRelease(sfrctx);
-            return C_ERR;
-        }
+#ifdef ENABLE_SWAP
+    swapForkRocksdbCtx *sfrctx = NULL;
+    sfrctx = swapForkRocksdbCtxCreate(SWAP_FORK_ROCKSDB_TYPE_SNAPSHOT);
+    if (swapForkRocksdbBefore(sfrctx)) {
+        swapForkRocksdbCtxRelease(sfrctx);
+        return C_ERR;
     }
-
+#endif
     if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) {
         char tmpfile[256];
-
-        if (server.swap_mode != SWAP_MODE_MEMORY) {
-            if (swapForkRocksdbAfterChild(sfrctx)) {
-                exit(1);
-            } else {
-                swapForkRocksdbCtxRelease(sfrctx);
-            }
+#ifdef ENABLE_SWAP
+        if (swapForkRocksdbAfterChild(sfrctx)) {
+            exit(1);
+        } else {
+            swapForkRocksdbCtxRelease(sfrctx);
         }
-
+#endif
         /* Child */
         redisSetProcTitle("redis-aof-rewrite");
         redisSetCpuAffinity(server.aof_rewrite_cpulist);
@@ -1801,12 +1809,11 @@ int rewriteAppendOnlyFileBackground(void) {
             exitFromChild(1);
         }
     } else {
+#ifdef ENABLE_SWAP
         /* Parent */
-        if (server.swap_mode != SWAP_MODE_MEMORY) {
-            swapForkRocksdbAfterParent(sfrctx,childpid);
-            swapForkRocksdbCtxRelease(sfrctx);
-        }
-
+        swapForkRocksdbAfterParent(sfrctx,childpid);
+        swapForkRocksdbCtxRelease(sfrctx);
+#endif
         if (childpid == -1) {
             serverLog(LL_WARNING,
                 "Can't rewrite append only file in background: fork: %s",