diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5966bf09643..9af99f7b09d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - submodules: true + submodules: true - name: get rocksdb sha id: rocksdb-sha run: | @@ -31,10 +31,9 @@ jobs: key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_SHA }} - name: unit-test run: | - sudo apt-get update - sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6 - make unit-test -j8 - + sudo apt-get update + sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6 + make unit-test -j8 mem: strategy: matrix: @@ -64,7 +63,6 @@ jobs: make -j8 - name: make test run: make test - swap: strategy: matrix: @@ -94,7 +92,6 @@ jobs: make -j8 - name: make test run: make test-disk - swap-asan: strategy: matrix: @@ -122,7 +119,7 @@ jobs: sudo apt-get update sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6 make SANITIZER=address -j8 - - name: make test + - name: make test run: make test-asan-disk diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 1aab75e50bd..471f5b46e6c 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -107,7 +107,7 @@ jobs: - name: test run: | sudo apt-get install tcl8.6 tclx -y - ./runtest --swap-mode disk --sanitizer --accurate --verbose --dump-logs + ./runtest --swap-mode disk --tags -nosanitizer --accurate --verbose --dump-logs - name: unittest run: ./src/redis-server test all @@ -138,7 +138,7 @@ jobs: - name: test run: | sudo apt-get install tcl8.6 tclx -y - ./runtest --swap-mode disk --sanitizer --accurate --verbose --dump-logs + ./runtest --swap-mode disk --tags -nosanitizer --accurate --verbose --dump-logs - name: unittest run: ./src/redis-server test all diff --git a/src/Makefile b/src/Makefile index 95c9395f6ad..7f4e7b16e86 100644 --- a/src/Makefile +++ b/src/Makefile @@ -430,7 +430,7 @@ test-disk: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDI @(cd ..; ./runtest --swap-mode disk) test-asan-disk: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) - @(cd ..; ./runtest --swap-mode disk --sanitizer) + @(cd ..; ./runtest --swap-mode disk --tags -nosanitizer) test-modules: $(REDIS_SERVER_NAME) @(cd ..; ./runtest-moduleapi) diff --git a/tests/integration/logging.tcl b/tests/integration/logging.tcl index 9c64dce1571..6f960995efa 100644 --- a/tests/integration/logging.tcl +++ b/tests/integration/logging.tcl @@ -36,9 +36,9 @@ if {$system_supported} { # Valgrind will complain that the process terminated by a signal, skip it. # Sanitizer will cause unknown crash, which seems a problem of gcc or sanitizer, skip it. - if {!$::valgrind && !$::sanitizer} { + if {!$::valgrind} { set server_path [tmpdir server1.log] - start_server [list overrides [list dir $server_path]] { + start_server [list overrides [list dir $server_path] tags {"nosanitizer"}] { test "Crash report generated on SIGABRT" { set pid [s process_id] exec kill -SIGABRT $pid diff --git a/tests/swap/integration/client_rate_limit_bug.tcl b/tests/swap/integration/client_rate_limit_bug.tcl index 2450e6fcd53..acfe87d715d 100644 --- a/tests/swap/integration/client_rate_limit_bug.tcl +++ b/tests/swap/integration/client_rate_limit_bug.tcl @@ -15,44 +15,42 @@ proc format_command {args} { } set _ $cmd } -start_server [list overrides [list save ""] ] { - if {!$::sanitizer} { - set master [srv 0 client] - $master config set swap-ratelimit-maxmemory-percentage 100 - $master config set maxmemory-policy allkeys-lru - $master config set maxmemory 20MB - $master config set swap-debug-evict-keys 0 - $master config set hz 100 - # puts [randstring 100000 100000 binary] - set host [srv 0 host] - set port [srv 0 port] - set load_handles [] - set j 0 - for {set j 0} {$j < 20} {incr j} { - set load_handle [start_hash_hset $host $port 0 [expr {$j * 100}] [expr {($j +1)*100 }] ] - lappend load_handles $load_handle - } - after 1000 - wait_for_condition 1000 500 { - [$master dbsize] == 2000 - } else { - fail "Fail to full sync" - } +start_server [list overrides [list save ""] tags {"nosanitizer"}] { + set master [srv 0 client] + $master config set swap-ratelimit-maxmemory-percentage 100 + $master config set maxmemory-policy allkeys-lru + $master config set maxmemory 20MB + $master config set swap-debug-evict-keys 0 + $master config set hz 100 + # puts [randstring 100000 100000 binary] + set host [srv 0 host] + set port [srv 0 port] + set load_handles [] + set j 0 + for {set j 0} {$j < 20} {incr j} { + set load_handle [start_hash_hset $host $port 0 [expr {$j * 100}] [expr {($j +1)*100 }] ] + lappend load_handles $load_handle + } + after 1000 + wait_for_condition 1000 500 { + [$master dbsize] == 2000 + } else { + fail "Fail to full sync" + } - for {set j 0} {$j < 20} {incr j} { - set ele [lindex $load_handles $j] - stop_bg_complex_data $ele - } + for {set j 0} {$j < 20} {incr j} { + set ele [lindex $load_handles $j] + stop_bg_complex_data $ele + } - # hget - for {set j 0} {$j < 100} {incr j} { - set load_handle [start_hash_hget $host $port 0 [expr {$j * 20}] [expr {($j +1)*20 }] ] - lappend load_handles $load_handle - } - after 10000 - for {set j 0} {$j < 100} {incr j} { - set ele [lindex $load_handles $j] - stop_bg_complex_data $ele - } + # hget + for {set j 0} {$j < 100} {incr j} { + set load_handle [start_hash_hget $host $port 0 [expr {$j * 20}] [expr {($j +1)*20 }] ] + lappend load_handles $load_handle + } + after 10000 + for {set j 0} {$j < 100} {incr j} { + set ele [lindex $load_handles $j] + stop_bg_complex_data $ele } } \ No newline at end of file diff --git a/tests/swap/ported/replication.tcl b/tests/swap/ported/replication.tcl index a6d60d42ed4..d7c650c0d66 100644 --- a/tests/swap/ported/replication.tcl +++ b/tests/swap/ported/replication.tcl @@ -51,302 +51,296 @@ start_server {tags {"repl network"}} { } } -start_server {tags {"repl"}} { - if {!$::sanitizer} { - set A [srv 0 client] - set A_host [srv 0 host] - set A_port [srv 0 port] - start_server {} { - set B [srv 0 client] - set B_host [srv 0 host] - set B_port [srv 0 port] +start_server {tags {"repl" "nosanitizer"}} { + set A [srv 0 client] + set A_host [srv 0 host] + set A_port [srv 0 port] + start_server {} { + set B [srv 0 client] + set B_host [srv 0 host] + set B_port [srv 0 port] - test {Set instance A as slave of B} { - $A slaveof $B_host $B_port - wait_for_condition 50 100 { - [lindex [$A role] 0] eq {slave} && - [string match {*master_link_status:up*} [$A info replication]] - } else { - fail "Can't turn the instance into a replica" - } + test {Set instance A as slave of B} { + $A slaveof $B_host $B_port + wait_for_condition 50 100 { + [lindex [$A role] 0] eq {slave} && + [string match {*master_link_status:up*} [$A info replication]] + } else { + fail "Can't turn the instance into a replica" } + } - test {INCRBYFLOAT replication, should not remove expire} { - r set test 1 EX 100 - r incrbyfloat test 0.1 - after 1000 - if {!$::swap_debug_evict_keys} { - assert_equal [$A debug digest] [$B debug digest] - } else { - assert_equal [$A ttl test] [$B ttl test] - } + test {INCRBYFLOAT replication, should not remove expire} { + r set test 1 EX 100 + r incrbyfloat test 0.1 + after 1000 + if {!$::swap_debug_evict_keys} { + assert_equal [$A debug digest] [$B debug digest] + } else { + assert_equal [$A ttl test] [$B ttl test] } + } - test {GETSET replication} { - $A config resetstat - $A config set loglevel debug - $B config set loglevel debug - r set test foo - assert_equal [r getset test bar] foo - wait_for_condition 500 10 { - [$A get test] eq "bar" - } else { - fail "getset wasn't propagated" - } - assert_equal [r set test vaz get] bar - wait_for_condition 500 10 { - [$A get test] eq "vaz" - } else { - fail "set get wasn't propagated" - } - assert_match {*calls=3,*} [cmdrstat set $A] - assert_match {} [cmdrstat getset $A] + test {GETSET replication} { + $A config resetstat + $A config set loglevel debug + $B config set loglevel debug + r set test foo + assert_equal [r getset test bar] foo + wait_for_condition 500 10 { + [$A get test] eq "bar" + } else { + fail "getset wasn't propagated" } - - test {BRPOPLPUSH replication, when blocking against empty list} { - $A config resetstat - set rd [redis_deferring_client] - $rd brpoplpush a b 5 - r lpush a foo - wait_for_condition 50 100 { - [$A debug digest] eq [$B debug digest] - } else { - fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]" - } - assert_match {*calls=1,*} [cmdrstat rpoplpush $A] - assert_match {} [cmdrstat lmove $A] + assert_equal [r set test vaz get] bar + wait_for_condition 500 10 { + [$A get test] eq "vaz" + } else { + fail "set get wasn't propagated" } + assert_match {*calls=3,*} [cmdrstat set $A] + assert_match {} [cmdrstat getset $A] + } - test {BRPOPLPUSH replication, list exists} { - $A config resetstat - set rd [redis_deferring_client] - r lpush c 1 - r lpush c 2 - r lpush c 3 - $rd brpoplpush c d 5 - after 1000 - assert_equal [$A llen c] [$B llen c] - assert_equal [$A llen d] [$B llen d] - assert_match {*calls=1,*} [cmdrstat rpoplpush $A] - assert_match {} [cmdrstat lmove $A] + test {BRPOPLPUSH replication, when blocking against empty list} { + $A config resetstat + set rd [redis_deferring_client] + $rd brpoplpush a b 5 + r lpush a foo + wait_for_condition 50 100 { + [$A debug digest] eq [$B debug digest] + } else { + fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]" } + assert_match {*calls=1,*} [cmdrstat rpoplpush $A] + assert_match {} [cmdrstat lmove $A] + } - foreach wherefrom {left right} { - foreach whereto {left right} { - test "BLMOVE ($wherefrom, $whereto) replication, when blocking against empty list" { - $A config resetstat - set rd [redis_deferring_client] - $rd blmove a b $wherefrom $whereto 5 - r lpush a foo - wait_for_condition 50 100 { - [$A debug digest-value a] eq [$B debug digest-value a] && [$A debug digest-value b] eq [$B debug digest-value b] - } else { - puts "A.a:[$A debug digest-value a], B.a:[$B debug digest-value a], A.b:[$A debug digest-value b] B.b:[$B debug digest-value b]" - fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]" - } - assert_match {*calls=1,*} [cmdrstat lmove $A] - assert_match {} [cmdrstat rpoplpush $A] - } + test {BRPOPLPUSH replication, list exists} { + $A config resetstat + set rd [redis_deferring_client] + r lpush c 1 + r lpush c 2 + r lpush c 3 + $rd brpoplpush c d 5 + after 1000 + assert_equal [$A llen c] [$B llen c] + assert_equal [$A llen d] [$B llen d] + assert_match {*calls=1,*} [cmdrstat rpoplpush $A] + assert_match {} [cmdrstat lmove $A] + } - test "BLMOVE ($wherefrom, $whereto) replication, list exists" { - $A config resetstat - set rd [redis_deferring_client] - r lpush c 1 - r lpush c 2 - r lpush c 3 - $rd blmove c d $wherefrom $whereto 5 - after 1000 - assert_equal [$A llen c] [$B llen c] - assert_equal [$A llen d] [$B llen d] - assert_match {*calls=1,*} [cmdrstat lmove $A] - assert_match {} [cmdrstat rpoplpush $A] + foreach wherefrom {left right} { + foreach whereto {left right} { + test "BLMOVE ($wherefrom, $whereto) replication, when blocking against empty list" { + $A config resetstat + set rd [redis_deferring_client] + $rd blmove a b $wherefrom $whereto 5 + r lpush a foo + wait_for_condition 50 100 { + [$A debug digest-value a] eq [$B debug digest-value a] && [$A debug digest-value b] eq [$B debug digest-value b] + } else { + puts "A.a:[$A debug digest-value a], B.a:[$B debug digest-value a], A.b:[$A debug digest-value b] B.b:[$B debug digest-value b]" + fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]" } + assert_match {*calls=1,*} [cmdrstat lmove $A] + assert_match {} [cmdrstat rpoplpush $A] + } + + test "BLMOVE ($wherefrom, $whereto) replication, list exists" { + $A config resetstat + set rd [redis_deferring_client] + r lpush c 1 + r lpush c 2 + r lpush c 3 + $rd blmove c d $wherefrom $whereto 5 + after 1000 + assert_equal [$A llen c] [$B llen c] + assert_equal [$A llen d] [$B llen d] + assert_match {*calls=1,*} [cmdrstat lmove $A] + assert_match {} [cmdrstat rpoplpush $A] } } + } - test {BLPOP followed by role change, issue #2473} { - set rd [redis_deferring_client] - $rd blpop foo 0 ; # Block while B is a master + test {BLPOP followed by role change, issue #2473} { + set rd [redis_deferring_client] + $rd blpop foo 0 ; # Block while B is a master - # Turn B into master of A - $A slaveof no one - $B slaveof $A_host $A_port - wait_for_condition 50 100 { - [lindex [$B role] 0] eq {slave} && - [string match {*master_link_status:up*} [$B info replication]] - } else { - fail "Can't turn the instance into a replica" - } + # Turn B into master of A + $A slaveof no one + $B slaveof $A_host $A_port + wait_for_condition 50 100 { + [lindex [$B role] 0] eq {slave} && + [string match {*master_link_status:up*} [$B info replication]] + } else { + fail "Can't turn the instance into a replica" + } - # Push elements into the "foo" list of the new replica. - # If the client is still attached to the instance, we'll get - # a desync between the two instances. - $A rpush foo a b c - after 100 + # Push elements into the "foo" list of the new replica. + # If the client is still attached to the instance, we'll get + # a desync between the two instances. + $A rpush foo a b c + after 100 - wait_for_condition 50 100 { - [$A debug digest] eq [$B debug digest] && - [$A lrange foo 0 -1] eq {a b c} && - [$B lrange foo 0 -1] eq {a b c} - } else { - fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]" - } + wait_for_condition 50 100 { + [$A debug digest] eq [$B debug digest] && + [$A lrange foo 0 -1] eq {a b c} && + [$B lrange foo 0 -1] eq {a b c} + } else { + fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]" } } } } -start_server {tags {"repl"}} { - if {!$::sanitizer} { - r set mykey foo +start_server {tags {"repl" "nosanitizer"}} { + r set mykey foo - start_server {} { - test {Second server should have role master at first} { - s role - } {master} + start_server {} { + test {Second server should have role master at first} { + s role + } {master} - test {SLAVEOF should start with link status "down"} { - r multi - r slaveof [srv -1 host] [srv -1 port] - r info replication - r exec - } {*master_link_status:down*} + test {SLAVEOF should start with link status "down"} { + r multi + r slaveof [srv -1 host] [srv -1 port] + r info replication + r exec + } {*master_link_status:down*} - test {The role should immediately be changed to "replica"} { - s role - } {slave} + test {The role should immediately be changed to "replica"} { + s role + } {slave} - wait_for_sync r - test {Sync should have transferred keys from master} { - r get mykey - } {foo} + wait_for_sync r + test {Sync should have transferred keys from master} { + r get mykey + } {foo} - test {The link status should be up} { - s master_link_status - } {up} + test {The link status should be up} { + s master_link_status + } {up} - test {SET on the master should immediately propagate} { - r -1 set mykey bar + test {SET on the master should immediately propagate} { + r -1 set mykey bar - wait_for_condition 500 100 { - [r 0 get mykey] eq {bar} - } else { - fail "SET on master did not propagated on replica" - } + wait_for_condition 500 100 { + [r 0 get mykey] eq {bar} + } else { + fail "SET on master did not propagated on replica" } + } - test {FLUSHALL should replicate} { - r -1 flushall - if {$::valgrind} {after 2000} - list [r -1 dbsize] [r 0 dbsize] - } {0 0} - - test {ROLE in master reports master with a slave} { - set res [r -1 role] - lassign $res role offset slaves - assert {$role eq {master}} - assert {$offset > 0} - assert {[llength $slaves] == 1} - lassign [lindex $slaves 0] master_host master_port slave_offset - assert {$slave_offset <= $offset} - } + test {FLUSHALL should replicate} { + r -1 flushall + if {$::valgrind} {after 2000} + list [r -1 dbsize] [r 0 dbsize] + } {0 0} + + test {ROLE in master reports master with a slave} { + set res [r -1 role] + lassign $res role offset slaves + assert {$role eq {master}} + assert {$offset > 0} + assert {[llength $slaves] == 1} + lassign [lindex $slaves 0] master_host master_port slave_offset + assert {$slave_offset <= $offset} + } - test {ROLE in slave reports slave in connected state} { - set res [r role] - lassign $res role master_host master_port slave_state slave_offset - assert {$role eq {slave}} - assert {$slave_state eq {connected}} - } + test {ROLE in slave reports slave in connected state} { + set res [r role] + lassign $res role master_host master_port slave_state slave_offset + assert {$role eq {slave}} + assert {$slave_state eq {connected}} } } } foreach mdl {no yes} { foreach sdl {disabled swapdb} { - if {!$::sanitizer} { - start_server {tags {"repl"}} { - set master [srv 0 client] - $master config set repl-diskless-sync $mdl - $master config set repl-diskless-sync-delay 1 - set master_host [srv 0 host] - set master_port [srv 0 port] - set slaves {} + start_server {tags {"repl" "nosanitizer"}} { + set master [srv 0 client] + $master config set repl-diskless-sync $mdl + $master config set repl-diskless-sync-delay 1 + set master_host [srv 0 host] + set master_port [srv 0 port] + set slaves {} + start_server {} { + lappend slaves [srv 0 client] start_server {} { lappend slaves [srv 0 client] start_server {} { lappend slaves [srv 0 client] - start_server {} { - lappend slaves [srv 0 client] - test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" { - # start load handles only inside the test, so that the test can be skipped - set load_handle0 [start_bg_complex_data $master_host $master_port 0 100000000] - set load_handle1 [start_bg_complex_data $master_host $master_port 0 100000000] - set load_handle2 [start_bg_complex_data $master_host $master_port 0 100000000] - set load_handle3 [start_write_load $master_host $master_port 8] - set load_handle4 [start_write_load $master_host $master_port 4] - after 5000 ;# wait for some data to accumulate so that we have RDB part for the fork - - # Send SLAVEOF commands to slaves - [lindex $slaves 0] config set repl-diskless-load $sdl - [lindex $slaves 1] config set repl-diskless-load $sdl - [lindex $slaves 2] config set repl-diskless-load $sdl - [lindex $slaves 0] slaveof $master_host $master_port - [lindex $slaves 1] slaveof $master_host $master_port - [lindex $slaves 2] slaveof $master_host $master_port - - # Wait for all the three slaves to reach the "online" - # state from the POV of the master. - set retry 500 - while {$retry} { - set info [r -3 info] - if {[string match {*slave0:*state=online*slave1:*state=online*slave2:*state=online*} $info]} { - break - } else { - incr retry -1 - after 100 - } - } - if {$retry == 0} { - error "assertion:Slaves not correctly synchronized" - } - - # Wait that slaves acknowledge they are online so - # we are sure that DBSIZE and DEBUG DIGEST will not - # fail because of timing issues. - wait_for_condition 500 100 { - [lindex [[lindex $slaves 0] role] 3] eq {connected} && - [lindex [[lindex $slaves 1] role] 3] eq {connected} && - [lindex [[lindex $slaves 2] role] 3] eq {connected} + test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" { + # start load handles only inside the test, so that the test can be skipped + set load_handle0 [start_bg_complex_data $master_host $master_port 0 100000000] + set load_handle1 [start_bg_complex_data $master_host $master_port 0 100000000] + set load_handle2 [start_bg_complex_data $master_host $master_port 0 100000000] + set load_handle3 [start_write_load $master_host $master_port 8] + set load_handle4 [start_write_load $master_host $master_port 4] + after 5000 ;# wait for some data to accumulate so that we have RDB part for the fork + + # Send SLAVEOF commands to slaves + [lindex $slaves 0] config set repl-diskless-load $sdl + [lindex $slaves 1] config set repl-diskless-load $sdl + [lindex $slaves 2] config set repl-diskless-load $sdl + [lindex $slaves 0] slaveof $master_host $master_port + [lindex $slaves 1] slaveof $master_host $master_port + [lindex $slaves 2] slaveof $master_host $master_port + + # Wait for all the three slaves to reach the "online" + # state from the POV of the master. + set retry 500 + while {$retry} { + set info [r -3 info] + if {[string match {*slave0:*state=online*slave1:*state=online*slave2:*state=online*} $info]} { + break } else { - fail "Slaves still not connected after some time" + incr retry -1 + after 100 } + } + if {$retry == 0} { + error "assertion:Slaves not correctly synchronized" + } - # Stop the write load - stop_bg_complex_data $load_handle0 - stop_bg_complex_data $load_handle1 - stop_bg_complex_data $load_handle2 - stop_write_load $load_handle3 - stop_write_load $load_handle4 - - # Make sure no more commands processed - wait_load_handlers_disconnected - - wait_for_ofs_sync $master [lindex $slaves 0] - wait_for_ofs_sync $master [lindex $slaves 1] - wait_for_ofs_sync $master [lindex $slaves 2] - - set dbsize [$master dbsize] - set dbsize0 [[lindex $slaves 0] dbsize] - set dbsize1 [[lindex $slaves 1] dbsize] - set dbsize2 [[lindex $slaves 2] dbsize] - assert {$dbsize>0} - assert {$dbsize==$dbsize0} - assert {$dbsize==$dbsize1} - assert {$dbsize==$dbsize2} - swap_data_comp $master [lindex $slaves 0] - swap_data_comp $master [lindex $slaves 1] - swap_data_comp $master [lindex $slaves 2] + # Wait that slaves acknowledge they are online so + # we are sure that DBSIZE and DEBUG DIGEST will not + # fail because of timing issues. + wait_for_condition 500 100 { + [lindex [[lindex $slaves 0] role] 3] eq {connected} && + [lindex [[lindex $slaves 1] role] 3] eq {connected} && + [lindex [[lindex $slaves 2] role] 3] eq {connected} + } else { + fail "Slaves still not connected after some time" } + + # Stop the write load + stop_bg_complex_data $load_handle0 + stop_bg_complex_data $load_handle1 + stop_bg_complex_data $load_handle2 + stop_write_load $load_handle3 + stop_write_load $load_handle4 + + # Make sure no more commands processed + wait_load_handlers_disconnected + + wait_for_ofs_sync $master [lindex $slaves 0] + wait_for_ofs_sync $master [lindex $slaves 1] + wait_for_ofs_sync $master [lindex $slaves 2] + + set dbsize [$master dbsize] + set dbsize0 [[lindex $slaves 0] dbsize] + set dbsize1 [[lindex $slaves 1] dbsize] + set dbsize2 [[lindex $slaves 2] dbsize] + assert {$dbsize>0} + assert {$dbsize==$dbsize0} + assert {$dbsize==$dbsize1} + assert {$dbsize==$dbsize2} + swap_data_comp $master [lindex $slaves 0] + swap_data_comp $master [lindex $slaves 1] + swap_data_comp $master [lindex $slaves 2] } } } @@ -607,150 +601,148 @@ proc compute_cpu_usage {start end} { # test diskless rdb pipe with multiple replicas, which may drop half way -start_server {tags {"repl"}} { - if {!$::sanitizer} { - set master [srv 0 client] - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 1 - set master_host [srv 0 host] - set master_port [srv 0 port] - set master_pid [srv 0 pid] - # put enough data in the db that the rdb file will be bigger than the socket buffers - # and since we'll have key-load-delay of 100, 20000 keys will take at least 2 seconds - # we also need the replica to process requests during transfer (which it does only once in 2mb) - $master config set rdbcompression no - $master debug populate 20000 test 10000 - $master config set swap-repl-rordb-sync no - # If running on Linux, we also measure utime/stime to detect possible I/O handling issues - set os [catch {exec uname}] - set measure_time [expr {$os == "Linux"} ? 1 : 0] - foreach all_drop {no slow fast all timeout} { - test "diskless $all_drop replicas drop during rdb pipe" { - set replicas {} - set replicas_alive {} - # start one replica that will read the rdb fast, and one that will be slow +start_server {tags {"repl" "nosanitizer"}} { + set master [srv 0 client] + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 1 + set master_host [srv 0 host] + set master_port [srv 0 port] + set master_pid [srv 0 pid] + # put enough data in the db that the rdb file will be bigger than the socket buffers + # and since we'll have key-load-delay of 100, 20000 keys will take at least 2 seconds + # we also need the replica to process requests during transfer (which it does only once in 2mb) + $master config set rdbcompression no + $master debug populate 20000 test 10000 + $master config set swap-repl-rordb-sync no + # If running on Linux, we also measure utime/stime to detect possible I/O handling issues + set os [catch {exec uname}] + set measure_time [expr {$os == "Linux"} ? 1 : 0] + foreach all_drop {no slow fast all timeout} { + test "diskless $all_drop replicas drop during rdb pipe" { + set replicas {} + set replicas_alive {} + # start one replica that will read the rdb fast, and one that will be slow + start_server {} { + lappend replicas [srv 0 client] + lappend replicas_alive [srv 0 client] start_server {} { lappend replicas [srv 0 client] lappend replicas_alive [srv 0 client] - start_server {} { - lappend replicas [srv 0 client] - lappend replicas_alive [srv 0 client] - - # start replication - # it's enough for just one replica to be slow, and have it's write handler enabled - # so that the whole rdb generation process is bound to that - set loglines [count_log_lines -1] - [lindex $replicas 0] config set repl-diskless-load swapdb - [lindex $replicas 0] config set key-load-delay 100 ;# 20k keys and 100 microseconds sleep means at least 2 seconds - [lindex $replicas 0] replicaof $master_host $master_port - [lindex $replicas 1] replicaof $master_host $master_port - - # wait for the replicas to start reading the rdb - # using the log file since the replica only responds to INFO once in 2mb - wait_for_log_messages -1 {"*Loading DB in memory*"} $loglines 800 10 - - if {$measure_time} { - set master_statfile "/proc/$master_pid/stat" - set master_start_metrics [get_cpu_metrics $master_statfile] - set start_time [clock seconds] - } - # wait a while so that the pipe socket writer will be - # blocked on write (since replica 0 is slow to read from the socket) - after 500 + # start replication + # it's enough for just one replica to be slow, and have it's write handler enabled + # so that the whole rdb generation process is bound to that + set loglines [count_log_lines -1] + [lindex $replicas 0] config set repl-diskless-load swapdb + [lindex $replicas 0] config set key-load-delay 100 ;# 20k keys and 100 microseconds sleep means at least 2 seconds + [lindex $replicas 0] replicaof $master_host $master_port + [lindex $replicas 1] replicaof $master_host $master_port + + # wait for the replicas to start reading the rdb + # using the log file since the replica only responds to INFO once in 2mb + wait_for_log_messages -1 {"*Loading DB in memory*"} $loglines 800 10 + + if {$measure_time} { + set master_statfile "/proc/$master_pid/stat" + set master_start_metrics [get_cpu_metrics $master_statfile] + set start_time [clock seconds] + } - # add some command to be present in the command stream after the rdb. - $master incr $all_drop + # wait a while so that the pipe socket writer will be + # blocked on write (since replica 0 is slow to read from the socket) + after 500 - # disconnect replicas depending on the current test - set loglines [count_log_lines -2] - if {$all_drop == "all" || $all_drop == "fast"} { - exec kill [srv 0 pid] - set replicas_alive [lreplace $replicas_alive 1 1] - } - if {$all_drop == "all" || $all_drop == "slow"} { - exec kill [srv -1 pid] - set replicas_alive [lreplace $replicas_alive 0 0] - } - if {$all_drop == "timeout"} { - $master config set repl-timeout 2 - # we want the slow replica to hang on a key for very long so it'll reach repl-timeout - exec kill -SIGSTOP [srv -1 pid] - after 2000 - } + # add some command to be present in the command stream after the rdb. + $master incr $all_drop - # wait for rdb child to exit - wait_for_condition 500 100 { - [s -2 rdb_bgsave_in_progress] == 0 - } else { - fail "rdb child didn't terminate" - } + # disconnect replicas depending on the current test + set loglines [count_log_lines -2] + if {$all_drop == "all" || $all_drop == "fast"} { + exec kill [srv 0 pid] + set replicas_alive [lreplace $replicas_alive 1 1] + } + if {$all_drop == "all" || $all_drop == "slow"} { + exec kill [srv -1 pid] + set replicas_alive [lreplace $replicas_alive 0 0] + } + if {$all_drop == "timeout"} { + $master config set repl-timeout 2 + # we want the slow replica to hang on a key for very long so it'll reach repl-timeout + exec kill -SIGSTOP [srv -1 pid] + after 2000 + } - # make sure we got what we were aiming for, by looking for the message in the log file - if {$all_drop == "all"} { - wait_for_log_messages -2 {"*Diskless rdb transfer, last replica dropped, killing fork child*"} $loglines 1 1 - } - if {$all_drop == "no"} { - wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 2 replicas still up*"} $loglines 1 1 + # wait for rdb child to exit + wait_for_condition 500 100 { + [s -2 rdb_bgsave_in_progress] == 0 + } else { + fail "rdb child didn't terminate" + } + + # make sure we got what we were aiming for, by looking for the message in the log file + if {$all_drop == "all"} { + wait_for_log_messages -2 {"*Diskless rdb transfer, last replica dropped, killing fork child*"} $loglines 1 1 + } + if {$all_drop == "no"} { + wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 2 replicas still up*"} $loglines 1 1 + } + if {$all_drop == "slow" || $all_drop == "fast"} { + wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1 1 + } + if {$all_drop == "timeout"} { + wait_for_log_messages -2 {"*Disconnecting timedout replica (full sync)*"} $loglines 1 1 + wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1 1 + # master disconnected the slow replica, remove from array + set replicas_alive [lreplace $replicas_alive 0 0] + # release it + exec kill -SIGCONT [srv -1 pid] + } + + # make sure we don't have a busy loop going thought epoll_wait + if {$measure_time} { + set master_end_metrics [get_cpu_metrics $master_statfile] + set time_elapsed [expr {[clock seconds]-$start_time}] + set master_cpu [compute_cpu_usage $master_start_metrics $master_end_metrics] + set master_utime [lindex $master_cpu 0] + set master_stime [lindex $master_cpu 1] + if {$::verbose} { + puts "elapsed: $time_elapsed" + puts "master utime: $master_utime" + puts "master stime: $master_stime" } - if {$all_drop == "slow" || $all_drop == "fast"} { - wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1 1 + if {!$::no_latency && ($all_drop == "all" || $all_drop == "slow" || $all_drop == "timeout")} { + assert {$master_utime < 70} + assert {$master_stime < 70} } - if {$all_drop == "timeout"} { - wait_for_log_messages -2 {"*Disconnecting timedout replica (full sync)*"} $loglines 1 1 - wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1 1 - # master disconnected the slow replica, remove from array - set replicas_alive [lreplace $replicas_alive 0 0] - # release it - exec kill -SIGCONT [srv -1 pid] + if {!$::no_latency && ($all_drop == "none" || $all_drop == "fast")} { + assert {$master_utime < 15} + assert {$master_stime < 15} } + } - # make sure we don't have a busy loop going thought epoll_wait - if {$measure_time} { - set master_end_metrics [get_cpu_metrics $master_statfile] - set time_elapsed [expr {[clock seconds]-$start_time}] - set master_cpu [compute_cpu_usage $master_start_metrics $master_end_metrics] - set master_utime [lindex $master_cpu 0] - set master_stime [lindex $master_cpu 1] - if {$::verbose} { - puts "elapsed: $time_elapsed" - puts "master utime: $master_utime" - puts "master stime: $master_stime" - } - if {!$::no_latency && ($all_drop == "all" || $all_drop == "slow" || $all_drop == "timeout")} { - assert {$master_utime < 70} - assert {$master_stime < 70} - } - if {!$::no_latency && ($all_drop == "none" || $all_drop == "fast")} { - assert {$master_utime < 15} - assert {$master_stime < 15} - } + # verify the data integrity + foreach replica $replicas_alive { + # Wait that replicas acknowledge they are online so + # we are sure that DBSIZE and DEBUG DIGEST will not + # fail because of timing issues. + wait_for_condition 150 100 { + [lindex [$replica role] 3] eq {connected} + } else { + fail "replicas still not connected after some time" } - # verify the data integrity - foreach replica $replicas_alive { - # Wait that replicas acknowledge they are online so - # we are sure that DBSIZE and DEBUG DIGEST will not - # fail because of timing issues. - wait_for_condition 150 100 { - [lindex [$replica role] 3] eq {connected} - } else { - fail "replicas still not connected after some time" - } - - # Make sure that replicas and master have same - # number of keys - wait_for_condition 50 100 { - [$master dbsize] == [$replica dbsize] - } else { - fail "Different number of keys between master and replicas after too long time." - } - - set dbsize [$master dbsize] - set dbsize0 [$replica dbsize] - assert {$dbsize > 0} - assert {$dbsize eq $dbsize0} + # Make sure that replicas and master have same + # number of keys + wait_for_condition 50 100 { + [$master dbsize] == [$replica dbsize] + } else { + fail "Different number of keys between master and replicas after too long time." } + + set dbsize [$master dbsize] + set dbsize0 [$replica dbsize] + assert {$dbsize > 0} + assert {$dbsize eq $dbsize0} } } } @@ -801,43 +793,41 @@ test "diskless replication child being killed is collected" { } test "diskless replication read pipe cleanup" { - if {!$::sanitizer} { - # In diskless replication, we create a read pipe for the RDB, between the child and the parent. - # When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered). - # Otherwise, next time we will use the same fd, the registration will be fail (panic), because - # we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl - start_server {tags {"repl"}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - set master_pid [srv 0 pid] - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 0 - $master config set swap-repl-rordb-sync no + # In diskless replication, we create a read pipe for the RDB, between the child and the parent. + # When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered). + # Otherwise, next time we will use the same fd, the registration will be fail (panic), because + # we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl + start_server {tags {"repl" "nosanitizer"}} { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + set master_pid [srv 0 pid] + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 0 + $master config set swap-repl-rordb-sync no - # put enough data in the db, and slowdown the save, to keep the parent busy at the read process - $master config set rdb-key-save-delay 100000 - $master config set rdbcompression no - $master debug populate 20000 test 10000 - start_server {} { - set replica [srv 0 client] - set loglines [count_log_lines 0] - $replica config set repl-diskless-load swapdb - $replica replicaof $master_host $master_port + # put enough data in the db, and slowdown the save, to keep the parent busy at the read process + $master config set rdb-key-save-delay 100000 + $master config set rdbcompression no + $master debug populate 20000 test 10000 + start_server {} { + set replica [srv 0 client] + set loglines [count_log_lines 0] + $replica config set repl-diskless-load swapdb + $replica replicaof $master_host $master_port - # wait for the replicas to start reading the rdb - wait_for_log_messages 0 {"*Loading DB in memory*"} $loglines 800 10 + # wait for the replicas to start reading the rdb + wait_for_log_messages 0 {"*Loading DB in memory*"} $loglines 800 10 - set loglines [count_log_lines 0] - # send FLUSHALL so the RDB child will be killed - $master flushall + set loglines [count_log_lines 0] + # send FLUSHALL so the RDB child will be killed + $master flushall - # wait for another RDB child process to be started - wait_for_log_messages -1 {"*Background RDB transfer started by pid*"} $loglines 800 10 + # wait for another RDB child process to be started + wait_for_log_messages -1 {"*Background RDB transfer started by pid*"} $loglines 800 10 - # make sure master is alive - $master ping - } + # make sure master is alive + $master ping } } } @@ -895,68 +885,66 @@ test {replicaof right after disconnection} { } test {Kill rdb child process if its dumping RDB is not useful} { - if {!$::sanitizer} { - start_server {tags {"repl"}} { - set slave1 [srv 0 client] + start_server {tags {"repl" "nosanitizer"}} { + set slave1 [srv 0 client] + start_server {} { + set slave2 [srv 0 client] start_server {} { - set slave2 [srv 0 client] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - for {set i 0} {$i < 10} {incr i} { - $master set $i $i - } - $master config set swap-repl-rordb-sync no - # Generating RDB will cost 10s(10 * 1s) - $master config set rdb-key-save-delay 1000000 - $master config set repl-diskless-sync no - $master config set save "" + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + for {set i 0} {$i < 10} {incr i} { + $master set $i $i + } + $master config set swap-repl-rordb-sync no + # Generating RDB will cost 10s(10 * 1s) + $master config set rdb-key-save-delay 1000000 + $master config set repl-diskless-sync no + $master config set save "" - $slave1 slaveof $master_host $master_port - $slave2 slaveof $master_host $master_port + $slave1 slaveof $master_host $master_port + $slave2 slaveof $master_host $master_port - # Wait for starting child - wait_for_condition 50 100 { - ([s 0 rdb_bgsave_in_progress] == 1) && - ([string match "*wait_bgsave*" [s 0 slave0]]) && - ([string match "*wait_bgsave*" [s 0 slave1]]) - } else { - fail "rdb child didn't start" - } + # Wait for starting child + wait_for_condition 50 100 { + ([s 0 rdb_bgsave_in_progress] == 1) && + ([string match "*wait_bgsave*" [s 0 slave0]]) && + ([string match "*wait_bgsave*" [s 0 slave1]]) + } else { + fail "rdb child didn't start" + } - # Slave1 disconnect with master - $slave1 slaveof no one - # Shouldn't kill child since another slave wait for rdb - after 100 - assert {[s 0 rdb_bgsave_in_progress] == 1} - - # Slave2 disconnect with master - $slave2 slaveof no one - # Should kill child - wait_for_condition 100 10 { - [s 0 rdb_bgsave_in_progress] eq 0 - } else { - fail "can't kill rdb child" - } + # Slave1 disconnect with master + $slave1 slaveof no one + # Shouldn't kill child since another slave wait for rdb + after 100 + assert {[s 0 rdb_bgsave_in_progress] == 1} - # If have save parameters, won't kill child - $master config set save "900 1" - $slave1 slaveof $master_host $master_port - $slave2 slaveof $master_host $master_port - wait_for_condition 50 100 { - ([s 0 rdb_bgsave_in_progress] == 1) && - ([string match "*wait_bgsave*" [s 0 slave0]]) && - ([string match "*wait_bgsave*" [s 0 slave1]]) - } else { - fail "rdb child didn't start" - } - $slave1 slaveof no one - $slave2 slaveof no one - after 200 - assert {[s 0 rdb_bgsave_in_progress] == 1} - catch {$master shutdown nosave} + # Slave2 disconnect with master + $slave2 slaveof no one + # Should kill child + wait_for_condition 100 10 { + [s 0 rdb_bgsave_in_progress] eq 0 + } else { + fail "can't kill rdb child" + } + + # If have save parameters, won't kill child + $master config set save "900 1" + $slave1 slaveof $master_host $master_port + $slave2 slaveof $master_host $master_port + wait_for_condition 50 100 { + ([s 0 rdb_bgsave_in_progress] == 1) && + ([string match "*wait_bgsave*" [s 0 slave0]]) && + ([string match "*wait_bgsave*" [s 0 slave1]]) + } else { + fail "rdb child didn't start" } + $slave1 slaveof no one + $slave2 slaveof no one + after 200 + assert {[s 0 rdb_bgsave_in_progress] == 1} + catch {$master shutdown nosave} } } } diff --git a/tests/swap/unit/client.tcl b/tests/swap/unit/client.tcl index e5de956e203..935f56fbe61 100644 --- a/tests/swap/unit/client.tcl +++ b/tests/swap/unit/client.tcl @@ -1,95 +1,88 @@ -start_server {tags "clients"} { +start_server {tags "clients" "nosanitizer"} { r config set swap-debug-rio-delay-micro 1000000 set host [srv 0 host] set port [srv 0 port] - # cases run too slow with sanitizer - if (!$::sanitizer) { - test {kill client just finish swap} { - set r1 [redis $host $port] - set r2 [redis $host $port] - $r1 blocking 0 - $r1 sadd s1 m1 m2 m3 - $r1 swap.evict s1 - $r1 sismember s1 m1 - $r2 multi - $r2 slaveof no one - $r2 client kill type normal - $r2 exec + test {kill client just finish swap} { + set r1 [redis $host $port] + set r2 [redis $host $port] + $r1 blocking 0 + $r1 sadd s1 m1 m2 m3 + $r1 swap.evict s1 + $r1 sismember s1 m1 + $r2 multi + $r2 slaveof no one + $r2 client kill type normal + $r2 exec - set r3 [redis $host $port] - $r3 ping - } + set r3 [redis $host $port] + $r3 ping } } -start_server {tags "clients"} { +start_server {tags "clients" "nosanitizer"} { set host [srv 0 host] set port [srv 0 port] - if (!$::sanitizer) { - test {server kill conns} { - set rounds 50 - set load 10 - for {set round 0} {$round < $rounds} {incr round} { - if {0 == $round%10} { puts "test rounds $round" } - set conns {} - for {set i 0} {$i < $load} {incr i} { - lappend conns [start_write_load_ignore_err $host $port 10] - } + test {server kill conns} { + set rounds 50 + set load 10 + for {set round 0} {$round < $rounds} {incr round} { + if {0 == $round%10} { puts "test rounds $round" } + set conns {} + for {set i 0} {$i < $load} {incr i} { + lappend conns [start_write_load_ignore_err $host $port 10] + } - set r [redis $host $port] - while {1} { - if {[llength [regexp -inline -all {name=LOAD_HANDLER} [$r client list]]] == $load} { - break - } + set r [redis $host $port] + while {1} { + if {[llength [regexp -inline -all {name=LOAD_HANDLER} [$r client list]]] == $load} { + break } - $r client kill type normal + } + $r client kill type normal - set r [redis $host $port] - wait_for_condition 10 100 { - ![string match {*name=LOAD_HANDLER*} [$r client list]] - } else { - puts [$r client list] - fail "load_handler(s) still connected after too long time." - } - foreach conn $conns { - stop_write_load $conn - } - $r ping + set r [redis $host $port] + wait_for_condition 10 100 { + ![string match {*name=LOAD_HANDLER*} [$r client list]] + } else { + puts [$r client list] + fail "load_handler(s) still connected after too long time." + } + foreach conn $conns { + stop_write_load $conn } + $r ping } } - if (!$::sanitizer) { - test {client kill conns} { - set rounds 50 - set load 10 - for {set round 0} {$round < $rounds} {incr round} { - if {0 == $round%10} { puts "test rounds $round" } - set conns {} - set r [redis $host $port] - for {set i 0} {$i < $load} {incr i} { - lappend conns [start_write_load $host $port 10] - } + test {client kill conns} { + set rounds 50 + set load 10 + for {set round 0} {$round < $rounds} {incr round} { + if {0 == $round%10} { puts "test rounds $round" } + set conns {} + set r [redis $host $port] + for {set i 0} {$i < $load} {incr i} { + lappend conns [start_write_load $host $port 10] + } - wait_for_condition 10 100 { - [llength [regexp -inline -all {name=LOAD_HANDLER} [$r client list]]] == $load - } else { - fail "start client too slow" - } - foreach conn $conns { - stop_write_load $conn - } + wait_for_condition 10 100 { + [llength [regexp -inline -all {name=LOAD_HANDLER} [$r client list]]] == $load + } else { + fail "start client too slow" + } + foreach conn $conns { + stop_write_load $conn + } - wait_for_condition 100 1000 { - ![string match {*name=LOAD_HANDLER*} [$r client list]] - } else { - puts [$r client list] - fail "load_handler(s) still connected after too long time." - } - $r ping + wait_for_condition 100 1000 { + ![string match {*name=LOAD_HANDLER*} [$r client list]] + } else { + puts [$r client list] + fail "load_handler(s) still connected after too long time." } + $r ping } } diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index e9fa76c0cc7..28849c70783 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -193,7 +193,6 @@ set ::baseport 21111; # initial port for spawned redis servers set ::portcount 8000; # we don't wanna use more than 10000 to avoid collision with cluster bus ports set ::traceleaks 0 set ::valgrind 0 -set ::sanitizer 0 set ::durable 0 set ::tls 0 set ::stack_logging 0 @@ -647,7 +646,6 @@ proc send_data_packet {fd status data} { proc print_help_screen {} { puts [join { "--valgrind Run the test over valgrind." - "--sanitizer Run the test over sanitizer." "--durable suppress test crashes and keep running" "--stack-logging Enable OSX leaks/malloc stack logging." "--accurate Run slow randomized tests for more iterations." @@ -708,8 +706,6 @@ for {set j 0} {$j < [llength $argv]} {incr j} { incr j } elseif {$opt eq {--valgrind}} { set ::valgrind 1 - } elseif {$opt eq {--sanitizer}} { - set ::sanitizer 1 } elseif {$opt eq {--stack-logging}} { if {[string match {*Darwin*} [exec uname -a]]} { set ::stack_logging 1