[BUG] performance drops after 30 seconds time period

[osevan]

hey devs,

i figured out some performance drops , my keydb starting with 200k gets and sets, and going to 2 million gets and sets and when i wait 30 seconds and benchmark again it drops again back

this happens on two different vps from different provider.
if this cache issue or buffers, maybe we can tweak that to perform always great like after some period explained here... on this vps no load or traffic registered

time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 218531.45 requests per second, p50=0.431 msec
GET: 233481.20 requests per second, p50=0.319 msec

real 0m8.882s
user 0m0.008s
sys 0m0.000s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 187265.91 requests per second, p50=0.535 msec
GET: 187899.28 requests per second, p50=0.439 msec

real 0m10.673s
user 0m0.007s
sys 0m0.000s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 244678.25 requests per second, p50=0.383 msec
GET: 212811.23 requests per second, p50=0.447 msec

real 0m8.800s
user 0m0.002s
sys 0m0.005s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 213675.22 requests per second, p50=0.407 msec
GET: 202101.86 requests per second, p50=0.423 msec

real 0m9.650s
user 0m0.000s
sys 0m0.010s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 192789.67 requests per second, p50=0.503 msec
GET: 254842.00 requests per second, p50=0.335 msec

real 0m9.123s
user 0m0.000s
sys 0m0.007s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 196656.83 requests per second, p50=0.479 msec
GET: 171762.28 requests per second, p50=0.511 msec

real 0m10.921s
user 0m0.000s
sys 0m0.007s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 177179.30 requests per second, p50=0.519 msec
GET: 255232.27 requests per second, p50=0.431 msec

real 0m9.578s
user 0m0.000s
sys 0m0.007s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 522466.03 requests per second, p50=0.527 msec
GET: 531632.12 requests per second, p50=0.479 msec

real 0m3.813s
user 0m0.004s
sys 0m0.003s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 664451.81 requests per second, p50=0.399 msec
GET: 739645.00 requests per second, p50=0.319 msec

real 0m2.872s
user 0m0.005s
sys 0m0.000s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 613120.75 requests per second, p50=0.447 msec
GET: 593824.25 requests per second, p50=0.383 msec

real 0m3.330s
user 0m0.002s
sys 0m0.004s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 719942.38 requests per second, p50=0.391 msec
GET: 725689.38 requests per second, p50=0.351 msec

real 0m2.778s
user 0m0.002s
sys 0m0.004s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 578703.69 requests per second, p50=0.495 msec
GET: 762776.50 requests per second, p50=0.375 msec

real 0m3.055s
user 0m0.006s
sys 0m0.000s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 583090.38 requests per second, p50=0.503 msec
GET: 594884.00 requests per second, p50=0.431 msec

real 0m3.407s
user 0m0.007s
sys 0m0.000s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 726744.19 requests per second, p50=0.383 msec
GET: 712250.69 requests per second, p50=0.343 msec

real 0m2.795s
user 0m0.003s
sys 0m0.003s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 834724.56 requests per second, p50=0.367 msec
GET: 831946.75 requests per second, p50=0.327 msec

real 0m2.411s
user 0m0.002s
sys 0m0.003s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 746268.62 requests per second, p50=0.359 msec
GET: 788022.06 requests per second, p50=0.319 msec

real 0m2.624s
user 0m0.000s
sys 0m0.005s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 719942.38 requests per second, p50=0.375 msec
GET: 743494.44 requests per second, p50=0.343 msec

real 0m2.747s
user 0m0.006s
sys 0m0.000s
root@vps:~# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 725163.19 requests per second, p50=0.375 msec
GET: 693000.69 requests per second, p50=0.391 msec

real 0m2.835s
user 0m0.157s
sys 0m0.469s

### here passing 1 million requests sec
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1945525.25 requests per second, p50=0.415 msec
GET: 2183406.00 requests per second, p50=0.383 msec

real 0m0.982s
user 0m0.002s
sys 0m0.003s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1039501.00 requests per second, p50=0.639 msec
GET: 1277139.25 requests per second, p50=0.551 msec

real 0m1.759s
user 0m0.006s
sys 0m0.000s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1037344.44 requests per second, p50=0.647 msec
GET: 1233045.62 requests per second, p50=0.543 msec

real 0m1.791s
user 0m0.004s
sys 0m0.005s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1798561.12 requests per second, p50=0.415 msec
GET: 1383125.88 requests per second, p50=0.503 msec

real 0m1.291s
user 0m0.000s
sys 0m0.006s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1848428.75 requests per second, p50=0.431 msec
GET: 1766784.50 requests per second, p50=0.407 msec

real 0m1.122s
user 0m0.002s
sys 0m0.004s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1592356.62 requests per second, p50=0.455 msec
GET: 1269035.50 requests per second, p50=0.535 msec

real 0m1.430s
user 0m0.006s
sys 0m0.000s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1745200.62 requests per second, p50=0.447 msec
GET: 1602564.12 requests per second, p50=0.423 msec

real 0m1.212s
user 0m0.006s
sys 0m0.000s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1251564.38 requests per second, p50=0.567 msec
GET: 1503759.38 requests per second, p50=0.439 msec

real 0m1.478s
user 0m0.003s
sys 0m0.003s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1154734.38 requests per second, p50=0.591 msec
GET: 1324503.38 requests per second, p50=0.487 msec

real 0m1.634s
user 0m0.006s
sys 0m0.000s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1443001.38 requests per second, p50=0.479 msec
GET: 1324503.38 requests per second, p50=0.503 msec

real 0m1.464s
user 0m0.002s
sys 0m0.004s
root@vps:/etc/# time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 1173708.88 requests per second, p50=0.567 msec
GET: 1392757.62 requests per second, p50=0.487 msec

real 0m1.587s
user 0m0.003s
sys 0m0.003s

after waiting 30 sec performance drops again back to 200k set and get

time sudo -u keydb -H keydb-benchmark -h 10.10.20.56 -n 1000000 -t set,get -P 16 -q
SET: 222866.06 requests per second, p50=0.479 msec
GET: 225530.00 requests per second, p50=0.359 msec

real 0m8.937s
user 0m0.000s
sys 0m0.007s

thanks and

best regards

[keithchew]

Hi @osevan

What version of keydb and flags are you running?

[osevan]

What /means you with flags?

Maybe keydb.conf flags? I will paste tomorrow some infos.

Normally master git clone master from 2 month ago

[keithchew]

Instead of master, try an official tag release. There are report of issues with v6.3.4, so maybe try v6.3.2 for a start. Yes, keydb.conf, in particular if you are using flash.

[osevan]

bind 127.0.0.1
#oder für alle interfaces
bind * -::*
unixsocket /tmp/keydb.sock
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
maxmemory 256mb
maxmemory-policy allkeys-lru
maxmemory-samples 10
daemonize no
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile ""
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /keydb/
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
server-threads 2

keydb-server --version
KeyDB server v=255.255.255 sha=603ebb27:1 malloc=jemalloc-5.2.1 bits=64 build=4f81eabfa9da205f

[keithchew]

Don't see anything suspicious there, my advise from my previous post is still valid, try v6.3.2 and perhaps check (and post here) the logs if you still encounter issues.