listpack ziplist 性能测试

/* 测试数据生成 */
 typedef struct {
    char **s;
    uint32_t *len;
    size_t n;
} items_t;

static void items_init(items_t *it, size_t n, uint32_t minL, uint32_t maxL, unsigned seed) {
    it->n = n;
    it->s = (char**)zmalloc(n * sizeof(char*));
    it->len = (uint32_t*)zmalloc(n * sizeof(uint32_t));
    assert(it->s && it->len);
    srand(seed);

    for (size_t i = 0; i < n; i++) {
        uint32_t L;
        if (rand() % 8 == 0) {
            L = minL + (rand() % (maxL * 4));
        } else {
            L = minL + (rand() % maxL);
        }
        if (L == 0) L = 1;
        it->len[i] = L;
        it->s[i] = (char*)zmalloc(L + 1);
        assert(it->s[i]);

        for (uint32_t j = 0; j < L; j++) {
            it->s[i][j] = (char)('a' + (int)((i + j) % 26));
        }
        it->s[i][L] = '\0';
    }
}

static void items_free(items_t *it) {
    if (!it || !it->s) return;
    for (size_t i = 0; i < it->n; i++) zfree(it->s[i]);
    zfree(it->s);
    zfree(it->len);
    it->s = NULL; it->len = NULL; it->n = 0;
}

/* 性能基准测试 */
static void benchmark_operations(size_t n, uint32_t minL, uint32_t maxL, unsigned seed) {
    printf("=== Performance benchmark (n=%zu, minL=%u, maxL=%u) ===\n", n, minL, maxL);

    items_t it;
    items_init(&it, n, minL, maxL, seed);

    printf("--- Listpack Performance ---\n");

    /* 创建 listpack */
    unsigned char *lp = lpNew(0);
    assert(lp != NULL);

    /* 添加元素 */
    for (size_t i = 0; i < it.n; i++) {
        unsigned char *new_lp = lpAppend(lp, (unsigned char*)it.s[i], it.len[i]);
        if (new_lp == NULL) {
            printf("Failed to append to listpack\n");
            lpFree(lp);
            items_free(&it);
            return;
        }
        lp = new_lp;
    }

    printf("Listpack size: %zu bytes, elements: %lu\n", lpBytes(lp), lpLength(lp));

    /* 正向遍历性能 */
    long long start = ustime();
    unsigned char *p = lpFirst(lp);
    size_t count = 0;
    while (p) {
        int64_t vlen;
        unsigned char intbuf[LP_INTBUF_SIZE];
        unsigned char *vstr = lpGet(p, &vlen, intbuf);
        if (vstr) {
            count++;
        }
        p = lpNext(lp, p);
    }
    long long forward_time = ustime() - start;

    printf("Forward traversal: %lld us, count: %zu\n", forward_time, count);

    /* 反向遍历性能 */
    start = ustime();
    p = lpLast(lp);
    count = 0;
    while (p) {
        int64_t vlen;
        unsigned char intbuf[LP_INTBUF_SIZE];
        unsigned char *vstr = lpGet(p, &vlen, intbuf);
        if (vstr) {
            count++;
        }
        p = lpPrev(lp, p);
    }
    long long reverse_time = ustime() - start;

    printf("Reverse traversal: %lld us, count: %zu\n", reverse_time, count);

    /* 随机访问性能 */
    start = ustime();
    for (size_t i = 0; i < it.n; i += 100) {
        if (i < it.n) {
            p = lpSeek(lp, i);
            if (p) {
                int64_t vlen;
                unsigned char intbuf[LP_INTBUF_SIZE];
                lpGet(p, &vlen, intbuf);
            }
        }
    }
    long long random_time = ustime() - start;

    printf("Random access: %lld us\n", random_time);

    lpFree(lp);

    printf("--- Ziplist Performance ---\n");

    /* 创建 ziplist */
    unsigned char *zl = ziplistNew();
    assert(zl != NULL);

    /* 添加元素 */
    for (size_t i = 0; i < it.n; i++) {
        zl = ziplistPush(zl, (unsigned char*)it.s[i], it.len[i], ZIPLIST_TAIL);
        if (zl == NULL) {
            printf("Failed to push to ziplist\n");
            zfree(zl);
            items_free(&it);
            return;
        }
    }

    printf("Ziplist size: %zu bytes, elements: %u\n", ziplistBlobLen(zl), ziplistLen(zl));

    /* 正向遍历性能 */
    start = ustime();
    p = ziplistIndex(zl, 0);
    count = 0;
    while (p) {
        unsigned char *vstr;
        unsigned int vlen;
        long long vlong;

        if (ziplistGet(p, &vstr, &vlen, &vlong)) {
            count++;
        }
        p = ziplistNext(zl, p);
    }
    long long zl_forward_time = ustime() - start;

    printf("Forward traversal: %lld us, count: %zu\n", zl_forward_time, count);

    /* 反向遍历性能 */
    start = ustime();
    p = ziplistIndex(zl, -1);
    count = 0;
    while (p) {
        unsigned char *vstr;
        unsigned int vlen;
        long long vlong;

        if (ziplistGet(p, &vstr, &vlen, &vlong)) {
            count++;
        }
        p = ziplistPrev(zl, p);
    }
    long long zl_reverse_time = ustime() - start;

    printf("Reverse traversal: %lld us, count: %zu\n", zl_reverse_time, count);

    /* 随机访问性能 */
    start = ustime();
    for (size_t i = 0; i < it.n; i += 100) {
        if (i < it.n) {
            p = ziplistIndex(zl, i);
            if (p) {
                unsigned char *vstr;
                unsigned int vlen;
                long long vlong;
                ziplistGet(p, &vstr, &vlen, &vlong);
            }
        }
    }
    long long zl_random_time = ustime() - start;

    printf("Random access: %lld us\n", zl_random_time);

    zfree(zl);

    /* 性能对比 */
    printf("--- Performance Comparison ---\n");
    printf("Forward: listpack=%.1fx faster\n", (double)zl_forward_time / forward_time);
    printf("Reverse: listpack=%.1fx faster\n", (double)zl_reverse_time / reverse_time);
    printf("Random: listpack=%.1fx faster\n", (double)zl_random_time / random_time);

    items_free(&it);
}

/* 内存使用对比 */
static void memory_usage_comparison(void) {
    printf("=== Memory usage comparison ===\n");

    items_t it;
    items_init(&it, 10000, 1, 64, 2028);

    /* Listpack 内存使用 */
    unsigned char *lp = lpNew(0);
    for (size_t i = 0; i < it.n; i++) {
        lp = lpAppend(lp, (unsigned char*)it.s[i], it.len[i]);
    }
    size_t lp_size = lpBytes(lp);
    unsigned long lp_elements = lpLength(lp);

    /* Ziplist 内存使用 */
    unsigned char *zl = ziplistNew();
    for (size_t i = 0; i < it.n; i++) {
        zl = ziplistPush(zl, (unsigned char*)it.s[i], it.len[i], ZIPLIST_TAIL);
    }
    size_t zl_size = ziplistBlobLen(zl);
    unsigned int zl_elements = ziplistLen(zl);

    printf("Listpack: %zu bytes, %lu elements, %.2f bytes/element\n",
           lp_size, lp_elements, (double)lp_size / lp_elements);
    printf("Ziplist:  %zu bytes, %u elements, %.2f bytes/element\n",
           zl_size, zl_elements, (double)zl_size / zl_elements);
    printf("Compression ratio: listpack=%.1f%%, ziplist=%.1f%%\n",
           100.0 * lp_size / (it.n * 32), 100.0 * zl_size / (it.n * 32));

    lpFree(lp);
    zfree(zl);
    items_free(&it);
}
 
int main(void) {

    printf("Redis listpack vs ziplist performance comparison test\n");
    printf("==================================================\n");

    /* 基准测试 */
    benchmark_operations(10000, 1, 32, 2024);

    /* 压力测试 */
    printf("=== Stress test: many small items ===\n");
    benchmark_operations(50000, 1, 8, 2025);

    printf("=== Stress test: mixed size items ===\n");
    benchmark_operations(20000, 1, 128, 2026);

    printf("=== Stress test: large items ===\n");
    benchmark_operations(10000, 64, 512, 2027);

    /* 内存使用对比 */
    memory_usage_comparison();

    printf("=== Test completed ===\n");
    return 0;
}
解读报告 
Redis listpack vs ziplist performance comparison test
==================================================
=== Performance benchmark (n=10000, minL=1, maxL=32) ===
--- Listpack Performance ---
Listpack size: 244081 bytes, elements: 10000
Forward traversal: 152 us, count: 10000
Reverse traversal: 147 us, count: 10000
Random access: 3131 us
--- Ziplist Performance ---
Ziplist size: 244051 bytes, elements: 10000
Forward traversal: 204 us, count: 10000
Reverse traversal: 160 us, count: 10000
Random access: 3695 us
--- Performance Comparison ---
Forward: listpack=1.3x faster
Reverse: listpack=1.1x faster
Random: listpack=1.2x faster
=== Stress test: many small items ===
=== Performance benchmark (n=50000, minL=1, maxL=8) ===
--- Listpack Performance ---
Listpack size: 396282 bytes, elements: 50000
Forward traversal: 694 us, count: 50000
Reverse traversal: 701 us, count: 50000
Random access: 81403 us
--- Ziplist Performance ---
Ziplist size: 396286 bytes, elements: 50000
Forward traversal: 981 us, count: 50000
Reverse traversal: 744 us, count: 50000
Random access: 86004 us
--- Performance Comparison ---
Forward: listpack=1.4x faster
Reverse: listpack=1.1x faster
Random: listpack=1.1x faster
=== Stress test: mixed size items ===
=== Performance benchmark (n=20000, minL=1, maxL=128) ===
--- Listpack Performance ---
Listpack size: 1818474 bytes, elements: 20000
Forward traversal: 443 us, count: 20000
Reverse traversal: 418 us, count: 20000
Random access: 18969 us
--- Ziplist Performance ---
Ziplist size: 1821221 bytes, elements: 20000
Forward traversal: 511 us, count: 20000
Reverse traversal: 387 us, count: 20000
Random access: 22066 us
--- Performance Comparison ---
Forward: listpack=1.2x faster
Reverse: listpack=0.9x faster
Random: listpack=1.2x faster
=== Stress test: large items ===
=== Performance benchmark (n=10000, minL=64, maxL=512) ===
--- Listpack Performance ---
Listpack size: 4211033 bytes, elements: 10000
Forward traversal: 294 us, count: 10000
Reverse traversal: 203 us, count: 10000
Random access: 4543 us
--- Ziplist Performance ---
Ziplist size: 4229211 bytes, elements: 10000
Forward traversal: 273 us, count: 10000
Reverse traversal: 194 us, count: 10000
Random access: 5431 us
--- Performance Comparison ---
Forward: listpack=0.9x faster
Reverse: listpack=1.0x faster
Random: listpack=1.2x faster
=== Memory usage comparison ===
Listpack: 463294 bytes, 10000 elements, 46.33 bytes/element
Ziplist:  462803 bytes, 10000 elements, 46.28 bytes/element
Compression ratio: listpack=144.8%, ziplist=144.6%
=== Test completed ===

Listpack 在大多数场景下都优于 Ziplist,但优势并不像预期那样显著

内存压缩效率对比

测试场景 Listpack Ziplist 差异
小数据 (n=10000, 1-32字节) 244,081 bytes 244,051 bytes 几乎相同
大量小数据 (n=50000, 1-8字节) 396,282 bytes 396,286 bytes 几乎相同
混合数据 (n=20000, 1-128字节) 1,818,474 bytes 1,821,221 bytes Listpack 略优
大数据 (n=10000, 64-512字节) 4,211,033 bytes 4,229,211 bytes Listpack 略优

内存压缩率差异很小,说明两种格式的编码效率相当

遍历性能分析

正向遍历(Forward)

  • 小数据: Listpack 1.3x 更快

  • 大量小数据: Listpack 1.4x 更快

  • 混合数据: Listpack 1.2x 更快

  • 大数据: Ziplist 1.1x 更快

分析:Listpack 在正向遍历上普遍有 20-40% 的性能优势。

反向遍历(Reverse)

  • 小数据: Listpack 1.1x 更快

  • 大量小数据: Listpack 1.1x 更快

  • 混合数据: Ziplist 1.1x 更快

  • 大数据: Listpack 1.0x 更快

关键发现:反向遍历性能差异很小!这说明:

  1. Ziplist 的反向遍历实现已经优化得很好

  2. Listpack 的 backlen 优势在实际测试中并不明显

随机访问性能

测试场景 Listpack Ziplist 性能比
小数据 3,131 us 3,695 us 1.2x
大量小数据 81,403 us 86,004 us 1.1x
混合数据 18,969 us 22,066 us 1.2x
大数据 4,543 us 5,431 us 1.2x

Listpack 在随机访问上稳定地有 10-20% 的优势

为什么 Listpack 优势不如预期?

1. Ziplist 已经优化得很好

  • Redis 的 Ziplist 实现经过多年优化

  • 反向遍历虽然理论上是 O(n²),但实际实现中可能使用了缓存或其他优化

2. 数据规模限制

  • 测试数据量相对较小(最大 50,000 元素)

  • 在实际 Redis 使用中,Ziplist 通常有大小限制,超过阈值会转换为其他数据结构

3. 内存局部性

  • 两种格式都存储在连续内存中

  • CPU 缓存友好的访问模式可能掩盖了算法复杂度的差异

实际应用意义

Listpack 的真正优势

  1. 安全性: 避免了 Ziplist 的整数溢出问题
  2. 一致性: 更简洁的编码逻辑,减少 bug 风险
  3. 维护性: 代码更清晰,便于维护和扩展
  4. 未来性: 为后续功能扩展提供更好的基础

性能提升

  • 虽然性能提升不如预期显著,但 1.1-1.4x 的提升仍然是可观的

  • 在 Redis 的高并发场景下,这些微小的性能提升会被放大