mirror/phpredis
1
0
Fork 0
mirror of https://github.com/phpredis/phpredis.git synced 2026-06-19 07:35:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for Zstd compression

Browse Source
This commit is contained in:
Remi Collet
2019-06-26 16:26:08 +02:00
parent 5276474812
commit 2abc61da31
8 changed files with 192 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+4 -2
View File
@@ -28,10 +28,12 @@ matrix:
env: CC=clang
addons:
apt:
packages: clang
packages:
- clang
- libzstd1-dev
before_install:
- phpize
- CFGARGS="--enable-redis-lzf"
- CFGARGS="--enable-redis-lzf --enable-redis-zstd"
- pecl install igbinary && CFGARGS="$CFGARGS --enable-redis-igbinary"
- pecl install msgpack && CFGARGS="$CFGARGS --enable-redis-msgpack"
- ./configure $CFGARGS
Changelog.md
+6
View File
@@ -7,6 +7,12 @@ and PhpRedis adhears to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
## [Unreleased]
### Added
- Add optional support for Zstd compression, using `--enable-redis-ztsd`.
This requires libzstd version >= 1.3.0 [PR #1382](https://github.com/phpredis/phpredis/pull/1582)
([Remi Collet](https://github.com/remicollet))
### Fixed
- RedisCluster segfaults after second connection with cache_slots enabled [f52cd237](https://github.com/phpredis/phpredis/pull/1592/commits/f52cd237)
common.h
+3
View File
@@ -79,6 +79,7 @@ typedef enum _PUBSUB_TYPE {
#define REDIS_OPT_TCP_KEEPALIVE 6
#define REDIS_OPT_COMPRESSION 7
#define REDIS_OPT_REPLY_LITERAL 8
#define REDIS_OPT_COMPRESSION_LEVEL 9
/* cluster options */
#define REDIS_FAILOVER_NONE 0
@@ -96,6 +97,7 @@ typedef enum {
/* compression */
#define REDIS_COMPRESSION_NONE 0
#define REDIS_COMPRESSION_LZF 1
#define REDIS_COMPRESSION_ZSTD 2
/* SCAN options */
#define REDIS_SCAN_NORETRY 0
@@ -258,6 +260,7 @@ typedef struct {
redis_serializer serializer;
int compression;
int compression_level;
long dbNumber;
zend_string *prefix;
config.m4
+35
View File
@@ -23,6 +23,12 @@ PHP_ARG_ENABLE(redis-lzf, whether to enable lzf compression,
PHP_ARG_WITH(liblzf, use system liblzf,
[ --with-liblzf[=DIR] Use system liblzf], no, no)
PHP_ARG_ENABLE(redis-zstd, whether to enable Zstd compression,
[ --enable-redis-zstd Enable Zstd compression support], no, no)
PHP_ARG_WITH(libzstd, use system libsztd,
[ --with-libzstd[=DIR] Use system libzstd], yes, no)
if test "$PHP_REDIS" != "no"; then
if test "$PHP_REDIS_SESSION" != "no"; then
@@ -188,6 +194,35 @@ if test "$PHP_REDIS" != "no"; then
fi
fi
if test "$PHP_REDIS_ZSTD" != "no"; then
AC_DEFINE(HAVE_REDIS_ZSTD, 1, [ ])
if test "$PHP_LIBZSTD" != "no"; then
AC_MSG_CHECKING(for libzstd files in default path)
for i in $PHP_LIBZSTD /usr/local /usr; do
if test -r $i/include/zstd.h; then
AC_MSG_RESULT(found in $i)
LIBZSTD_DIR=$i
break
fi
done
if test -z "$LIBZSTD_DIR"; then
AC_MSG_RESULT([not found])
AC_MSG_ERROR([Please reinstall the libzstd distribution])
fi
PHP_CHECK_LIBRARY(zstd, ZSTD_getFrameContentSize,
[
PHP_ADD_LIBRARY_WITH_PATH(zstd, $LIBZSTD_DIR/$PHP_LIBDIR, REDIS_SHARED_LIBADD)
], [
AC_MSG_ERROR([could not find usable libzstd, version 1.3.0 required])
], [
-L$LIBZSTD_DIR/$PHP_LIBDIR
])
PHP_SUBST(REDIS_SHARED_LIBADD)
else
AC_MSG_ERROR([only system libzstd is supported])
fi
fi
AC_CHECK_PROG([GIT], [git], [yes], [no])
if test "$GIT" == "yes" && test -d "$srcdir/.git"; then
AC_DEFINE_UNQUOTED(GIT_REVISION, ["$(git log -1 --format=%H)"], [ ])
library.c
+93 -32
View File
@@ -21,6 +21,10 @@
#endif
#endif
#ifdef HAVE_REDIS_ZSTD
#include <zstd.h>
#endif
#include <zend_exceptions.h>
#include "php_redis.h"
#include "library.h"
@@ -1764,6 +1768,7 @@ redis_sock_create(char *host, int host_len, unsigned short port,
redis_sock->serializer = REDIS_SERIALIZER_NONE;
redis_sock->compression = REDIS_COMPRESSION_NONE;
redis_sock->compression_level = 0; /* default */
redis_sock->mode = ATOMIC;
redis_sock->head = NULL;
redis_sock->current = NULL;
@@ -2186,26 +2191,60 @@ redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len TSRMLS_DC
char *buf;
int valfree;
size_t len;
#ifdef HAVE_REDIS_LZF
char *data;
uint32_t res;
double size;
#endif
valfree = redis_serialize(redis_sock, z, &buf, &len TSRMLS_CC);
switch (redis_sock->compression) {
case REDIS_COMPRESSION_LZF:
#ifdef HAVE_REDIS_LZF
/* preserve compatibility with PECL lzf_compress margin (greater of 4% and LZF_MARGIN) */
size = len + MIN(UINT_MAX - len, MAX(LZF_MARGIN, len / 25));
data = emalloc(size);
if ((res = lzf_compress(buf, len, data, size)) > 0) {
if (valfree) efree(buf);
*val = data;
*val_len = res;
return 1;
{
char *data;
uint32_t res;
double size;
/* preserve compatibility with PECL lzf_compress margin (greater of 4% and LZF_MARGIN) */
size = len + MIN(UINT_MAX - len, MAX(LZF_MARGIN, len / 25));
data = emalloc(size);
if ((res = lzf_compress(buf, len, data, size)) > 0) {
if (valfree) efree(buf);
*val = data;
*val_len = res;
return 1;
}
efree(data);
}
#endif
break;
case REDIS_COMPRESSION_ZSTD:
#ifdef HAVE_REDIS_ZSTD
{
char *data;
size_t size;
int level;
if (redis_sock->compression_level < 1) {
#ifdef ZSTD_CLEVEL_DEFAULT
level = ZSTD_CLEVEL_DEFAULT;
#else
level = 3;
#endif
} else if (redis_sock->compression_level > ZSTD_maxCLevel()) {
level = ZSTD_maxCLevel();
} else {
level = redis_sock->compression_level;
}
size = ZSTD_compressBound(len);
data = emalloc(size);
size = ZSTD_compress(data, size, buf, len, level);
if (!ZSTD_isError(size)) {
if (valfree) efree(buf);
data = erealloc(data, size);
*val = data;
*val_len = size;
return 1;
}
efree(data);
}
efree(data);
#endif
break;
}
@@ -2217,29 +2256,51 @@ redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len TSRMLS_DC
PHP_REDIS_API int
redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret TSRMLS_DC)
{
#ifdef HAVE_REDIS_LZF
char *data;
int i;
uint32_t res;
#endif
switch (redis_sock->compression) {
case REDIS_COMPRESSION_LZF:
#ifdef HAVE_REDIS_LZF
errno = E2BIG;
/* start from two-times bigger buffer and
* increase it exponentially if needed */
for (i = 2; errno == E2BIG; i *= 2) {
data = emalloc(i * val_len);
if ((res = lzf_decompress(val, val_len, data, i * val_len)) == 0) {
/* errno != E2BIG will brake for loop */
{
char *data;
int i;
uint32_t res;
errno = E2BIG;
/* start from two-times bigger buffer and
* increase it exponentially if needed */
for (i = 2; errno == E2BIG; i *= 2) {
data = emalloc(i * val_len);
if ((res = lzf_decompress(val, val_len, data, i * val_len)) == 0) {
/* errno != E2BIG will brake for loop */
efree(data);
continue;
} else if (redis_unserialize(redis_sock, data, res, z_ret TSRMLS_CC) == 0) {
ZVAL_STRINGL(z_ret, data, res);
}
efree(data);
continue;
} else if (redis_unserialize(redis_sock, data, res, z_ret TSRMLS_CC) == 0) {
ZVAL_STRINGL(z_ret, data, res);
return 1;
}
}
#endif
break;
case REDIS_COMPRESSION_ZSTD:
#ifdef HAVE_REDIS_ZSTD
{
char *data;
size_t len;
len = ZSTD_getFrameContentSize(val, val_len);
if (len >= 0) {
data = emalloc(len);
len = ZSTD_decompress(data, len, val, val_len);
if (ZSTD_isError(len)) {
efree(data);
break;
} else if (redis_unserialize(redis_sock, data, len, z_ret TSRMLS_CC) == 0) {
ZVAL_STRINGL(z_ret, data, len);
}
efree(data);
return 1;
}
efree(data);
return 1;
}
#endif
break;
redis.c
+28 -1
View File
@@ -38,6 +38,10 @@
#include "library.h"
#ifdef HAVE_REDIS_ZSTD
#include <zstd.h>
#endif
#ifdef PHP_SESSION
extern ps_module ps_mod_redis;
extern ps_module ps_mod_redis_cluster;
@@ -685,6 +689,7 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster TSRMLS_DC)
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_TCP_KEEPALIVE"), REDIS_OPT_TCP_KEEPALIVE TSRMLS_CC);
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_COMPRESSION"), REDIS_OPT_COMPRESSION TSRMLS_CC);
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_REPLY_LITERAL"), REDIS_OPT_REPLY_LITERAL);
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_COMPRESSION_LEVEL"), REDIS_OPT_COMPRESSION_LEVEL);
/* serializer */
zend_declare_class_constant_long(ce, ZEND_STRL("SERIALIZER_NONE"), REDIS_SERIALIZER_NONE TSRMLS_CC);
@@ -702,6 +707,16 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster TSRMLS_DC)
#ifdef HAVE_REDIS_LZF
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_LZF"), REDIS_COMPRESSION_LZF TSRMLS_CC);
#endif
#ifdef HAVE_REDIS_ZSTD
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_ZSTD"), REDIS_COMPRESSION_ZSTD);
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_ZSTD_MIN"), 1);
#ifdef ZSTD_CLEVEL_DEFAULT
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_ZSTD_DEFAULT"), ZSTD_CLEVEL_DEFAULT);
#else
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_ZSTD_DEFAULT"), 3);
#endif
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_ZSTD_MAX"), ZSTD_maxCLevel());
#endif
/* scan options*/
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_SCAN"), REDIS_OPT_SCAN TSRMLS_CC);
@@ -852,6 +867,8 @@ get_available_serializers(void)
*/
PHP_MINFO_FUNCTION(redis)
{
smart_str names = {0,};
php_info_print_table_start();
php_info_print_table_header(2, "Redis Support", "enabled");
php_info_print_table_row(2, "Redis Version", PHP_REDIS_VERSION);
@@ -860,8 +877,18 @@ PHP_MINFO_FUNCTION(redis)
#endif
php_info_print_table_row(2, "Available serializers", get_available_serializers());
#ifdef HAVE_REDIS_LZF
php_info_print_table_row(2, "Available compression", "lzf");
smart_str_appends(&names, "lzf");
#endif
#ifdef HAVE_REDIS_ZSTD
if (names.s) {
smart_str_appends(&names, ", ");
}
smart_str_appends(&names, "zstd");
#endif
if (names.s) {
php_info_print_table_row(2, "Available compression", names.s->val);
}
smart_str_free(&names);
php_info_print_table_end();
DISPLAY_INI_ENTRIES();
redis_commands.c
+9
View File
@@ -3888,6 +3888,8 @@ void redis_getoption_handler(INTERNAL_FUNCTION_PARAMETERS,
RETURN_LONG(redis_sock->serializer);
case REDIS_OPT_COMPRESSION:
RETURN_LONG(redis_sock->compression);
case REDIS_OPT_COMPRESSION_LEVEL:
RETURN_LONG(redis_sock->compression_level);
case REDIS_OPT_PREFIX:
if (redis_sock->prefix) {
RETURN_STRINGL(ZSTR_VAL(redis_sock->prefix), ZSTR_LEN(redis_sock->prefix));
@@ -3950,12 +3952,19 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS,
if (val_long == REDIS_COMPRESSION_NONE
#ifdef HAVE_REDIS_LZF
|| val_long == REDIS_COMPRESSION_LZF
#endif
#ifdef HAVE_REDIS_ZSTD
|| val_long == REDIS_COMPRESSION_ZSTD
#endif
) {
redis_sock->compression = val_long;
RETURN_TRUE;
}
break;
case REDIS_OPT_COMPRESSION_LEVEL:
val_long = zval_get_long(val);
redis_sock->compression_level = val_long;
RETURN_TRUE;
case REDIS_OPT_PREFIX:
if (redis_sock->prefix) {
zend_string_release(redis_sock->prefix);
tests/RedisTest.php
+14 -2
View File
@@ -4485,14 +4485,26 @@ class Redis_Test extends TestSuite
if (!defined('Redis::COMPRESSION_LZF')) {
$this->markTestSkipped();
}
$this->checkCompression(Redis::COMPRESSION_LZF);
$this->checkCompression(Redis::COMPRESSION_LZF, 0);
}
private function checkCompression($mode)
public function testCompressionZSTD()
{
if (!defined('Redis::COMPRESSION_ZSTD')) {
$this->markTestSkipped();
}
$this->checkCompression(Redis::COMPRESSION_ZSTD, 0);
$this->checkCompression(Redis::COMPRESSION_ZSTD, 9);
}
private function checkCompression($mode, $level)
{
$this->assertTrue($this->redis->setOption(Redis::OPT_COMPRESSION, $mode) === TRUE); // set ok
$this->assertTrue($this->redis->getOption(Redis::OPT_COMPRESSION) === $mode); // get ok
$this->assertTrue($this->redis->setOption(Redis::OPT_COMPRESSION_LEVEL, $level) === TRUE);
$this->assertTrue($this->redis->getOption(Redis::OPT_COMPRESSION_LEVEL) === $level);
$val = 'xxxxxxxxxx';
$this->redis->set('key', $val);
$this->assertEquals($val, $this->redis->get('key'));