Experimental support for JSON serialization with simdjson

This commit adds conditional support for simdjson based JSON
deserialization.

To enable compile with `--enable-redis-simdjson` and PhpRedis will
attempt to find and link with the `simdjson` shared library. The feature
likely requires simdjson >= `4.0.0` and was tested with `4.3.1`.

Enabling is just like other serializers:

```php
$redis->setOption(Redis::OPT_SERIALIZER, Redis::SERIALIZER_SIMDJSON);
$redis->set('foo', ['bar' => 'baz']);
print_r($redis->get('foo'));
```

Initial tests do show aa huge performance improvement in deserializing
json payloads (2-3.5x depending on the payloads themselves).
This commit is contained in:
michael-grunder
2026-03-05 14:19:31 -08:00
parent d90cfdb6bd
commit 0a784e8f94
11 changed files with 513 additions and 37 deletions
+2 -1
View File
@@ -156,7 +156,8 @@ typedef enum {
REDIS_SERIALIZER_PHP,
REDIS_SERIALIZER_IGBINARY,
REDIS_SERIALIZER_MSGPACK,
REDIS_SERIALIZER_JSON
REDIS_SERIALIZER_JSON,
REDIS_SERIALIZER_SIMDJSON,
} redis_serializer;
/* compression */
#define REDIS_COMPRESSION_NONE 0
+90 -1
View File
@@ -11,6 +11,12 @@ PHP_ARG_ENABLE(redis-session, whether to enable sessions,
PHP_ARG_ENABLE(redis-json, whether to enable json serializer support,
[ --disable-redis-json Disable json serializer support], yes, no)
PHP_ARG_ENABLE(redis-simdjson, whether to enable simdjson json decode support,
[ --enable-redis-simdjson Enable simdjson-based JSON decoding], no, no)
PHP_ARG_WITH(simdjson, path to system simdjson,
[ --with-simdjson[=DIR] Use system simdjson (optional DIR prefix)], no, no)
PHP_ARG_ENABLE(redis-igbinary, whether to enable igbinary serializer support,
[ --enable-redis-igbinary Enable igbinary serializer support], no, no)
@@ -103,6 +109,63 @@ if test "$PHP_REDIS" != "no"; then
AC_MSG_RESULT([disabled])
fi
AC_MSG_CHECKING([for redis simdjson support])
if test "$PHP_REDIS_SIMDJSON" != "no"; then
AC_MSG_RESULT([enabled])
AC_DEFINE(HAVE_REDIS_SIMDJSON,1,
[Whether simdjson support is enabled])
PHP_REQUIRE_CXX()
PHP_CXX_COMPILE_STDCXX([17], [mandatory],
[PHP_REDIS_SIMDJSON_STDCXX])
AC_PATH_PROG([PKG_CONFIG], [pkg-config], [no])
SIMDJ_CFLAGS=""
SIMDJ_LIBS=""
if test -x "$PKG_CONFIG" && $PKG_CONFIG --exists simdjson; then
AC_MSG_CHECKING([for simdjson using pkg-config])
SIMDJ_CFLAGS=`$PKG_CONFIG simdjson --cflags`
SIMDJ_LIBS=`$PKG_CONFIG simdjson --libs`
AC_MSG_RESULT([yes])
PHP_EVAL_LIBLINE([$SIMDJ_LIBS], [REDIS_SHARED_LIBADD])
PHP_EVAL_INCLINE([$SIMDJ_CFLAGS])
else
AS_VAR_IF([PHP_SIMDJSON], [no], [
AC_MSG_CHECKING([for libsimdjson in default paths])
PHP_CHECK_LIBRARY([simdjson], [simdjson_version],
[
AC_MSG_RESULT([yes])
PHP_ADD_LIBRARY([simdjson], [1], [REDIS_SHARED_LIBADD])
],
[
AC_MSG_RESULT([no])
AC_MSG_ERROR([simdjson not found. Install it, or use \
pkg-config, or pass --with-simdjson=DIR])
])
], [
AC_MSG_CHECKING([for simdjson in $PHP_SIMDJSON])
PHP_ADD_INCLUDE([$PHP_SIMDJSON/include])
PHP_CHECK_LIBRARY([simdjson], [simdjson_version],
[
AC_MSG_RESULT([yes])
PHP_ADD_LIBRARY_WITH_PATH([simdjson],
[$PHP_SIMDJSON/$PHP_LIBDIR],
[REDIS_SHARED_LIBADD])
],
[
AC_MSG_RESULT([no])
AC_MSG_ERROR([could not find libsimdjson in $PHP_SIMDJSON])
],
[-L$PHP_SIMDJSON/$PHP_LIBDIR])
])
fi
else
AC_MSG_RESULT([disabled])
fi
if test "$PHP_REDIS_IGBINARY" != "no"; then
AC_MSG_CHECKING([for igbinary includes])
igbinary_inc_path=""
@@ -325,5 +388,31 @@ if test "$PHP_REDIS" != "no"; then
fi
PHP_SUBST(REDIS_SHARED_LIBADD)
PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c backoff.c $lzf_sources, $ext_shared)
REDIS_SOURCES="redis.c redis_commands.c library.c redis_session.c \
redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c \
redis_sentinel.c sentinel_library.c backoff.c $lzf_sources"
if test "$PHP_REDIS_SIMDJSON" != "no"; then
PHP_NEW_EXTENSION(redis, [$REDIS_SOURCES], [$ext_shared],,, [cxx])
REDIS_SIMDJSON_CXX_SOURCES="redis_simdjson.cc"
AS_VAR_IF([ZEND_DEBUG], [yes], [
REDIS_SIMDJSON_CXX_FLAGS="$PHP_REDIS_SIMDJSON_STDCXX -O2 -g"
], [
REDIS_SIMDJSON_CXX_FLAGS="$PHP_REDIS_SIMDJSON_STDCXX -O2"
])
AS_VAR_IF([ext_shared], [no],
[PHP_ADD_SOURCES([$ext_dir],
[$REDIS_SIMDJSON_CXX_SOURCES],
[$REDIS_SIMDJSON_CXX_FLAGS])],
[PHP_ADD_SOURCES_X([$ext_dir],
[$REDIS_SIMDJSON_CXX_SOURCES],
[$REDIS_SIMDJSON_CXX_FLAGS],
[shared_objects_redis],
[yes])])
else
PHP_NEW_EXTENSION(redis, [$REDIS_SOURCES], [$ext_shared])
fi
fi
+14 -2
View File
@@ -8,6 +8,10 @@
#include "php_network.h"
#include <sys/types.h>
#ifdef HAVE_REDIS_SIMDJSON
#include "redis_simdjson.h"
#endif
#ifdef HAVE_REDIS_IGBINARY
#include "igbinary/igbinary.h"
#endif
@@ -4356,7 +4360,8 @@ redis_serialize(RedisSock *redis_sock, zval *z, char **val, size_t *val_len)
}
#endif
break;
case REDIS_SERIALIZER_JSON:
case REDIS_SERIALIZER_JSON: /* fallthrough */
case REDIS_SERIALIZER_SIMDJSON:
#ifdef HAVE_REDIS_JSON
php_json_encode(&sstr, z, PHP_JSON_OBJECT_AS_ARRAY);
*val = estrndup(ZSTR_VAL(sstr.s), ZSTR_LEN(sstr.s));
@@ -4429,7 +4434,14 @@ redis_unserialize(RedisSock* redis_sock, const char *val, int val_len,
break;
case REDIS_SERIALIZER_JSON:
#ifdef HAVE_REDIS_JSON
ret = !php_json_decode(z_ret, (char *)val, val_len, 1, PHP_JSON_PARSER_DEFAULT_DEPTH);
ret = !php_json_decode(z_ret, (char *)val, val_len, 1,
PHP_JSON_PARSER_DEFAULT_DEPTH);
#endif
break;
#ifdef HAVE_REDIS_SIMDJSON
case REDIS_SERIALIZER_SIMDJSON:
ret = !redis_json_to_zval_ex((void*)z_ret, val, val_len, 0,
PHP_JSON_PARSER_DEFAULT_DEPTH);
#endif
break;
EMPTY_SWITCH_DEFAULT_CASE()
+32 -30
View File
@@ -397,38 +397,40 @@ PHP_MINIT_FUNCTION(redis)
return SUCCESS;
}
static const char *
get_available_serializers(void)
{
static const char *get_available_serializers(void) {
smart_string aux = {0};
static char buf[256];
#define append_serializer(name) \
smart_string_appendl(&aux, ", " name, sizeof(", " name) - 1)
if (EXPECTED(*buf))
goto exit;
smart_string_appendl(&aux, "php", sizeof("php") - 1);
#ifdef HAVE_REDIS_JSON
#ifdef HAVE_REDIS_IGBINARY
#ifdef HAVE_REDIS_MSGPACK
return "php, json, igbinary, msgpack";
#else
return "php, json, igbinary";
#endif
#else
#ifdef HAVE_REDIS_MSGPACK
return "php, json, msgpack";
#else
return "php, json";
#endif
#endif
#else
#ifdef HAVE_REDIS_IGBINARY
#ifdef HAVE_REDIS_MSGPACK
return "php, igbinary, msgpack";
#else
return "php, igbinary";
#endif
#else
#ifdef HAVE_REDIS_MSGPACK
return "php, msgpack";
#else
return "php";
#endif
#endif
append_serializer("json");
#endif
#ifdef HAVE_REDIS_SIMDJSON
append_serializer("simdjson");
#endif
#ifdef HAVE_REDIS_IGBINARY
append_serializer("igbinary");
#endif
#ifdef HAVE_REDIS_MSGPACK
append_serializer("msgpack");
#endif
/* Will probably never happen but check anyway */
ZEND_ASSERT(aux.len < sizeof(buf) - 1);
memcpy(buf, aux.c, aux.len);
buf[aux.len] = '\0';
smart_string_free(&aux);
exit:
return buf;
}
/**
+11
View File
@@ -260,6 +260,17 @@ class Redis {
*/
public const SERIALIZER_JSON = UNKNOWN;
#ifdef HAVE_REDIS_SIMDJSON
/**
* Sets the serializer to JSON and deserializes with the SIMDJSON library.
*
* @var int
* @cvalue REDIS_SERIALIZER_SIMDJSON
*
*/
public const SERIALIZER_SIMDJSON = UNKNOWN;
#endif
/**
* Disables compression.
*
+9 -1
View File
@@ -1,5 +1,5 @@
/* This is a generated file, edit the .stub.php file instead.
* Stub hash: adcbf21ebb463f2911a1565705262bbe88390ac3 */
* Stub hash: a84f7b02cf70c2018eb55806af19ecb46f108b06 */
ZEND_BEGIN_ARG_INFO_EX(arginfo_class_Redis___construct, 0, 0, 0)
ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, options, IS_ARRAY, 1, "null")
@@ -2081,6 +2081,14 @@ static zend_class_entry *register_class_Redis(void)
zend_string *const_SERIALIZER_JSON_name = zend_string_init_interned("SERIALIZER_JSON", sizeof("SERIALIZER_JSON") - 1, 1);
zend_declare_class_constant_ex(class_entry, const_SERIALIZER_JSON_name, &const_SERIALIZER_JSON_value, ZEND_ACC_PUBLIC, NULL);
zend_string_release(const_SERIALIZER_JSON_name);
#if defined(HAVE_REDIS_SIMDJSON)
zval const_SERIALIZER_SIMDJSON_value;
ZVAL_LONG(&const_SERIALIZER_SIMDJSON_value, REDIS_SERIALIZER_SIMDJSON);
zend_string *const_SERIALIZER_SIMDJSON_name = zend_string_init_interned("SERIALIZER_SIMDJSON", sizeof("SERIALIZER_SIMDJSON") - 1, 1);
zend_declare_class_constant_ex(class_entry, const_SERIALIZER_SIMDJSON_name, &const_SERIALIZER_SIMDJSON_value, ZEND_ACC_PUBLIC, NULL);
zend_string_release(const_SERIALIZER_SIMDJSON_name);
#endif
zval const_COMPRESSION_NONE_value;
ZVAL_LONG(&const_COMPRESSION_NONE_value, REDIS_COMPRESSION_NONE);
+3
View File
@@ -7575,6 +7575,9 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS,
if (val_long == REDIS_SERIALIZER_NONE
|| val_long == REDIS_SERIALIZER_PHP
|| val_long == REDIS_SERIALIZER_JSON
#ifdef HAVE_REDIS_SIMDJSON
|| val_long == REDIS_SERIALIZER_SIMDJSON
#endif
#ifdef HAVE_REDIS_IGBINARY
|| val_long == REDIS_SERIALIZER_IGBINARY
#endif
+9 -1
View File
@@ -1,5 +1,5 @@
/* This is a generated file, edit the .stub.php file instead.
* Stub hash: adcbf21ebb463f2911a1565705262bbe88390ac3 */
* Stub hash: a84f7b02cf70c2018eb55806af19ecb46f108b06 */
ZEND_BEGIN_ARG_INFO_EX(arginfo_class_Redis___construct, 0, 0, 0)
ZEND_ARG_INFO(0, options)
@@ -1910,6 +1910,14 @@ static zend_class_entry *register_class_Redis(void)
zend_string *const_SERIALIZER_JSON_name = zend_string_init_interned("SERIALIZER_JSON", sizeof("SERIALIZER_JSON") - 1, 1);
zend_declare_class_constant_ex(class_entry, const_SERIALIZER_JSON_name, &const_SERIALIZER_JSON_value, ZEND_ACC_PUBLIC, NULL);
zend_string_release(const_SERIALIZER_JSON_name);
#if defined(HAVE_REDIS_SIMDJSON)
zval const_SERIALIZER_SIMDJSON_value;
ZVAL_LONG(&const_SERIALIZER_SIMDJSON_value, REDIS_SERIALIZER_SIMDJSON);
zend_string *const_SERIALIZER_SIMDJSON_name = zend_string_init_interned("SERIALIZER_SIMDJSON", sizeof("SERIALIZER_SIMDJSON") - 1, 1);
zend_declare_class_constant_ex(class_entry, const_SERIALIZER_SIMDJSON_name, &const_SERIALIZER_SIMDJSON_value, ZEND_ACC_PUBLIC, NULL);
zend_string_release(const_SERIALIZER_SIMDJSON_name);
#endif
zval const_COMPRESSION_NONE_value;
ZVAL_LONG(&const_COMPRESSION_NONE_value, REDIS_COMPRESSION_NONE);
+296
View File
@@ -0,0 +1,296 @@
extern "C" {
#include "php.h"
}
#include "redis_simdjson.h"
#include <simdjson.h>
#include <string>
#include <cstring>
#include <limits>
struct redis_zval {};
static thread_local simdjson::ondemand::parser tl_parser;
static thread_local std::string tl_scratch;
static inline const char *pad_json(const char *json, size_t len) {
const size_t need = len + simdjson::SIMDJSON_PADDING;
if (tl_scratch.size() < need) {
tl_scratch.resize(need);
}
memcpy(tl_scratch.data(), json, len);
memset(tl_scratch.data() + len, 0, simdjson::SIMDJSON_PADDING);
return tl_scratch.data();
}
static
simdjson::error_code value_to_zval(zval *dst, simdjson::ondemand::value v,
uint32_t flags, uint32_t depth);
static simdjson::error_code
array_to_zval(zval *dst, simdjson::ondemand::array a, uint32_t flags,
uint32_t depth)
{
if (depth == 0) {
return simdjson::DEPTH_ERROR;
}
array_init(dst);
for (auto elem : a) {
simdjson::error_code err;
zval zv;
ZVAL_UNDEF(&zv);
err = value_to_zval(&zv, elem.value(), flags, depth - 1);
if (err) {
zval_ptr_dtor(dst);
ZVAL_UNDEF(dst);
return err;
}
add_next_index_zval(dst, &zv);
}
return simdjson::SUCCESS;
}
static simdjson::error_code
object_to_zval(zval *dst, simdjson::ondemand::object o, uint32_t flags,
uint32_t depth)
{
if (depth == 0) {
return simdjson::DEPTH_ERROR;
}
const bool assoc = !(flags & RJ_OBJECT);
if (assoc) {
array_init(dst);
} else {
object_init(dst);
}
for (auto field : o) {
simdjson::error_code err;
std::string_view ksv;
zval zv;
ZVAL_UNDEF(&zv);
err = field.unescaped_key().get(ksv);
if (err)
return err;
err = value_to_zval(&zv, field.value(), flags, depth - 1);
if (err) {
zval_ptr_dtor(dst);
ZVAL_UNDEF(dst);
return err;
}
if (assoc) {
add_assoc_zval_ex(dst, ksv.data(), (uint32_t)ksv.size(), &zv);
} else {
zend_string *zs = zend_string_init(ksv.data(), ksv.size(), 0);
zend_update_property(Z_OBJCE_P(dst), Z_OBJ_P(dst),
ZSTR_VAL(zs), ZSTR_LEN(zs), &zv);
zend_string_release(zs);
zval_ptr_dtor(&zv);
}
}
return simdjson::SUCCESS;
}
static simdjson::error_code
number_to_zval(zval *dst, simdjson::ondemand::number n, uint32_t flags)
{
if (n.is_int64()) {
int64_t i = n.get_int64();
if (sizeof(zend_long) == 8) {
ZVAL_LONG(dst, i);
return simdjson::SUCCESS;
}
if (i >= (int64_t)std::numeric_limits<zend_long>::min() &&
i <= (int64_t)std::numeric_limits<zend_long>::max())
{
ZVAL_LONG(dst, i);
return simdjson::SUCCESS;
}
if (flags & RJ_BIGINT_AS_STRING) {
char buf[32];
int nbytes = snprintf(buf, sizeof(buf), "%lld", (long long)i);
ZVAL_STRINGL(dst, buf, nbytes);
return simdjson::SUCCESS;
}
ZVAL_DOUBLE(dst, (double)i);
return simdjson::SUCCESS;
}
if (n.is_uint64()) {
uint64_t u = n.get_uint64();
if (sizeof(zend_long) == 8 &&
u <= (uint64_t)std::numeric_limits<zend_long>::max()) {
ZVAL_LONG(dst, (zend_long)u);
return simdjson::SUCCESS;
}
if (flags & RJ_BIGINT_AS_STRING) {
char buf[32];
int nbytes = snprintf(buf, sizeof(buf), "%llu",
(unsigned long long)u);
ZVAL_STRINGL(dst, buf, (size_t)nbytes);
return simdjson::SUCCESS;
}
ZVAL_DOUBLE(dst, (double)u);
return simdjson::SUCCESS;
}
double d = n.get_double();
ZVAL_DOUBLE(dst, d);
return simdjson::SUCCESS;
}
template <typename T>
static simdjson::error_code
any_to_zval(zval *dst, T &src, uint32_t flags, uint32_t depth)
{
using simdjson::ondemand::json_type;
simdjson::error_code err;
json_type t;
err = src.type().get(t);
if (err)
return err;
switch (t) {
case json_type::null:
ZVAL_NULL(dst);
return simdjson::SUCCESS;
case json_type::boolean: {
bool b;
err = src.get_bool().get(b);
if (err)
return err;
ZVAL_BOOL(dst, b);
return simdjson::SUCCESS;
}
case json_type::number: {
simdjson::ondemand::number n;
err = src.get_number().get(n);
if (err)
return err;
return number_to_zval(dst, n, flags);
}
case json_type::string: {
std::string_view sv;
err = src.get_string().get(sv);
if (err)
return err;
ZVAL_STRINGL(dst, sv.data(), sv.size());
return simdjson::SUCCESS;
}
case json_type::array: {
simdjson::ondemand::array a;
err = src.get_array().get(a);
if (err)
return err;
return array_to_zval(dst, a, flags, depth);
}
case json_type::object: {
simdjson::ondemand::object o;
err = src.get_object().get(o);
if (err)
return err;
return object_to_zval(dst, o, flags, depth);
}
case json_type::unknown:
return simdjson::INCORRECT_TYPE;
}
ZEND_UNREACHABLE();
}
static simdjson::error_code
document_to_zval(zval *dst, simdjson::ondemand::document &doc,
uint32_t flags, uint32_t depth)
{
return any_to_zval(dst, doc, flags, depth);
}
static simdjson::error_code
value_to_zval(zval *dst, simdjson::ondemand::value v, uint32_t flags,
uint32_t depth)
{
return any_to_zval(dst, v, flags, depth);
}
static void redis_json_emit_error(int err) {
if (err == 0)
return;
if (err < 0)
err = -err;
if (err < 0 || err > simdjson::NUM_ERROR_CODES) {
php_error_docref(NULL, E_WARNING, "Unknown error code: %d", err);
return;
}
simdjson::error_code code = static_cast<simdjson::error_code>(err);
php_error_docref(NULL, E_WARNING, "Error parsing JSON: %s",
simdjson::error_message(code));
}
extern "C" int
redis_json_to_zval_ex(redis_zval *dst_, const char *json, size_t len,
uint32_t flags, uint32_t max_depth)
{
auto *dst = reinterpret_cast<zval *>(dst_);
simdjson::error_code err = simdjson::SUCCESS;
if (max_depth == 0) {
max_depth = 512;
}
const char *padded = pad_json(json, len);
auto doc_res = tl_parser.iterate(padded, len, tl_scratch.size());
err = doc_res.error();
if (err) {
ZVAL_NULL(dst);
redis_json_emit_error(err);
return -err;
}
simdjson::ondemand::document doc = std::move(doc_res).value_unsafe();
err = document_to_zval(dst, doc, flags, max_depth);
if (err) {
ZVAL_NULL(dst);
redis_json_emit_error(err);
return -err;
}
return 0;
}
+25
View File
@@ -0,0 +1,25 @@
#ifndef REDIS_SIMDJSON_H
#define REDIS_SIMDJSON_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct redis_zval redis_zval;
enum redisJsonFlags : uint32_t {
RJ_OBJECT = 1u << 0,
RJ_BIGINT_AS_STRING = 1u << 1,
};
int redis_json_to_zval_ex(redis_zval *dst_, const char *json, size_t len,
uint32_t flags, uint32_t max_depth);
#ifdef __cplusplus
}
#endif
#endif /* REDIS_SIMDJSON_H */
+22 -1
View File
@@ -34,6 +34,8 @@ class Redis_Test extends TestSuite {
$result[] = Redis::SERIALIZER_IGBINARY;
if (defined('Redis::SERIALIZER_JSON'))
$result[] = Redis::SERIALIZER_JSON;
if (defined('Redis::SERIALIZER_SIMDJSON'))
$result[] = Redis::SERIALIZER_SIMDJSON;
if (defined('Redis::SERIALIZER_MSGPACK'))
$result[] = Redis::SERIALIZER_MSGPACK;
@@ -5332,11 +5334,30 @@ class Redis_Test extends TestSuite {
$this->redis->setOption(Redis::OPT_PREFIX, '');
}
public function testSerializerSimdJSON() {
if ( ! defined('Redis::SERIALIZER_SIMDJSON'))
$this->markTestSkipped();
$this->checkSerializer(Redis::SERIALIZER_SIMDJSON);
$this->redis->setOption(Redis::OPT_PREFIX, 'test:');
$this->checkSerializer(Redis::SERIALIZER_SIMDJSON);
$this->redis->setOption(Redis::OPT_PREFIX, '');
}
private function isJsonSerializer($mode) {
return $mode == Redis::SERIALIZER_JSON ||
(defined('Redis::SERIALIZER_SIMDJSON') &&
$mode == Redis::SERIALIZER_SIMDJSON);
}
private function checkSerializer($mode) {
$this->redis->del('key');
$this->assertEquals(Redis::SERIALIZER_NONE, $this->redis->getOption(Redis::OPT_SERIALIZER)); // default
$this->assertTrue($this->redis->setOption(Redis::OPT_SERIALIZER, $mode)); // set ok
$this->assertEquals($mode, $this->redis->getOption(Redis::OPT_SERIALIZER)); // get ok
// lPush, rPush
@@ -5537,7 +5558,7 @@ class Redis_Test extends TestSuite {
$this->redis->set('x', [new stdClass, new stdClass]);
$x = $this->redis->get('x');
$this->assertIsArray($x);
if ($mode === Redis::SERIALIZER_JSON) {
if ($this->isJsonSerializer($mode)) {
$this->assertIsArray($x[0]);
$this->assertIsArray($x[1]);
} else {