mercurial: comparison hgext/fsmonitor/pywatchman/bser.c

equal deleted inserted replaced

-:f35397fe0c04
+:16f4b341288d
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 #include <Python.h>
+#include <bytesobject.h>
 #ifdef _MSC_VER
 #define inline __inline
-#include "msc_stdint.h"
+#if _MSC_VER >= 1800
+#include <stdint.h>
+#else
+// The compiler associated with Python 2.7 on Windows doesn't ship
+// with stdint.h, so define the small subset that we use here.
+typedef __int8 int8_t;
+typedef __int16 int16_t;
+typedef __int32 int32_t;
+typedef __int64 int64_t;
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int16 uint16_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+#define UINT32_MAX 4294967295U
 #endif
+#endif
+// clang-format off
 /* Return the smallest size int that can store the value */
 #define INT_SIZE(x) (((x) == ((int8_t)x))  ? 1 :    \
 ((x) == ((int16_t)x)) ? 2 :    \
 ((x) == ((int32_t)x)) ? 4 : 8)
 #define BSER_ARRAY     0x00
 #define BSER_OBJECT    0x01
-#define BSER_STRING    0x02
+#define BSER_BYTESTRING 0x02
 #define BSER_INT8      0x03
 #define BSER_INT16     0x04
 #define BSER_INT32     0x05
 #define BSER_INT64     0x06
 #define BSER_REAL      0x07
 #define BSER_TRUE      0x08
 #define BSER_FALSE     0x09
 #define BSER_NULL      0x0a
 #define BSER_TEMPLATE  0x0b
 #define BSER_SKIP      0x0c
+#define BSER_UTF8STRING 0x0d
+// clang-format on
 // An immutable object representation of BSER_OBJECT.
 // Rather than build a hash table, key -> value are obtained
 // by walking the list of keys to determine the offset into
 // the values array.  The assumption is that the number of
 // so that the time overhead for this is small compared to
 // using a proper hash table.  Even with this simplistic
 // approach, this is still faster for the mercurial use case
 // as it helps to eliminate creating N other objects to
 // represent the stat information in the hgwatchman extension
+// clang-format off
 typedef struct {
 PyObject_HEAD
 PyObject *keys;   // tuple of field names
 PyObject *values; // tuple of values
 } bserObject;
+// clang-format on
-static Py_ssize_t bserobj_tuple_length(PyObject *o) {
-bserObject *obj = (bserObject*)o;
+static Py_ssize_t bserobj_tuple_length(PyObject* o) {
+bserObject* obj = (bserObject*)o;
 return PySequence_Length(obj->keys);
 }
-static PyObject *bserobj_tuple_item(PyObject *o, Py_ssize_t i) {
+static PyObject* bserobj_tuple_item(PyObject* o, Py_ssize_t i) {
-bserObject *obj = (bserObject*)o;
+bserObject* obj = (bserObject*)o;
 return PySequence_GetItem(obj->values, i);
 }
+// clang-format off
 static PySequenceMethods bserobj_sq = {
 bserobj_tuple_length,      /* sq_length */
 0,                         /* sq_concat */
 0,                         /* sq_repeat */
 bserobj_tuple_item,        /* sq_item */
 0,                         /* sq_ass_item */
 0,                         /* sq_contains */
 0,                         /* sq_inplace_concat */
 0                          /* sq_inplace_repeat */
 };
+// clang-format on
-static void bserobj_dealloc(PyObject *o) {
-bserObject *obj = (bserObject*)o;
+static void bserobj_dealloc(PyObject* o) {
+bserObject* obj = (bserObject*)o;
 Py_CLEAR(obj->keys);
 Py_CLEAR(obj->values);
 PyObject_Del(o);
 }
-static PyObject *bserobj_getattrro(PyObject *o, PyObject *name) {
+static PyObject* bserobj_getattrro(PyObject* o, PyObject* name) {
-bserObject *obj = (bserObject*)o;
+bserObject* obj = (bserObject*)o;
 Py_ssize_t i, n;
-const char *namestr;
+PyObject* name_bytes = NULL;
+PyObject* ret = NULL;
+const char* namestr;
 if (PyIndex_Check(name)) {
 i = PyNumber_AsSsize_t(name, PyExc_IndexError);
 if (i == -1 && PyErr_Occurred()) {
-return NULL;
+goto bail;
 }
-return PySequence_GetItem(obj->values, i);
+ret = PySequence_GetItem(obj->values, i);
-}
+goto bail;
+}
+// We can be passed in Unicode objects here -- we don't support anything other
+// than UTF-8 for keys.
+if (PyUnicode_Check(name)) {
+name_bytes = PyUnicode_AsUTF8String(name);
+if (name_bytes == NULL) {
+goto bail;
+}
+namestr = PyBytes_AsString(name_bytes);
+} else {
+namestr = PyBytes_AsString(name);
+}
+if (namestr == NULL) {
+goto bail;
+}
 // hack^Wfeature to allow mercurial to use "st_size" to reference "size"
-namestr = PyString_AsString(name);
 if (!strncmp(namestr, "st_", 3)) {
 namestr += 3;
 }
 n = PyTuple_GET_SIZE(obj->keys);
 for (i = 0; i < n; i++) {
-const char *item_name = NULL;
+const char* item_name = NULL;
-PyObject *key = PyTuple_GET_ITEM(obj->keys, i);
+PyObject* key = PyTuple_GET_ITEM(obj->keys, i);
-item_name = PyString_AsString(key);
+item_name = PyBytes_AsString(key);
 if (!strcmp(item_name, namestr)) {
-return PySequence_GetItem(obj->values, i);
+ret = PySequence_GetItem(obj->values, i);
-}
+goto bail;
 }
-PyErr_Format(PyExc_AttributeError,
+}
-"bserobject has no attribute '%.400s'", namestr);
-return NULL;
+PyErr_Format(
-}
+PyExc_AttributeError, "bserobject has no attribute '%.400s'", namestr);
+bail:
+Py_XDECREF(name_bytes);
+return ret;
+}
+// clang-format off
 static PyMappingMethods bserobj_map = {
 bserobj_tuple_length,     /* mp_length */
 bserobj_getattrro,        /* mp_subscript */
 0                         /* mp_ass_subscript */
 };
 0,                         /* tp_dictoffset */
 0,                         /* tp_init */
 0,                         /* tp_alloc */
 0,                         /* tp_new */
 };
+// clang-format on
-static PyObject *bser_loads_recursive(const char **ptr, const char *end,
+typedef struct loads_ctx {
-int mutable);
+int mutable;
+const char* value_encoding;
+const char* value_errors;
+uint32_t bser_version;
+uint32_t bser_capabilities;
+} unser_ctx_t;
+static PyObject*
+bser_loads_recursive(const char** ptr, const char* end, const unser_ctx_t* ctx);
 static const char bser_true = BSER_TRUE;
 static const char bser_false = BSER_FALSE;
 static const char bser_null = BSER_NULL;
-static const char bser_string_hdr = BSER_STRING;
+static const char bser_bytestring_hdr = BSER_BYTESTRING;
 static const char bser_array_hdr = BSER_ARRAY;
 static const char bser_object_hdr = BSER_OBJECT;
-static inline uint32_t next_power_2(uint32_t n)
+static inline uint32_t next_power_2(uint32_t n) {
-{
 n |= (n >> 16);
 n |= (n >> 8);
 n |= (n >> 4);
 n |= (n >> 2);
 n |= (n >> 1);
 return n + 1;
 }
 // A buffer we use for building up the serialized result
 struct bser_buffer {
-char *buf;
+char* buf;
 int wpos, allocd;
+uint32_t bser_version;
+uint32_t capabilities;
 };
 typedef struct bser_buffer bser_t;
-static int bser_append(bser_t *bser, const char *data, uint32_t len)
+static int bser_append(bser_t* bser, const char* data, uint32_t len) {
-{
 int newlen = next_power_2(bser->wpos + len);
 if (newlen > bser->allocd) {
-char *nbuf = realloc(bser->buf, newlen);
+char* nbuf = realloc(bser->buf, newlen);
 if (!nbuf) {
 return 0;
 }
 bser->buf = nbuf;
 memcpy(bser->buf + bser->wpos, data, len);
 bser->wpos += len;
 return 1;
 }
-static int bser_init(bser_t *bser)
+static int bser_init(bser_t* bser, uint32_t version, uint32_t capabilities) {
-{
 bser->allocd = 8192;
 bser->wpos = 0;
 bser->buf = malloc(bser->allocd);
+bser->bser_version = version;
+bser->capabilities = capabilities;
 if (!bser->buf) {
 return 0;
 }
 // Leave room for the serialization header, which includes
 // our overall length.  To make things simpler, we'll use an
 // int32 for the header
 #define EMPTY_HEADER "\x00\x01\x05\x00\x00\x00\x00"
-bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER)-1);
+// Version 2 also carries an integer indicating the capabilities. The
+// capabilities integer comes before the PDU size.
+#define EMPTY_HEADER_V2 "\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00"
+if (version == 2) {
+bser_append(bser, EMPTY_HEADER_V2, sizeof(EMPTY_HEADER_V2) - 1);
+} else {
+bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER) - 1);
+}
 return 1;
 }
-static void bser_dtor(bser_t *bser)
+static void bser_dtor(bser_t* bser) {
-{
 free(bser->buf);
 bser->buf = NULL;
 }
-static int bser_long(bser_t *bser, int64_t val)
+static int bser_long(bser_t* bser, int64_t val) {
-{
 int8_t i8;
 int16_t i16;
 int32_t i32;
 int64_t i64;
 char sz;
 int size = INT_SIZE(val);
-char *iptr;
+char* iptr;
 switch (size) {
 case 1:
 sz = BSER_INT8;
 i8 = (int8_t)val;
 sz = BSER_INT64;
 i64 = (int64_t)val;
 iptr = (char*)&i64;
 break;
 default:
-PyErr_SetString(PyExc_RuntimeError,
+PyErr_SetString(PyExc_RuntimeError, "Cannot represent this long value!?");
-"Cannot represent this long value!?");
 return 0;
 }
 if (!bser_append(bser, &sz, sizeof(sz))) {
 return 0;
 }
 return bser_append(bser, iptr, size);
 }
-static int bser_string(bser_t *bser, PyObject *sval)
+static int bser_bytestring(bser_t* bser, PyObject* sval) {
-{
+char* buf = NULL;
-char *buf = NULL;
 Py_ssize_t len;
 int res;
-PyObject *utf = NULL;
+PyObject* utf = NULL;
 if (PyUnicode_Check(sval)) {
 utf = PyUnicode_AsEncodedString(sval, "utf-8", "ignore");
 sval = utf;
 }
-res = PyString_AsStringAndSize(sval, &buf, &len);
+res = PyBytes_AsStringAndSize(sval, &buf, &len);
 if (res == -1) {
 res = 0;
 goto out;
 }
-if (!bser_append(bser, &bser_string_hdr, sizeof(bser_string_hdr))) {
+if (!bser_append(bser, &bser_bytestring_hdr, sizeof(bser_bytestring_hdr))) {
 res = 0;
 goto out;
 }
 if (!bser_long(bser, len)) {
 }
 return res;
 }
-static int bser_recursive(bser_t *bser, PyObject *val)
+static int bser_recursive(bser_t* bser, PyObject* val) {
-{
 if (PyBool_Check(val)) {
 if (val == Py_True) {
 return bser_append(bser, &bser_true, sizeof(bser_true));
 }
 return bser_append(bser, &bser_false, sizeof(bser_false));
 if (val == Py_None) {
 return bser_append(bser, &bser_null, sizeof(bser_null));
 }
+// Python 3 has one integer type.
+#if PY_MAJOR_VERSION < 3
 if (PyInt_Check(val)) {
 return bser_long(bser, PyInt_AS_LONG(val));
 }
+#endif // PY_MAJOR_VERSION < 3
 if (PyLong_Check(val)) {
 return bser_long(bser, PyLong_AsLongLong(val));
 }
-if (PyString_Check(val) || PyUnicode_Check(val)) {
+if (PyBytes_Check(val) || PyUnicode_Check(val)) {
-return bser_string(bser, val);
+return bser_bytestring(bser, val);
 }
 if (PyFloat_Check(val)) {
 double dval = PyFloat_AS_DOUBLE(val);
 char sz = BSER_REAL;
 if (!bser_long(bser, len)) {
 return 0;
 }
 for (i = 0; i < len; i++) {
-PyObject *ele = PyList_GET_ITEM(val, i);
+PyObject* ele = PyList_GET_ITEM(val, i);
 if (!bser_recursive(bser, ele)) {
 return 0;
 }
 }
 if (!bser_long(bser, len)) {
 return 0;
 }
 for (i = 0; i < len; i++) {
-PyObject *ele = PyTuple_GET_ITEM(val, i);
+PyObject* ele = PyTuple_GET_ITEM(val, i);
 if (!bser_recursive(bser, ele)) {
 return 0;
 }
 }
 if (!bser_long(bser, len)) {
 return 0;
 }
 while (PyDict_Next(val, &pos, &key, &ele)) {
-if (!bser_string(bser, key)) {
+if (!bser_bytestring(bser, key)) {
 return 0;
 }
 if (!bser_recursive(bser, ele)) {
 return 0;
 }
 PyErr_SetString(PyExc_ValueError, "Unsupported value type");
 return 0;
 }
-static PyObject *bser_dumps(PyObject *self, PyObject *args)
+static PyObject* bser_dumps(PyObject* self, PyObject* args, PyObject* kw) {
-{
 PyObject *val = NULL, *res;
 bser_t bser;
-uint32_t len;
+uint32_t len, bser_version = 1, bser_capabilities = 0;
-if (!PyArg_ParseTuple(args, "O", &val)) {
+static char* kw_list[] = {"val", "version", "capabilities", NULL};
-return NULL;
-}
+if (!PyArg_ParseTupleAndKeywords(
+args,
-if (!bser_init(&bser)) {
+kw,
+"O|ii:dumps",
+kw_list,
+&val,
+&bser_version,
+&bser_capabilities)) {
+return NULL;
+}
+if (!bser_init(&bser, bser_version, bser_capabilities)) {
 return PyErr_NoMemory();
 }
 if (!bser_recursive(&bser, val)) {
 bser_dtor(&bser);
 // otherwise, we've already set the error to something reasonable
 return NULL;
 }
 // Now fill in the overall length
-len = bser.wpos - (sizeof(EMPTY_HEADER) - 1);
+if (bser_version == 1) {
-memcpy(bser.buf + 3, &len, sizeof(len));
+len = bser.wpos - (sizeof(EMPTY_HEADER) - 1);
+memcpy(bser.buf + 3, &len, sizeof(len));
-res = PyString_FromStringAndSize(bser.buf, bser.wpos);
+} else {
+len = bser.wpos - (sizeof(EMPTY_HEADER_V2) - 1);
+// The BSER capabilities block comes before the PDU length
+memcpy(bser.buf + 2, &bser_capabilities, sizeof(bser_capabilities));
+memcpy(bser.buf + 7, &len, sizeof(len));
+}
+res = PyBytes_FromStringAndSize(bser.buf, bser.wpos);
 bser_dtor(&bser);
 return res;
 }
-int bunser_int(const char **ptr, const char *end, int64_t *val)
+int bunser_int(const char** ptr, const char* end, int64_t* val) {
-{
 int needed;
-const char *buf = *ptr;
+const char* buf = *ptr;
 int8_t i8;
 int16_t i16;
 int32_t i32;
 int64_t i64;
 break;
 case BSER_INT64:
 needed = 9;
 break;
 default:
-PyErr_Format(PyExc_ValueError,
+PyErr_Format(
-"invalid bser int encoding 0x%02x", buf[0]);
+PyExc_ValueError, "invalid bser int encoding 0x%02x", buf[0]);
 return 0;
 }
 if (end - buf < needed) {
 PyErr_SetString(PyExc_ValueError, "input buffer to small for int encoding");
 return 0;
 default:
 return 0;
 }
 }
-static int bunser_string(const char **ptr, const char *end,
+static int bunser_bytestring(
-const char **start, int64_t *len)
+const char** ptr,
-{
+const char* end,
-const char *buf = *ptr;
+const char** start,
+int64_t* len) {
+const char* buf = *ptr;
 // skip string marker
 buf++;
 if (!bunser_int(&buf, end, len)) {
 return 0;
 *ptr = buf + *len;
 *start = buf;
 return 1;
 }
-static PyObject *bunser_array(const char **ptr, const char *end, int mutable)
+static PyObject*
-{
+bunser_array(const char** ptr, const char* end, const unser_ctx_t* ctx) {
-const char *buf = *ptr;
+const char* buf = *ptr;
 int64_t nitems, i;
-PyObject *res;
+int mutable = ctx->mutable;
+PyObject* res;
 // skip array header
 buf++;
 if (!bunser_int(&buf, end, &nitems)) {
 return 0;
 } else {
 res = PyTuple_New((Py_ssize_t)nitems);
 }
 for (i = 0; i < nitems; i++) {
-PyObject *ele = bser_loads_recursive(ptr, end, mutable);
+PyObject* ele = bser_loads_recursive(ptr, end, ctx);
 if (!ele) {
 Py_DECREF(res);
 return NULL;
 }
 }
 return res;
 }
-static PyObject *bunser_object(const char **ptr, const char *end,
+static PyObject*
-int mutable)
+bunser_object(const char** ptr, const char* end, const unser_ctx_t* ctx) {
-{
+const char* buf = *ptr;
-const char *buf = *ptr;
 int64_t nitems, i;
-PyObject *res;
+int mutable = ctx->mutable;
-bserObject *obj;
+PyObject* res;
+bserObject* obj;
 // skip array header
 buf++;
 if (!bunser_int(&buf, end, &nitems)) {
 return 0;
 obj->values = PyTuple_New((Py_ssize_t)nitems);
 res = (PyObject*)obj;
 }
 for (i = 0; i < nitems; i++) {
-const char *keystr;
+const char* keystr;
 int64_t keylen;
-PyObject *key;
+PyObject* key;
-PyObject *ele;
+PyObject* ele;
-if (!bunser_string(ptr, end, &keystr, &keylen)) {
+if (!bunser_bytestring(ptr, end, &keystr, &keylen)) {
 Py_DECREF(res);
 return NULL;
 }
 if (keylen > LONG_MAX) {
 PyErr_Format(PyExc_ValueError, "string too big for python");
 Py_DECREF(res);
 return NULL;
 }
-key = PyString_FromStringAndSize(keystr, (Py_ssize_t)keylen);
+if (mutable) {
+// This will interpret the key as UTF-8.
+key = PyUnicode_FromStringAndSize(keystr, (Py_ssize_t)keylen);
+} else {
+// For immutable objects we'll manage key lookups, so we can avoid going
+// through the Unicode APIs. This avoids a potentially expensive and
+// definitely unnecessary conversion to UTF-16 and back for Python 2.
+// TODO: On Python 3 the Unicode APIs are smarter: we might be able to use
+// Unicode keys there without an appreciable performance loss.
+key = PyBytes_FromStringAndSize(keystr, (Py_ssize_t)keylen);
+}
 if (!key) {
 Py_DECREF(res);
 return NULL;
 }
-ele = bser_loads_recursive(ptr, end, mutable);
+ele = bser_loads_recursive(ptr, end, ctx);
 if (!ele) {
 Py_DECREF(key);
 Py_DECREF(res);
 return NULL;
 }
 return res;
 }
-static PyObject *bunser_template(const char **ptr, const char *end,
+static PyObject*
-int mutable)
+bunser_template(const char** ptr, const char* end, const unser_ctx_t* ctx) {
-{
+const char* buf = *ptr;
-const char *buf = *ptr;
 int64_t nitems, i;
-PyObject *arrval;
+int mutable = ctx->mutable;
-PyObject *keys;
+PyObject* arrval;
+PyObject* keys;
 Py_ssize_t numkeys, keyidx;
+unser_ctx_t keys_ctx = {0};
+if (mutable) {
+keys_ctx.mutable = 1;
+// Decode keys as UTF-8 in this case.
+keys_ctx.value_encoding = "utf-8";
+keys_ctx.value_errors = "strict";
+} else {
+// Treat keys as bytestrings in this case -- we'll do Unicode conversions at
+// lookup time.
+}
 if (buf[1] != BSER_ARRAY) {
 PyErr_Format(PyExc_ValueError, "Expect ARRAY to follow TEMPLATE");
 return NULL;
 }
 // skip header
 buf++;
 *ptr = buf;
-// Load template keys
+// Load template keys.
-keys = bunser_array(ptr, end, mutable);
+// For keys we don't want to do any decoding right now.
+keys = bunser_array(ptr, end, &keys_ctx);
 if (!keys) {
 return NULL;
 }
 numkeys = PySequence_Length(keys);
 Py_DECREF(keys);
 return NULL;
 }
 for (i = 0; i < nitems; i++) {
-PyObject *dict = NULL;
+PyObject* dict = NULL;
-bserObject *obj = NULL;
+bserObject* obj = NULL;
 if (mutable) {
 dict = PyDict_New();
 } else {
 obj = PyObject_New(bserObject, &bserObjectType);
 obj->values = PyTuple_New(numkeys);
 }
 dict = (PyObject*)obj;
 }
 if (!dict) {
 fail:
 Py_DECREF(keys);
 Py_DECREF(arrval);
 return NULL;
 }
 for (keyidx = 0; keyidx < numkeys; keyidx++) {
-PyObject *key;
+PyObject* key;
-PyObject *ele;
+PyObject* ele;
 if (**ptr == BSER_SKIP) {
 *ptr = *ptr + 1;
 ele = Py_None;
 Py_INCREF(ele);
 } else {
-ele = bser_loads_recursive(ptr, end, mutable);
+ele = bser_loads_recursive(ptr, end, ctx);
 }
 if (!ele) {
 goto fail;
 }
 Py_DECREF(keys);
 return arrval;
 }
-static PyObject *bser_loads_recursive(const char **ptr, const char *end,
+static PyObject* bser_loads_recursive(
-int mutable)
+const char** ptr,
-{
+const char* end,
-const char *buf = *ptr;
+const unser_ctx_t* ctx) {
+const char* buf = *ptr;
 switch (buf[0]) {
 case BSER_INT8:
 case BSER_INT16:
 case BSER_INT32:
-case BSER_INT64:
+case BSER_INT64: {
-{
+int64_t ival;
-int64_t ival;
+if (!bunser_int(ptr, end, &ival)) {
-if (!bunser_int(ptr, end, &ival)) {
+return NULL;
-return NULL;
+}
-}
+// Python 3 has one integer type.
-if (ival < LONG_MIN || ival > LONG_MAX) {
+#if PY_MAJOR_VERSION >= 3
 return PyLong_FromLongLong(ival);
-}
+#else
-return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t));
+if (ival < LONG_MIN || ival > LONG_MAX) {
-}
+return PyLong_FromLongLong(ival);
+}
-case BSER_REAL:
+return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t));
-{
+#endif // PY_MAJOR_VERSION >= 3
-double dval;
+}
-memcpy(&dval, buf + 1, sizeof(dval));
-*ptr = buf + 1 + sizeof(double);
+case BSER_REAL: {
-return PyFloat_FromDouble(dval);
+double dval;
-}
+memcpy(&dval, buf + 1, sizeof(dval));
+*ptr = buf + 1 + sizeof(double);
+return PyFloat_FromDouble(dval);
+}
 case BSER_TRUE:
 *ptr = buf + 1;
 Py_INCREF(Py_True);
 return Py_True;
 case BSER_NULL:
 *ptr = buf + 1;
 Py_INCREF(Py_None);
 return Py_None;
-case BSER_STRING:
+case BSER_BYTESTRING: {
-{
+const char* start;
-const char *start;
+int64_t len;
-int64_t len;
+if (!bunser_bytestring(ptr, end, &start, &len)) {
-if (!bunser_string(ptr, end, &start, &len)) {
+return NULL;
-return NULL;
+}
-}
+if (len > LONG_MAX) {
-if (len > LONG_MAX) {
+PyErr_Format(PyExc_ValueError, "string too long for python");
-PyErr_Format(PyExc_ValueError, "string too long for python");
+return NULL;
-return NULL;
+}
-}
+if (ctx->value_encoding != NULL) {
-return PyString_FromStringAndSize(start, (long)len);
+return PyUnicode_Decode(
-}
+start, (long)len, ctx->value_encoding, ctx->value_errors);
+} else {
+return PyBytes_FromStringAndSize(start, (long)len);
+}
+}
+case BSER_UTF8STRING: {
+const char* start;
+int64_t len;
+if (!bunser_bytestring(ptr, end, &start, &len)) {
+return NULL;
+}
+if (len > LONG_MAX) {
+PyErr_Format(PyExc_ValueError, "string too long for python");
+return NULL;
+}
+return PyUnicode_Decode(start, (long)len, "utf-8", "strict");
+}
 case BSER_ARRAY:
-return bunser_array(ptr, end, mutable);
+return bunser_array(ptr, end, ctx);
 case BSER_OBJECT:
-return bunser_object(ptr, end, mutable);
+return bunser_object(ptr, end, ctx);
 case BSER_TEMPLATE:
-return bunser_template(ptr, end, mutable);
+return bunser_template(ptr, end, ctx);
 default:
 PyErr_Format(PyExc_ValueError, "unhandled bser opcode 0x%02x", buf[0]);
 }
 return NULL;
 }
-// Expected use case is to read a packet from the socket and
+static int _pdu_info_helper(
-// then call bser.pdu_len on the packet.  It returns the total
+const char* data,
-// length of the entire response that the peer is sending,
+const char* end,
-// including the bytes already received.  This allows the client
+uint32_t* bser_version_out,
-// to compute the data size it needs to read before it can
+uint32_t* bser_capabilities_out,
-// decode the data
+int64_t* expected_len_out,
-static PyObject *bser_pdu_len(PyObject *self, PyObject *args)
+off_t* position_out) {
-{
+uint32_t bser_version;
-const char *start = NULL;
+uint32_t bser_capabilities = 0;
-const char *data = NULL;
+int64_t expected_len;
-int datalen = 0;
-const char *end;
+const char* start;
-int64_t expected_len, total_len;
+start = data;
-if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) {
-return NULL;
-}
-data = start;
-end = data + datalen;
 // Validate the header and length
-if (memcmp(data, EMPTY_HEADER, 2) != 0) {
+if (memcmp(data, EMPTY_HEADER, 2) == 0) {
+bser_version = 1;
+} else if (memcmp(data, EMPTY_HEADER_V2, 2) == 0) {
+bser_version = 2;
+} else {
 PyErr_SetString(PyExc_ValueError, "invalid bser header");
-return NULL;
+return 0;
 }
 data += 2;
+if (bser_version == 2) {
+// Expect an integer telling us what capabilities are supported by the
+// remote server (currently unused).
+if (!memcpy(&bser_capabilities, &data, sizeof(bser_capabilities))) {
+return 0;
+}
+data += sizeof(bser_capabilities);
+}
 // Expect an integer telling us how big the rest of the data
 // should be
 if (!bunser_int(&data, end, &expected_len)) {
-return NULL;
+return 0;
 }
-total_len = expected_len + (data - start);
+*bser_version_out = bser_version;
-if (total_len > LONG_MAX) {
+*bser_capabilities_out = (uint32_t)bser_capabilities;
-return PyLong_FromLongLong(total_len);
+*expected_len_out = expected_len;
-}
+*position_out = (off_t)(data - start);
-return PyInt_FromLong((long)total_len);
+return 1;
 }
-static PyObject *bser_loads(PyObject *self, PyObject *args)
+// This function parses the PDU header and provides info about the packet
-{
+// Returns false if unsuccessful
-const char *data = NULL;
+static int pdu_info_helper(
+PyObject* self,
+PyObject* args,
+uint32_t* bser_version_out,
+uint32_t* bser_capabilities_out,
+int64_t* total_len_out) {
+const char* start = NULL;
+const char* data = NULL;
 int datalen = 0;
-const char *end;
+const char* end;
 int64_t expected_len;
-int mutable = 1;
+off_t position;
-PyObject *mutable_obj = NULL;
+if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) {
-if (!PyArg_ParseTuple(args, "s#|O:loads", &data, &datalen, &mutable_obj)) {
+return 0;
-return NULL;
+}
-}
+data = start;
+end = data + datalen;
+if (!_pdu_info_helper(
+data,
+end,
+bser_version_out,
+bser_capabilities_out,
+&expected_len,
+&position)) {
+return 0;
+}
+*total_len_out = (int64_t)(expected_len + position);
+return 1;
+}
+// Expected use case is to read a packet from the socket and then call
+// bser.pdu_info on the packet.  It returns the BSER version, BSER capabilities,
+// and the total length of the entire response that the peer is sending,
+// including the bytes already received. This allows the client  to compute the
+// data size it needs to read before it can decode the data.
+static PyObject* bser_pdu_info(PyObject* self, PyObject* args) {
+uint32_t version, capabilities;
+int64_t total_len;
+if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) {
+return NULL;
+}
+return Py_BuildValue("kkL", version, capabilities, total_len);
+}
+static PyObject* bser_pdu_len(PyObject* self, PyObject* args) {
+uint32_t version, capabilities;
+int64_t total_len;
+if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) {
+return NULL;
+}
+return Py_BuildValue("L", total_len);
+}
+static PyObject* bser_loads(PyObject* self, PyObject* args, PyObject* kw) {
+const char* data = NULL;
+int datalen = 0;
+const char* start;
+const char* end;
+int64_t expected_len;
+off_t position;
+PyObject* mutable_obj = NULL;
+const char* value_encoding = NULL;
+const char* value_errors = NULL;
+unser_ctx_t ctx = {1, 0};
+static char* kw_list[] = {
+"buf", "mutable", "value_encoding", "value_errors", NULL};
+if (!PyArg_ParseTupleAndKeywords(
+args,
+kw,
+"s#|Ozz:loads",
+kw_list,
+&start,
+&datalen,
+&mutable_obj,
+&value_encoding,
+&value_errors)) {
+return NULL;
+}
 if (mutable_obj) {
-mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0;
+ctx.mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0;
 }
+ctx.value_encoding = value_encoding;
+if (value_encoding == NULL) {
+ctx.value_errors = NULL;
+} else if (value_errors == NULL) {
+ctx.value_errors = "strict";
+} else {
+ctx.value_errors = value_errors;
+}
+data = start;
 end = data + datalen;
-// Validate the header and length
+if (!_pdu_info_helper(
-if (memcmp(data, EMPTY_HEADER, 2) != 0) {
+data,
-PyErr_SetString(PyExc_ValueError, "invalid bser header");
+end,
-return NULL;
+&ctx.bser_version,
-}
+&ctx.bser_capabilities,
+&expected_len,
-data += 2;
+&position)) {
+return NULL;
-// Expect an integer telling us how big the rest of the data
+}
-// should be
-if (!bunser_int(&data, end, &expected_len)) {
+data = start + position;
-return NULL;
-}
 // Verify
 if (expected_len + data != end) {
 PyErr_SetString(PyExc_ValueError, "bser data len != header len");
 return NULL;
 }
-return bser_loads_recursive(&data, end, mutable);
+return bser_loads_recursive(&data, end, &ctx);
 }
+static PyObject* bser_load(PyObject* self, PyObject* args, PyObject* kw) {
+PyObject *load, *string;
+PyObject* fp = NULL;
+PyObject* mutable_obj = NULL;
+const char* value_encoding = NULL;
+const char* value_errors = NULL;
+static char* kw_list[] = {
+"fp", "mutable", "value_encoding", "value_errors", NULL};
+if (!PyArg_ParseTupleAndKeywords(
+args,
+kw,
+"OOzz:load",
+kw_list,
+&fp,
+&mutable_obj,
+&value_encoding,
+&value_errors)) {
+return NULL;
+}
+load = PyImport_ImportModule("pywatchman.load");
+if (load == NULL) {
+return NULL;
+}
+string = PyObject_CallMethod(
+load, "load", "OOzz", fp, mutable_obj, value_encoding, value_errors);
+Py_DECREF(load);
+return string;
+}
+// clang-format off
 static PyMethodDef bser_methods[] = {
-{"loads",  bser_loads, METH_VARARGS, "Deserialize string."},
+{"loads", (PyCFunction)bser_loads, METH_VARARGS | METH_KEYWORDS,
-{"pdu_len", bser_pdu_len, METH_VARARGS, "Extract PDU length."},
+"Deserialize string."},
-{"dumps",  bser_dumps, METH_VARARGS, "Serialize string."},
+{"load", (PyCFunction)bser_load, METH_VARARGS | METH_KEYWORDS,
+"Deserialize a file object"},
+{"pdu_info", (PyCFunction)bser_pdu_info, METH_VARARGS,
+"Extract PDU information."},
+{"pdu_len", (PyCFunction)bser_pdu_len, METH_VARARGS,
+"Extract total PDU length."},
+{"dumps",  (PyCFunction)bser_dumps, METH_VARARGS | METH_KEYWORDS,
+"Serialize string."},
 {NULL, NULL, 0, NULL}
 };
-PyMODINIT_FUNC initbser(void)
+#if PY_MAJOR_VERSION >= 3
-{
+static struct PyModuleDef bser_module = {
+PyModuleDef_HEAD_INIT,
+"bser",
+"Efficient encoding and decoding of BSER.",
+-1,
+bser_methods
+};
+// clang-format on
+PyMODINIT_FUNC PyInit_bser(void) {
+PyObject* mod;
+mod = PyModule_Create(&bser_module);
+PyType_Ready(&bserObjectType);
+return mod;
+}
+#else
+PyMODINIT_FUNC initbser(void) {
 (void)Py_InitModule("bser", bser_methods);
 PyType_Ready(&bserObjectType);
 }
+#endif // PY_MAJOR_VERSION >= 3
 /* vim:ts=2:sw=2:et:
 */
-// no-check-code -- this is a 3rd party library

changeset 30656	16f4b341288d
parent 28528	a011080fdb7b
child 37594	b1f62cd39b5c