Add tests based on TestAllTypes from the C++ test.capnp. Fix problems uncovered in capnp.pyx. Requires capnproto-c++-0.3.0-rc5, as I found a bug in the base library.

capnp/capnp.pyx

@@ -30,14 +30,7 @@ ctypedef bint Bool
 ctypedef float Float32
 ctypedef double Float64
 from libc.stdlib cimport malloc, free
-
+from libcpp cimport bool as cbool
-ctypedef fused valid_values:
-    int
-    long
-    float
-    double
-    bint
-    cython.p_char
 
 def _make_enum(enum_name, *sequential, **named):
     enums = dict(zip(sequential, range(len(sequential))), **named)
@@ -264,31 +257,76 @@ cdef class _DynamicListBuilder:
         index = index % size
         return self._get(index)
 
-    def _setitem(self, index, valid_values value):
+    cdef _setitemInt(self, index, value):
-        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value)
+        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<long long>value)
         self.thisptr.set(index, temp)
 
-    cdef _setattrDynamicStructBuilder(self, index, _DynamicStructBuilder value):
+    cdef _setitemLong(self, index, value):
+        cdef C_DynamicValue.Reader temp
+        if value < 0:
+           temp = C_DynamicValue.Reader(<long long>value)
+        else:
+           temp = C_DynamicValue.Reader(<unsigned long long>value)
+        self.thisptr.set(index, temp)
+
+    cdef _setitemDouble(self, index, value):
+        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<double>value)
+        self.thisptr.set(index, temp)
+
+    cdef _setitemBool(self, index, value):
+        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<cbool>value)
+        self.thisptr.set(index, temp)
+
+    cdef _setitemString(self, index, value):
+        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<char*>value)
+        self.thisptr.set(index, temp)
+
+    cdef _setitemVoid(self, index):
+        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(VOID)
+        self.thisptr.set(index, temp)
+
+    cdef _setitemList(self, index, value):
+        builder = toPython(self.thisptr.init(index, len(value)), self._parent)
+        for (i, v) in enumerate(value):
+            builder[i] = v
+
+    cdef _setitemDynamicStructBuilder(self, index, _DynamicStructBuilder value):
         cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value.thisptr.asReader())
         self.thisptr.set(index, temp)
 
-    cdef _setattrDynamicStructReader(self, index, _DynamicStructReader value):
+    cdef _setitemDynamicStructReader(self, index, _DynamicStructReader value):
         cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value.thisptr)
         self.thisptr.set(index, temp)
 
     def __setitem__(self, index, value):
+        # TODO: share code with _DynamicStructBuilder.__setattr__
+        
         size = self.thisptr.size()
         if index >= size:
             raise IndexError('Out of bounds')
         index = index % size
         value_type = type(value)
 
-        if value_type is _DynamicStructBuilder:
+        if value_type is int:
+            self._setitemInt(index, value)
+        elif value_type is long:
+            self._setitemLong(index, value)
+        elif value_type is float:
+            self._setitemDouble(index, value)
+        elif value_type is bool:
+            self._setitemBool(index, value)
+        elif value_type is str:
+            self._setitemString(index, value)
+        elif value_type is list:
+            self._setitemList(index, value)
+        elif value is None:
+            self._setitemVoid(index)
+        elif value_type is _DynamicStructBuilder:
             self._setattrDynamicStructBuilder(index, value)
         elif value_type is _DynamicStructReader:
             self._setattrDynamicStructReader(index, value)
         else:
-            self._setitem(index, value)
+            raise ValueError("Non primitive type")
 
     def __len__(self):
         return self.thisptr.size()
@@ -504,12 +542,20 @@ cdef class _DynamicStructBuilder:
         cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<long long>value)
         self.thisptr.set(field, temp)
 
+    cdef _setattrLong(self, field, value):
+        cdef C_DynamicValue.Reader temp
+        if value < 0:
+           temp = C_DynamicValue.Reader(<long long>value)
+        else:
+           temp = C_DynamicValue.Reader(<unsigned long long>value)
+        self.thisptr.set(field, temp)
+
     cdef _setattrDouble(self, field, value):
         cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<double>value)
         self.thisptr.set(field, temp)
 
     cdef _setattrBool(self, field, value):
-        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<bint>value)
+        cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(<cbool>value)
         self.thisptr.set(field, temp)
 
     cdef _setattrString(self, field, value):
@@ -520,6 +566,11 @@ cdef class _DynamicStructBuilder:
         cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(VOID)
         self.thisptr.set(field, temp)
 
+    cdef _setattrList(self, field, value):
+        builder = toPython(self.thisptr.init(field, len(value)), self._parent)
+        for (i, v) in enumerate(value):
+            builder[i] = v
+
     cdef _setattrDynamicStructBuilder(self, field, _DynamicStructBuilder value):
         cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value.thisptr.asReader())
         self.thisptr.set(field, temp)
@@ -529,16 +580,21 @@ cdef class _DynamicStructBuilder:
         self.thisptr.set(field, temp)
 
     def __setattr__(self, field, value):
+        # TODO: share code with _DynamicListBuilder.__setitem__
         value_type = type(value)
 
         if value_type is int:
             self._setattrInt(field, value)
+        elif value_type is long:
+            self._setattrLong(field, value)
         elif value_type is float:
             self._setattrDouble(field, value)
         elif value_type is bool:
             self._setattrBool(field, value)
         elif value_type is str:
             self._setattrString(field, value)
+        elif value_type is list:
+            self._setattrList(field, value)
         elif value is None:
             self._setattrVoid(field)
         elif value_type is _DynamicStructBuilder:

capnp/capnp_cpp.pxd

@@ -6,6 +6,7 @@ from schema_cpp cimport Node, Data, StructNode, EnumNode
 
 from libc.stdint cimport *
 ctypedef unsigned int uint
+from libcpp cimport bool as cbool
 
 cdef extern from "capnp/common.h" namespace " ::capnp":
     enum Void:
@@ -137,7 +138,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
         cppclass Reader:
             Reader()
             Reader(Void value)
-            Reader(bint value)
+            Reader(cbool value)
             Reader(char value)
             Reader(short value)
             Reader(int value)

test/all-types.capnp

@@ -0,0 +1,52 @@
+# This is copied from test.capnp in the Cap'n Proto C++ code.
+
+@0xcaa67a26d16950d9;
+
+enum TestEnum {
+  foo @0;
+  bar @1;
+  baz @2;
+  qux @3;
+  quux @4;
+  corge @5;
+  grault @6;
+  garply @7;
+}
+
+struct TestAllTypes {
+  voidField      @0  : Void;
+  boolField      @1  : Bool;
+  int8Field      @2  : Int8;
+  int16Field     @3  : Int16;
+  int32Field     @4  : Int32;
+  int64Field     @5  : Int64;
+  uInt8Field     @6  : UInt8;
+  uInt16Field    @7  : UInt16;
+  uInt32Field    @8  : UInt32;
+  uInt64Field    @9  : UInt64;
+  float32Field   @10 : Float32;
+  float64Field   @11 : Float64;
+  textField      @12 : Text;
+  dataField      @13 : Data;
+  structField    @14 : TestAllTypes;
+  enumField      @15 : TestEnum;
+  interfaceField @16 : Void;  # TODO
+
+  voidList      @17 : List(Void);
+  boolList      @18 : List(Bool);
+  int8List      @19 : List(Int8);
+  int16List     @20 : List(Int16);
+  int32List     @21 : List(Int32);
+  int64List     @22 : List(Int64);
+  uInt8List     @23 : List(UInt8);
+  uInt16List    @24 : List(UInt16);
+  uInt32List    @25 : List(UInt32);
+  uInt64List    @26 : List(UInt64);
+  float32List   @27 : List(Float32);
+  float64List   @28 : List(Float64);
+  textList      @29 : List(Text);
+  dataList      @30 : List(Data);
+  structList    @31 : List(TestAllTypes);
+  enumList      @32 : List(TestEnum);
+  interfaceList @33 : List(Void);  # TODO
+}

test/all-types.txt

@@ -0,0 +1,71 @@
+( boolField = true,
+  int8Field = -123,
+  int16Field = -12345,
+  int32Field = -12345678,
+  int64Field = -123456789012345,
+  uInt8Field = 234,
+  uInt16Field = 45678,
+  uInt32Field = 3456789012,
+  uInt64Field = 12345678901234567890,
+  float32Field = 1234.5,
+  float64Field = -1.23e47,
+  textField = "foo",
+  dataField = "bar",
+  structField = (
+    boolField = true,
+    int8Field = -12,
+    int16Field = 3456,
+    int32Field = -78901234,
+    int64Field = 56789012345678,
+    uInt8Field = 90,
+    uInt16Field = 1234,
+    uInt32Field = 56789012,
+    uInt64Field = 345678901234567890,
+    float32Field = -1.25e-10,
+    float64Field = 345,
+    textField = "baz",
+    dataField = "qux",
+    structField = (
+      textField = "nested",
+      structField = (
+        textField = "really nested" ) ),
+    enumField = baz,
+    voidList = [void, void, void],
+    boolList = [false, true, false, true, true],
+    int8List = [12, -34, -128, 127],
+    int16List = [1234, -5678, -32768, 32767],
+    int32List = [12345678, -90123456, -2147483648, 2147483647],
+    int64List = [123456789012345, -678901234567890, -9223372036854775808, 9223372036854775807],
+    uInt8List = [12, 34, 0, 255],
+    uInt16List = [1234, 5678, 0, 65535],
+    uInt32List = [12345678, 90123456, 0, 4294967295],
+    uInt64List = [123456789012345, 678901234567890, 0, 18446744073709551615],
+    float32List = [0, 1234567, 1e37, -1e37, 1e-37, -1e-37],
+    float64List = [0, 123456789012345, 1e306, -1e306, 1e-306, -1e-306],
+    textList = ["quux", "corge", "grault"],
+    dataList = ["garply", "waldo", "fred"],
+    structList = [
+      ( textField = "x structlist 1" ),
+      ( textField = "x structlist 2" ),
+      ( textField = "x structlist 3" ) ],
+    enumList = [qux, bar, grault] ),
+  enumField = corge,
+  voidList = [void, void, void, void, void, void],
+  boolList = [true, false, false, true],
+  int8List = [111, -111],
+  int16List = [11111, -11111],
+  int32List = [111111111, -111111111],
+  int64List = [1111111111111111111, -1111111111111111111],
+  uInt8List = [111, 222],
+  uInt16List = [33333, 44444],
+  uInt32List = [3333333333],
+  uInt64List = [11111111111111111111],
+  float32List = [5555.5, inf, -inf, nan],
+  float64List = [7777.75, inf, -inf, nan],
+  textList = ["plugh", "xyzzy", "thud"],
+  dataList = ["oops", "exhausted", "rfc3092"],
+  structList = [
+    ( textField = "structlist 1" ),
+    ( textField = "structlist 2" ),
+    ( textField = "structlist 3" ) ],
+  enumList = [foo, garply] )

test/test_regression.py

@@ -1,6 +1,7 @@
 import pytest
 import capnp
 import os
+import math
 
 this_dir = os.path.dirname(__file__)
 
@@ -65,3 +66,226 @@ def test_addressbook(addressbook):
 
     f = open('example', 'r')
     printAddressBook(f.fileno())
+
+@pytest.fixture
+def all_types():
+    return capnp.load(os.path.join(this_dir, 'all-types.capnp'))
+
+# TODO:  These tests should be extended to:
+# - Read each field in Python and assert that it is equal to the expected value.
+# - Build an identical message using Python code and compare it to the golden.
+#
+
+def init_all_types(builder):
+    builder.voidField = None
+    builder.boolField = True
+    builder.int8Field = -123
+    builder.int16Field = -12345
+    builder.int32Field = -12345678
+    builder.int64Field = -123456789012345
+    builder.uInt8Field = 234
+    builder.uInt16Field = 45678
+    builder.uInt32Field = 3456789012
+    builder.uInt64Field = 12345678901234567890
+    builder.float32Field = 1234.5
+    builder.float64Field = -123e45
+    builder.textField = "foo"
+    builder.dataField = "bar"
+
+    subBuilder = builder.structField
+    subBuilder.voidField = None
+    subBuilder.boolField = True
+    subBuilder.int8Field = -12
+    subBuilder.int16Field = 3456
+    subBuilder.int32Field = -78901234
+    subBuilder.int64Field = 56789012345678
+    subBuilder.uInt8Field = 90
+    subBuilder.uInt16Field = 1234
+    subBuilder.uInt32Field = 56789012
+    subBuilder.uInt64Field = 345678901234567890
+    subBuilder.float32Field = -1.25e-10
+    subBuilder.float64Field = 345
+    subBuilder.textField = "baz"
+    subBuilder.dataField = "qux"
+    subSubBuilder = subBuilder.structField
+    subSubBuilder.textField = "nested"
+    subSubBuilder.structField.textField = "really nested"
+    subBuilder.enumField = "baz"
+
+    subBuilder.voidList = [None, None, None]
+    subBuilder.boolList = [False, True, False, True, True]
+    subBuilder.int8List = [12, -34, -0x80, 0x7f]
+    subBuilder.int16List = [1234, -5678, -0x8000, 0x7fff]
+    subBuilder.int32List = [12345678, -90123456, -0x80000000, 0x7fffffff]
+    subBuilder.int64List = [123456789012345, -678901234567890, -0x8000000000000000, 0x7fffffffffffffff]
+    subBuilder.uInt8List = [12, 34, 0, 0xff]
+    subBuilder.uInt16List = [1234, 5678, 0, 0xffff]
+    subBuilder.uInt32List = [12345678, 90123456, 0, 0xffffffff]
+    subBuilder.uInt64List = [123456789012345, 678901234567890, 0, 0xffffffffffffffff]
+    subBuilder.float32List = [0, 1234567, 1e37, -1e37, 1e-37, -1e-37]
+    subBuilder.float64List = [0, 123456789012345, 1e306, -1e306, 1e-306, -1e-306]
+    subBuilder.textList = ["quux", "corge", "grault"]
+    subBuilder.dataList = ["garply", "waldo", "fred"]
+    listBuilder = subBuilder.init('structList', 3)
+    listBuilder[0].textField = "x structlist 1"
+    listBuilder[1].textField = "x structlist 2"
+    listBuilder[2].textField = "x structlist 3"
+    subBuilder.enumList = ["qux", "bar", "grault"]
+
+    builder.enumField = "corge"
+
+    builder.init("voidList", 6)
+    builder.boolList = [True, False, False, True]
+    builder.int8List = [111, -111]
+    builder.int16List = [11111, -11111]
+    builder.int32List = [111111111, -111111111]
+    builder.int64List = [1111111111111111111, -1111111111111111111]
+    builder.uInt8List = [111, 222]
+    builder.uInt16List = [33333, 44444]
+    builder.uInt32List = [3333333333]
+    builder.uInt64List = [11111111111111111111]
+    builder.float32List = [5555.5, float("inf"), float("-inf"), float("nan")]
+    builder.float64List = [7777.75, float("inf"), float("-inf"), float("nan")]
+    builder.textList = ["plugh", "xyzzy", "thud"]
+    builder.dataList = ["oops", "exhausted", "rfc3092"]
+    listBuilder = builder.init('structList', 3)
+    listBuilder[0].textField = "structlist 1"
+    listBuilder[1].textField = "structlist 2"
+    listBuilder[2].textField = "structlist 3"
+    builder.enumList = ["foo", "garply"]
+
+def assert_almost(float1, float2):
+    if float1 != float2:
+        assert abs((float1 - float2) / float1) < 0.00001
+
+def check_list(reader, expected):
+    assert len(reader) == len(expected)
+    for (i, v) in enumerate(expected):
+        if type(v) is float:
+            assert_almost(reader[i], v)
+        else:
+            assert reader[i] == v
+
+def check_all_types(reader):
+    assert reader.voidField == None
+    assert reader.boolField == True
+    assert reader.int8Field == -123
+    assert reader.int16Field == -12345
+    assert reader.int32Field == -12345678
+    assert reader.int64Field == -123456789012345
+    assert reader.uInt8Field == 234
+    assert reader.uInt16Field == 45678
+    assert reader.uInt32Field == 3456789012
+    assert reader.uInt64Field == 12345678901234567890
+    assert reader.float32Field == 1234.5
+    assert_almost(reader.float64Field, -123e45)
+    assert reader.textField == "foo"
+    assert reader.dataField == "bar"
+
+    subReader = reader.structField
+    assert subReader.voidField == None
+    assert subReader.boolField == True
+    assert subReader.int8Field == -12
+    assert subReader.int16Field == 3456
+    assert subReader.int32Field == -78901234
+    assert subReader.int64Field == 56789012345678
+    assert subReader.uInt8Field == 90
+    assert subReader.uInt16Field == 1234
+    assert subReader.uInt32Field == 56789012
+    assert subReader.uInt64Field == 345678901234567890
+    assert_almost(subReader.float32Field, -1.25e-10)
+    assert subReader.float64Field == 345
+    assert subReader.textField == "baz"
+    assert subReader.dataField == "qux"
+
+    subSubReader = subReader.structField
+    assert subSubReader.textField == "nested"
+    assert subSubReader.structField.textField == "really nested"
+
+    assert subReader.enumField == "baz"
+
+    check_list(subReader.voidList, [None, None, None])
+    check_list(subReader.boolList, [False, True, False, True, True])
+    check_list(subReader.int8List, [12, -34, -0x80, 0x7f])
+    check_list(subReader.int16List, [1234, -5678, -0x8000, 0x7fff])
+    check_list(subReader.int32List, [12345678, -90123456, -0x80000000, 0x7fffffff])
+    check_list(subReader.int64List, [123456789012345, -678901234567890, -0x8000000000000000, 0x7fffffffffffffff])
+    check_list(subReader.uInt8List, [12, 34, 0, 0xff])
+    check_list(subReader.uInt16List, [1234, 5678, 0, 0xffff])
+    check_list(subReader.uInt32List, [12345678, 90123456, 0, 0xffffffff])
+    check_list(subReader.uInt64List, [123456789012345, 678901234567890, 0, 0xffffffffffffffff])
+    check_list(subReader.float32List, [0.0, 1234567.0, 1e37, -1e37, 1e-37, -1e-37])
+    check_list(subReader.float64List, [0.0, 123456789012345.0, 1e306, -1e306, 1e-306, -1e-306])
+    check_list(subReader.textList, ["quux", "corge", "grault"])
+    check_list(subReader.dataList, ["garply", "waldo", "fred"])
+
+    listReader = subReader.structList
+    assert len(listReader) == 3
+    assert listReader[0].textField == "x structlist 1"
+    assert listReader[1].textField == "x structlist 2"
+    assert listReader[2].textField == "x structlist 3"
+
+    check_list(subReader.enumList, ["qux", "bar", "grault"])
+
+    assert reader.enumField == "corge"
+
+    assert len(reader.voidList) == 6
+    check_list(reader.boolList, [True, False, False, True])
+    check_list(reader.int8List, [111, -111])
+    check_list(reader.int16List, [11111, -11111])
+    check_list(reader.int32List, [111111111, -111111111])
+    check_list(reader.int64List, [1111111111111111111, -1111111111111111111])
+    check_list(reader.uInt8List, [111, 222])
+    check_list(reader.uInt16List, [33333, 44444])
+    check_list(reader.uInt32List, [3333333333])
+    check_list(reader.uInt64List, [11111111111111111111])
+
+    listReader = reader.float32List
+    assert len(listReader) == 4
+    assert listReader[0] == 5555.5
+    assert listReader[1] == float("inf")
+    assert listReader[2] == -float("inf")
+    assert math.isnan(listReader[3])
+
+    listReader = reader.float64List
+    len(listReader) == 4
+    assert listReader[0] == 7777.75
+    assert listReader[1] == float("inf")
+    assert listReader[2] == -float("inf")
+    assert math.isnan(listReader[3])
+
+    check_list(reader.textList, ["plugh", "xyzzy", "thud"])
+    check_list(reader.dataList, ["oops", "exhausted", "rfc3092"])
+
+    listReader = reader.structList
+    len(listReader) == 3
+    assert listReader[0].textField == "structlist 1"
+    assert listReader[1].textField == "structlist 2"
+    assert listReader[2].textField == "structlist 3"
+
+    check_list(reader.enumList, ["foo", "garply"])
+
+def test_build(all_types):
+    builder = capnp.MallocMessageBuilder()
+    root = builder.getRoot(all_types.TestAllTypes)
+    init_all_types(root)
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    assert str(root) + '\n' == expectedText
+
+def test_binary_read(all_types):
+    f = open(os.path.join(this_dir, 'all-types.binary'), 'r')
+    message = capnp.StreamFdMessageReader(f.fileno())
+    root = message.getRoot(all_types.TestAllTypes)
+    check_all_types(root)
+
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    assert str(root) + '\n' == expectedText
+
+def test_packed_read(all_types):
+    f = open(os.path.join(this_dir, 'all-types.packed'), 'r')
+    message = capnp.PackedFdMessageReader(f.fileno())
+    root = message.getRoot(all_types.TestAllTypes)
+    check_all_types(root)
+
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    assert str(root) + '\n' == expectedText