From 688cecffab7dd2816813d6a7fd4e424245cc90ae Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 31 Aug 2013 13:21:29 -0700 Subject: [PATCH 1/6] Remove stray print statement. --- capnp/capnp.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index de1cecd..719ea26 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -347,7 +347,6 @@ cdef toPythonReader(C_DynamicValue.Reader self, object parent): temp = self.asData() return (temp.begin())[:temp.size()] elif type == capnp.TYPE_LIST: - print 'list' return list(_DynamicListReader()._init(self.asList(), parent)) elif type == capnp.TYPE_STRUCT: return _DynamicStructReader()._init(self.asStruct(), parent) From 10c6f40c2e640f282297fd46eb76e27c972be09b Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 31 Aug 2013 15:07:13 -0700 Subject: [PATCH 2/6] Implement __str__ and __repr__ for struct and list builders. __str__ uses prettyPrint while __repr__ shows the type name and the low-whitespace stringification. Also implement __repr__ for StructSchema, just because why not? --- capnp/capnp.pyx | 49 ++++++++++++++++++++++++++++++++++++++++++++- capnp/capnp_cpp.pxd | 6 ++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index 719ea26..84c730f 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -9,7 +9,7 @@ cimport cython cimport capnp_cpp as capnp cimport schema_cpp -from capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, fixMaybe, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, DynamicOrphan as C_DynamicOrphan +from capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, fixMaybe, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan from schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode from cython.operator cimport dereference as deref @@ -73,6 +73,18 @@ cdef extern from "capnp/list.h" namespace " ::capnp": cdef extern from "" namespace "std": C_DynamicOrphan moveOrphan"std::move"(C_DynamicOrphan) +cdef extern from "" namespace " ::capnp": + StringTree printStructReader" ::capnp::prettyPrint"(C_DynamicStruct.Reader) + StringTree printStructBuilder" ::capnp::prettyPrint"(C_DynamicStruct.Builder) + StringTree printListReader" ::capnp::prettyPrint"(C_DynamicList.Reader) + StringTree printListBuilder" ::capnp::prettyPrint"(C_DynamicList.Builder) + +cdef extern from "" namespace " ::kj": + String strStructReader" ::kj::str"(C_DynamicStruct.Reader) + String strStructBuilder" ::kj::str"(C_DynamicStruct.Builder) + String strListReader" ::kj::str"(C_DynamicList.Reader) + String strListBuilder" ::kj::str"(C_DynamicList.Builder) + cdef class _NodeReader: cdef C_Node.Reader thisptr cdef init(self, C_Node.Reader other): @@ -144,6 +156,13 @@ cdef class _DynamicListReader: def __len__(self): return self.thisptr.size() + def __str__(self): + return printListReader(self.thisptr).flatten().cStr() + + def __repr__(self): + # TODO: Print the list type. + return '' % strListReader(self.thisptr).cStr() + cdef class _DynamicResizableListBuilder: """Class for building growable Cap'n Proto Lists @@ -315,6 +334,13 @@ cdef class _DynamicListBuilder: """ return _DynamicOrphan()._init(self.thisptr.disown(index), self._parent) + def __str__(self): + return printListBuilder(deref(self.thisptr)).flatten().cStr() + + def __repr__(self): + # TODO: Print the list type. + return '' % strListBuilder(deref(self.thisptr)).cStr() + cdef class _List_NestedNode_Reader: cdef List[C_Node.NestedNode].Reader thisptr cdef _init(self, List[C_Node.NestedNode].Reader other): @@ -439,6 +465,12 @@ cdef class _DynamicStructReader: def __dir__(self): return list(self.schema.fieldnames) + def __str__(self): + return printStructReader(self.thisptr).flatten().cStr() + + def __repr__(self): + return '<%s reader %s>' % (self.schema.node.displayName, strStructReader(self.thisptr).cStr()) + cdef class _DynamicStructBuilder: """Builds Cap'n Proto structs @@ -638,6 +670,12 @@ cdef class _DynamicStructBuilder: def __dir__(self): return list(self.schema.fieldnames) + def __str__(self): + return printStructBuilder(deref(self.thisptr)).flatten().cStr() + + def __repr__(self): + return '<%s builder %s>' % (self.schema.node.displayName, strStructBuilder(deref(self.thisptr)).cStr()) + cdef class _DynamicOrphan: cdef C_DynamicOrphan thisptr cdef public object _parent @@ -656,6 +694,12 @@ cdef class _DynamicOrphan: """ return toPython(self.thisptr.get(), self._parent) + def __str__(self): + return str(self.get()) + + def __repr__(self): + return repr(self.get()) + cdef class _Schema: cdef C_Schema thisptr cdef _init(self, C_Schema other): @@ -698,6 +742,9 @@ cdef class _StructSchema: def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), None) + def __repr__(self): + return '' % self.node.displayName + cdef class _ParsedSchema: cdef C_ParsedSchema thisptr cdef _init(self, C_ParsedSchema other): diff --git a/capnp/capnp_cpp.pxd b/capnp/capnp_cpp.pxd index 5397f03..832a14e 100644 --- a/capnp/capnp_cpp.pxd +++ b/capnp/capnp_cpp.pxd @@ -14,6 +14,12 @@ cdef extern from "capnp/common.h" namespace " ::capnp": cdef extern from "kj/string.h" namespace " ::kj": cdef cppclass StringPtr: StringPtr(char *) + cdef cppclass String: + char* cStr() + +cdef extern from "kj/string-tree.h" namespace " ::kj": + cdef cppclass StringTree: + String flatten() cdef extern from "kj/common.h" namespace " ::kj": cdef cppclass Maybe[T]: From 0478ee30814a2390affb0fad5df906de3b39adbc Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 31 Aug 2013 16:44:54 -0700 Subject: [PATCH 3/6] Add tests based on TestAllTypes from the C++ test.capnp. Fix problems uncovered in capnp.pyx. Requires capnproto-c++-0.3.0-rc5, as I found a bug in the base library. --- capnp/capnp.pyx | 86 ++++++++++++--- capnp/capnp_cpp.pxd | 3 +- test/all-types.binary | Bin 0 -> 2816 bytes test/all-types.capnp | 52 ++++++++++ test/all-types.packed | Bin 0 -> 831 bytes test/all-types.txt | 71 +++++++++++++ test/test_regression.py | 224 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 420 insertions(+), 16 deletions(-) create mode 100644 test/all-types.binary create mode 100644 test/all-types.capnp create mode 100644 test/all-types.packed create mode 100644 test/all-types.txt diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index 84c730f..9c19c73 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -30,14 +30,7 @@ ctypedef bint Bool ctypedef float Float32 ctypedef double Float64 from libc.stdlib cimport malloc, free - -ctypedef fused valid_values: - int - long - float - double - bint - cython.p_char +from libcpp cimport bool as cbool def _make_enum(enum_name, *sequential, **named): enums = dict(zip(sequential, range(len(sequential))), **named) @@ -264,31 +257,76 @@ cdef class _DynamicListBuilder: index = index % size return self._get(index) - def _setitem(self, index, valid_values value): - cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + cdef _setitemInt(self, index, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) self.thisptr.set(index, temp) - cdef _setattrDynamicStructBuilder(self, index, _DynamicStructBuilder value): + cdef _setitemLong(self, index, value): + cdef C_DynamicValue.Reader temp + if value < 0: + temp = C_DynamicValue.Reader(value) + else: + temp = C_DynamicValue.Reader(value) + self.thisptr.set(index, temp) + + cdef _setitemDouble(self, index, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + self.thisptr.set(index, temp) + + cdef _setitemBool(self, index, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + self.thisptr.set(index, temp) + + cdef _setitemString(self, index, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + self.thisptr.set(index, temp) + + cdef _setitemVoid(self, index): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(VOID) + self.thisptr.set(index, temp) + + cdef _setitemList(self, index, value): + builder = toPython(self.thisptr.init(index, len(value)), self._parent) + for (i, v) in enumerate(value): + builder[i] = v + + cdef _setitemDynamicStructBuilder(self, index, _DynamicStructBuilder value): cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value.thisptr.asReader()) self.thisptr.set(index, temp) - cdef _setattrDynamicStructReader(self, index, _DynamicStructReader value): + cdef _setitemDynamicStructReader(self, index, _DynamicStructReader value): cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value.thisptr) self.thisptr.set(index, temp) def __setitem__(self, index, value): + # TODO: share code with _DynamicStructBuilder.__setattr__ + size = self.thisptr.size() if index >= size: raise IndexError('Out of bounds') index = index % size value_type = type(value) - if value_type is _DynamicStructBuilder: + if value_type is int: + self._setitemInt(index, value) + elif value_type is long: + self._setitemLong(index, value) + elif value_type is float: + self._setitemDouble(index, value) + elif value_type is bool: + self._setitemBool(index, value) + elif value_type is str: + self._setitemString(index, value) + elif value_type is list: + self._setitemList(index, value) + elif value is None: + self._setitemVoid(index) + elif value_type is _DynamicStructBuilder: self._setattrDynamicStructBuilder(index, value) elif value_type is _DynamicStructReader: self._setattrDynamicStructReader(index, value) else: - self._setitem(index, value) + raise ValueError("Non primitive type") def __len__(self): return self.thisptr.size() @@ -504,12 +542,20 @@ cdef class _DynamicStructBuilder: cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) self.thisptr.set(field, temp) + cdef _setattrLong(self, field, value): + cdef C_DynamicValue.Reader temp + if value < 0: + temp = C_DynamicValue.Reader(value) + else: + temp = C_DynamicValue.Reader(value) + self.thisptr.set(field, temp) + cdef _setattrDouble(self, field, value): cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) self.thisptr.set(field, temp) cdef _setattrBool(self, field, value): - cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) self.thisptr.set(field, temp) cdef _setattrString(self, field, value): @@ -520,6 +566,11 @@ cdef class _DynamicStructBuilder: cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(VOID) self.thisptr.set(field, temp) + cdef _setattrList(self, field, value): + builder = toPython(self.thisptr.init(field, len(value)), self._parent) + for (i, v) in enumerate(value): + builder[i] = v + cdef _setattrDynamicStructBuilder(self, field, _DynamicStructBuilder value): cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value.thisptr.asReader()) self.thisptr.set(field, temp) @@ -529,16 +580,21 @@ cdef class _DynamicStructBuilder: self.thisptr.set(field, temp) def __setattr__(self, field, value): + # TODO: share code with _DynamicListBuilder.__setitem__ value_type = type(value) if value_type is int: self._setattrInt(field, value) + elif value_type is long: + self._setattrLong(field, value) elif value_type is float: self._setattrDouble(field, value) elif value_type is bool: self._setattrBool(field, value) elif value_type is str: self._setattrString(field, value) + elif value_type is list: + self._setattrList(field, value) elif value is None: self._setattrVoid(field) elif value_type is _DynamicStructBuilder: diff --git a/capnp/capnp_cpp.pxd b/capnp/capnp_cpp.pxd index 832a14e..aa495df 100644 --- a/capnp/capnp_cpp.pxd +++ b/capnp/capnp_cpp.pxd @@ -6,6 +6,7 @@ from schema_cpp cimport Node, Data, StructNode, EnumNode from libc.stdint cimport * ctypedef unsigned int uint +from libcpp cimport bool as cbool cdef extern from "capnp/common.h" namespace " ::capnp": enum Void: @@ -137,7 +138,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": cppclass Reader: Reader() Reader(Void value) - Reader(bint value) + Reader(cbool value) Reader(char value) Reader(short value) Reader(int value) diff --git a/test/all-types.binary b/test/all-types.binary new file mode 100644 index 0000000000000000000000000000000000000000..ea39763774b2ed570407a3384a8865fbeaa79213 GIT binary patch literal 2816 zcmds&Ur1AN6vxkWS&HkAuIPW8=7SmHpCql9^$#>me6W{5?5dmphIhN$&20o*hW)W9 z1(Oo7dI`i#kfet)ETUCJ!Lqkly#!@FNCf?r+xOhv#rDvHK7{PRxu4(popUbt+ehzpE+zbwD zd;oSN5v{_*aeh>6qNpzG1xcmuYrwF4W#z9b*N4&@#Y9Iaq&J?aPY$Jt8Rm7*^>vQa zG2A91J+kFr_2uB5twod>%+<*6f$eOK{Jb2;&szwxpUHkE`%5(c9&iWj3a|vM1-U+_ z#x9WSdkx)y{ixInK6UAoJ<+Xsk?Y$PlTnT}f4{{d?N*oD+BpBG;kqJUeEzxrBSW$Z zcDvW~w>+L5mh^DI&kX59A1R>ttAu-Gd3Eh4wgQTx_`Popr^+2f`Eur5cmJHiu@LsL z_|vmU-3#gS*AOWR=TB{~Q%9BqqWPij&q;C%nn@G(v9MWsyT9V$2S=HQuitvi3_5R4j9o4opuy1t7<-%*0nF?ah^uhl!X%niZ4f?|@BY|1 zjLsv^nZqtOHG>|n&xbK1ceAXXwj9s%$2S-3_n4?*;H53<=6{-L1 kh0bLw-#9^p=Q5G%;kjJQ^~6slQH;&)9&2MNS(d8TKa`mRLjV8( literal 0 HcmV?d00001 diff --git a/test/all-types.capnp b/test/all-types.capnp new file mode 100644 index 0000000..7dd1550 --- /dev/null +++ b/test/all-types.capnp @@ -0,0 +1,52 @@ +# This is copied from test.capnp in the Cap'n Proto C++ code. + +@0xcaa67a26d16950d9; + +enum TestEnum { + foo @0; + bar @1; + baz @2; + qux @3; + quux @4; + corge @5; + grault @6; + garply @7; +} + +struct TestAllTypes { + voidField @0 : Void; + boolField @1 : Bool; + int8Field @2 : Int8; + int16Field @3 : Int16; + int32Field @4 : Int32; + int64Field @5 : Int64; + uInt8Field @6 : UInt8; + uInt16Field @7 : UInt16; + uInt32Field @8 : UInt32; + uInt64Field @9 : UInt64; + float32Field @10 : Float32; + float64Field @11 : Float64; + textField @12 : Text; + dataField @13 : Data; + structField @14 : TestAllTypes; + enumField @15 : TestEnum; + interfaceField @16 : Void; # TODO + + voidList @17 : List(Void); + boolList @18 : List(Bool); + int8List @19 : List(Int8); + int16List @20 : List(Int16); + int32List @21 : List(Int32); + int64List @22 : List(Int64); + uInt8List @23 : List(UInt8); + uInt16List @24 : List(UInt16); + uInt32List @25 : List(UInt32); + uInt64List @26 : List(UInt64); + float32List @27 : List(Float32); + float64List @28 : List(Float64); + textList @29 : List(Text); + dataList @30 : List(Data); + structList @31 : List(TestAllTypes); + enumList @32 : List(TestEnum); + interfaceList @33 : List(Void); # TODO +} diff --git a/test/all-types.packed b/test/all-types.packed new file mode 100644 index 0000000000000000000000000000000000000000..8627833b2caa63e59b0a1c9ed6d317187bb1219f GIT binary patch literal 831 zcmZ9JZAep57{|}q-Al8r>)n}QR(sQZP($37r1oL!4K!2~>`NeabuYeLyxZMwZX>wM zuosI`FewqMFM*f|lJudB711h0VcEAZzXWALBLeNkPG?ghz3_Vu|NnCyc>d>>Hp6dD}wXgz) z-(%Vx*)_j_vo83t{H|l|WKyob*h1lC7Oyk!nw0}%vtK0<>y7cq9ToSUb;g2&yUAupjodtnshGBfGY6eVvF zHU&e1*ct*(u00qs*|r+?t%f6qf=Z~S_yO8id~*ROI&Uc}22cHvB?8Zj$Hh*%;_AVL z#=gN6#V^ujJtGtJi97dS;6&iw^yCfuFqj*eI2My;V>m9Q;_MSwJSD|bIKC=nb|aXf#5H+C(?W!>Erl+De+|{Xy&rx3MDmp8&cg!Q1L6t+4+K*O3mFE=?UI U+Hq)$ literal 0 HcmV?d00001 diff --git a/test/all-types.txt b/test/all-types.txt new file mode 100644 index 0000000..e0910d3 --- /dev/null +++ b/test/all-types.txt @@ -0,0 +1,71 @@ +( boolField = true, + int8Field = -123, + int16Field = -12345, + int32Field = -12345678, + int64Field = -123456789012345, + uInt8Field = 234, + uInt16Field = 45678, + uInt32Field = 3456789012, + uInt64Field = 12345678901234567890, + float32Field = 1234.5, + float64Field = -1.23e47, + textField = "foo", + dataField = "bar", + structField = ( + boolField = true, + int8Field = -12, + int16Field = 3456, + int32Field = -78901234, + int64Field = 56789012345678, + uInt8Field = 90, + uInt16Field = 1234, + uInt32Field = 56789012, + uInt64Field = 345678901234567890, + float32Field = -1.25e-10, + float64Field = 345, + textField = "baz", + dataField = "qux", + structField = ( + textField = "nested", + structField = ( + textField = "really nested" ) ), + enumField = baz, + voidList = [void, void, void], + boolList = [false, true, false, true, true], + int8List = [12, -34, -128, 127], + int16List = [1234, -5678, -32768, 32767], + int32List = [12345678, -90123456, -2147483648, 2147483647], + int64List = [123456789012345, -678901234567890, -9223372036854775808, 9223372036854775807], + uInt8List = [12, 34, 0, 255], + uInt16List = [1234, 5678, 0, 65535], + uInt32List = [12345678, 90123456, 0, 4294967295], + uInt64List = [123456789012345, 678901234567890, 0, 18446744073709551615], + float32List = [0, 1234567, 1e37, -1e37, 1e-37, -1e-37], + float64List = [0, 123456789012345, 1e306, -1e306, 1e-306, -1e-306], + textList = ["quux", "corge", "grault"], + dataList = ["garply", "waldo", "fred"], + structList = [ + ( textField = "x structlist 1" ), + ( textField = "x structlist 2" ), + ( textField = "x structlist 3" ) ], + enumList = [qux, bar, grault] ), + enumField = corge, + voidList = [void, void, void, void, void, void], + boolList = [true, false, false, true], + int8List = [111, -111], + int16List = [11111, -11111], + int32List = [111111111, -111111111], + int64List = [1111111111111111111, -1111111111111111111], + uInt8List = [111, 222], + uInt16List = [33333, 44444], + uInt32List = [3333333333], + uInt64List = [11111111111111111111], + float32List = [5555.5, inf, -inf, nan], + float64List = [7777.75, inf, -inf, nan], + textList = ["plugh", "xyzzy", "thud"], + dataList = ["oops", "exhausted", "rfc3092"], + structList = [ + ( textField = "structlist 1" ), + ( textField = "structlist 2" ), + ( textField = "structlist 3" ) ], + enumList = [foo, garply] ) diff --git a/test/test_regression.py b/test/test_regression.py index e5d3245..8db871b 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -1,6 +1,7 @@ import pytest import capnp import os +import math this_dir = os.path.dirname(__file__) @@ -65,3 +66,226 @@ def test_addressbook(addressbook): f = open('example', 'r') printAddressBook(f.fileno()) + +@pytest.fixture +def all_types(): + return capnp.load(os.path.join(this_dir, 'all-types.capnp')) + +# TODO: These tests should be extended to: +# - Read each field in Python and assert that it is equal to the expected value. +# - Build an identical message using Python code and compare it to the golden. +# + +def init_all_types(builder): + builder.voidField = None + builder.boolField = True + builder.int8Field = -123 + builder.int16Field = -12345 + builder.int32Field = -12345678 + builder.int64Field = -123456789012345 + builder.uInt8Field = 234 + builder.uInt16Field = 45678 + builder.uInt32Field = 3456789012 + builder.uInt64Field = 12345678901234567890 + builder.float32Field = 1234.5 + builder.float64Field = -123e45 + builder.textField = "foo" + builder.dataField = "bar" + + subBuilder = builder.structField + subBuilder.voidField = None + subBuilder.boolField = True + subBuilder.int8Field = -12 + subBuilder.int16Field = 3456 + subBuilder.int32Field = -78901234 + subBuilder.int64Field = 56789012345678 + subBuilder.uInt8Field = 90 + subBuilder.uInt16Field = 1234 + subBuilder.uInt32Field = 56789012 + subBuilder.uInt64Field = 345678901234567890 + subBuilder.float32Field = -1.25e-10 + subBuilder.float64Field = 345 + subBuilder.textField = "baz" + subBuilder.dataField = "qux" + subSubBuilder = subBuilder.structField + subSubBuilder.textField = "nested" + subSubBuilder.structField.textField = "really nested" + subBuilder.enumField = "baz" + + subBuilder.voidList = [None, None, None] + subBuilder.boolList = [False, True, False, True, True] + subBuilder.int8List = [12, -34, -0x80, 0x7f] + subBuilder.int16List = [1234, -5678, -0x8000, 0x7fff] + subBuilder.int32List = [12345678, -90123456, -0x80000000, 0x7fffffff] + subBuilder.int64List = [123456789012345, -678901234567890, -0x8000000000000000, 0x7fffffffffffffff] + subBuilder.uInt8List = [12, 34, 0, 0xff] + subBuilder.uInt16List = [1234, 5678, 0, 0xffff] + subBuilder.uInt32List = [12345678, 90123456, 0, 0xffffffff] + subBuilder.uInt64List = [123456789012345, 678901234567890, 0, 0xffffffffffffffff] + subBuilder.float32List = [0, 1234567, 1e37, -1e37, 1e-37, -1e-37] + subBuilder.float64List = [0, 123456789012345, 1e306, -1e306, 1e-306, -1e-306] + subBuilder.textList = ["quux", "corge", "grault"] + subBuilder.dataList = ["garply", "waldo", "fred"] + listBuilder = subBuilder.init('structList', 3) + listBuilder[0].textField = "x structlist 1" + listBuilder[1].textField = "x structlist 2" + listBuilder[2].textField = "x structlist 3" + subBuilder.enumList = ["qux", "bar", "grault"] + + builder.enumField = "corge" + + builder.init("voidList", 6) + builder.boolList = [True, False, False, True] + builder.int8List = [111, -111] + builder.int16List = [11111, -11111] + builder.int32List = [111111111, -111111111] + builder.int64List = [1111111111111111111, -1111111111111111111] + builder.uInt8List = [111, 222] + builder.uInt16List = [33333, 44444] + builder.uInt32List = [3333333333] + builder.uInt64List = [11111111111111111111] + builder.float32List = [5555.5, float("inf"), float("-inf"), float("nan")] + builder.float64List = [7777.75, float("inf"), float("-inf"), float("nan")] + builder.textList = ["plugh", "xyzzy", "thud"] + builder.dataList = ["oops", "exhausted", "rfc3092"] + listBuilder = builder.init('structList', 3) + listBuilder[0].textField = "structlist 1" + listBuilder[1].textField = "structlist 2" + listBuilder[2].textField = "structlist 3" + builder.enumList = ["foo", "garply"] + +def assert_almost(float1, float2): + if float1 != float2: + assert abs((float1 - float2) / float1) < 0.00001 + +def check_list(reader, expected): + assert len(reader) == len(expected) + for (i, v) in enumerate(expected): + if type(v) is float: + assert_almost(reader[i], v) + else: + assert reader[i] == v + +def check_all_types(reader): + assert reader.voidField == None + assert reader.boolField == True + assert reader.int8Field == -123 + assert reader.int16Field == -12345 + assert reader.int32Field == -12345678 + assert reader.int64Field == -123456789012345 + assert reader.uInt8Field == 234 + assert reader.uInt16Field == 45678 + assert reader.uInt32Field == 3456789012 + assert reader.uInt64Field == 12345678901234567890 + assert reader.float32Field == 1234.5 + assert_almost(reader.float64Field, -123e45) + assert reader.textField == "foo" + assert reader.dataField == "bar" + + subReader = reader.structField + assert subReader.voidField == None + assert subReader.boolField == True + assert subReader.int8Field == -12 + assert subReader.int16Field == 3456 + assert subReader.int32Field == -78901234 + assert subReader.int64Field == 56789012345678 + assert subReader.uInt8Field == 90 + assert subReader.uInt16Field == 1234 + assert subReader.uInt32Field == 56789012 + assert subReader.uInt64Field == 345678901234567890 + assert_almost(subReader.float32Field, -1.25e-10) + assert subReader.float64Field == 345 + assert subReader.textField == "baz" + assert subReader.dataField == "qux" + + subSubReader = subReader.structField + assert subSubReader.textField == "nested" + assert subSubReader.structField.textField == "really nested" + + assert subReader.enumField == "baz" + + check_list(subReader.voidList, [None, None, None]) + check_list(subReader.boolList, [False, True, False, True, True]) + check_list(subReader.int8List, [12, -34, -0x80, 0x7f]) + check_list(subReader.int16List, [1234, -5678, -0x8000, 0x7fff]) + check_list(subReader.int32List, [12345678, -90123456, -0x80000000, 0x7fffffff]) + check_list(subReader.int64List, [123456789012345, -678901234567890, -0x8000000000000000, 0x7fffffffffffffff]) + check_list(subReader.uInt8List, [12, 34, 0, 0xff]) + check_list(subReader.uInt16List, [1234, 5678, 0, 0xffff]) + check_list(subReader.uInt32List, [12345678, 90123456, 0, 0xffffffff]) + check_list(subReader.uInt64List, [123456789012345, 678901234567890, 0, 0xffffffffffffffff]) + check_list(subReader.float32List, [0.0, 1234567.0, 1e37, -1e37, 1e-37, -1e-37]) + check_list(subReader.float64List, [0.0, 123456789012345.0, 1e306, -1e306, 1e-306, -1e-306]) + check_list(subReader.textList, ["quux", "corge", "grault"]) + check_list(subReader.dataList, ["garply", "waldo", "fred"]) + + listReader = subReader.structList + assert len(listReader) == 3 + assert listReader[0].textField == "x structlist 1" + assert listReader[1].textField == "x structlist 2" + assert listReader[2].textField == "x structlist 3" + + check_list(subReader.enumList, ["qux", "bar", "grault"]) + + assert reader.enumField == "corge" + + assert len(reader.voidList) == 6 + check_list(reader.boolList, [True, False, False, True]) + check_list(reader.int8List, [111, -111]) + check_list(reader.int16List, [11111, -11111]) + check_list(reader.int32List, [111111111, -111111111]) + check_list(reader.int64List, [1111111111111111111, -1111111111111111111]) + check_list(reader.uInt8List, [111, 222]) + check_list(reader.uInt16List, [33333, 44444]) + check_list(reader.uInt32List, [3333333333]) + check_list(reader.uInt64List, [11111111111111111111]) + + listReader = reader.float32List + assert len(listReader) == 4 + assert listReader[0] == 5555.5 + assert listReader[1] == float("inf") + assert listReader[2] == -float("inf") + assert math.isnan(listReader[3]) + + listReader = reader.float64List + len(listReader) == 4 + assert listReader[0] == 7777.75 + assert listReader[1] == float("inf") + assert listReader[2] == -float("inf") + assert math.isnan(listReader[3]) + + check_list(reader.textList, ["plugh", "xyzzy", "thud"]) + check_list(reader.dataList, ["oops", "exhausted", "rfc3092"]) + + listReader = reader.structList + len(listReader) == 3 + assert listReader[0].textField == "structlist 1" + assert listReader[1].textField == "structlist 2" + assert listReader[2].textField == "structlist 3" + + check_list(reader.enumList, ["foo", "garply"]) + +def test_build(all_types): + builder = capnp.MallocMessageBuilder() + root = builder.getRoot(all_types.TestAllTypes) + init_all_types(root) + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + assert str(root) + '\n' == expectedText + +def test_binary_read(all_types): + f = open(os.path.join(this_dir, 'all-types.binary'), 'r') + message = capnp.StreamFdMessageReader(f.fileno()) + root = message.getRoot(all_types.TestAllTypes) + check_all_types(root) + + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + assert str(root) + '\n' == expectedText + +def test_packed_read(all_types): + f = open(os.path.join(this_dir, 'all-types.packed'), 'r') + message = capnp.PackedFdMessageReader(f.fileno()) + root = message.getRoot(all_types.TestAllTypes) + check_all_types(root) + + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + assert str(root) + '\n' == expectedText From 86b742e91dcd4b86c319800d0422a76ad6a3d818 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 31 Aug 2013 17:07:44 -0700 Subject: [PATCH 4/6] Wrap MessageBuilder::setRoot(). --- capnp/capnp.pyx | 17 +++++++++++++++++ capnp/schema_cpp.pxd | 1 + test/test_regression.py | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index 9c19c73..827a213 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -32,6 +32,10 @@ ctypedef double Float64 from libc.stdlib cimport malloc, free from libcpp cimport bool as cbool +ctypedef fused _DynamicStructReaderOrBuilder: + _DynamicStructReader + _DynamicStructBuilder + def _make_enum(enum_name, *sequential, **named): enums = dict(zip(sequential, range(len(sequential))), **named) reverse = dict((value, key) for key, value in enums.iteritems()) @@ -971,6 +975,19 @@ cdef class MessageBuilder: else: s = schema return _DynamicStructBuilder()._init(self.thisptr.getRootDynamicStruct(s.thisptr), self) + + cpdef setRoot(self, value): + """A method for instantiating Cap'n Proto structs by copying from an existing struct + + :type value: :class:`_DynamicStructReader` + :param value: A Cap'n Proto struct value to copy + + :rtype: void + """ + + if type(value) is _DynamicStructBuilder: + value = value.asReader(); + self.thisptr.setRootDynamicStruct((<_DynamicStructReader>value).thisptr) cpdef newOrphan(self, schema): """A method for instantiating Cap'n Proto orphans diff --git a/capnp/schema_cpp.pxd b/capnp/schema_cpp.pxd index f9a56b8..c8dcc7b 100644 --- a/capnp/schema_cpp.pxd +++ b/capnp/schema_cpp.pxd @@ -664,6 +664,7 @@ cdef extern from "capnp/message.h" namespace " ::capnp": DynamicStruct.Builder getRootDynamicStruct'getRoot< ::capnp::DynamicStruct>'(StructSchema) DynamicStruct.Builder initRootDynamicStruct'initRoot< ::capnp::DynamicStruct>'(StructSchema) + void setRootDynamicStruct'setRoot< ::capnp::DynamicStruct::Reader>'(DynamicStruct.Reader) DynamicOrphan newOrphan'getOrphanage().newOrphan'(StructSchema) diff --git a/test/test_regression.py b/test/test_regression.py index 8db871b..f19683e 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -281,6 +281,15 @@ def test_binary_read(all_types): expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() assert str(root) + '\n' == expectedText + # Test setRoot(). + builder = capnp.MallocMessageBuilder() + builder.setRoot(root) + check_all_types(builder.getRoot(all_types.TestAllTypes)) + + builder2 = capnp.MallocMessageBuilder() + builder2.setRoot(builder.getRoot(all_types.TestAllTypes)) + check_all_types(builder2.getRoot(all_types.TestAllTypes)) + def test_packed_read(all_types): f = open(os.path.join(this_dir, 'all-types.packed'), 'r') message = capnp.PackedFdMessageReader(f.fileno()) From 6731d7eb7d55789cfed225b515f19438a85fdad7 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 31 Aug 2013 17:22:11 -0700 Subject: [PATCH 5/6] Store Builders by value rather than allocate them separately on the heap (matches treatment of Readers). v0.3 fixes the bug that made this not work. --- capnp/capnp.pyx | 24 +++++++++--------------- capnp/capnp_cpp.pxd | 3 +++ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index 827a213..8ef44ca 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -241,18 +241,15 @@ cdef class _DynamicListBuilder: for phone in phones: print phone.number """ - cdef C_DynamicList.Builder * thisptr + cdef C_DynamicList.Builder thisptr cdef public object _parent cdef _init(self, C_DynamicList.Builder other, object parent): - self.thisptr = new C_DynamicList.Builder(other) + self.thisptr = other self._parent = parent return self - def __dealloc__(self): - del self.thisptr - cdef _get(self, index) except +ValueError: - return toPython(deref(self.thisptr)[index], self._parent) + return toPython(self.thisptr[index], self._parent) def __getitem__(self, index): size = self.thisptr.size() @@ -377,11 +374,11 @@ cdef class _DynamicListBuilder: return _DynamicOrphan()._init(self.thisptr.disown(index), self._parent) def __str__(self): - return printListBuilder(deref(self.thisptr)).flatten().cStr() + return printListBuilder(self.thisptr).flatten().cStr() def __repr__(self): # TODO: Print the list type. - return '' % strListBuilder(deref(self.thisptr)).cStr() + return '' % strListBuilder(self.thisptr).cStr() cdef class _List_NestedNode_Reader: cdef List[C_Node.NestedNode].Reader thisptr @@ -526,16 +523,13 @@ cdef class _DynamicStructBuilder: setattr(person, 'field-with-hyphens', 'foo') # for names that are invalid for python, use setattr print getattr(person, 'field-with-hyphens') # for names that are invalid for python, use getattr """ - cdef C_DynamicStruct.Builder * thisptr + cdef C_DynamicStruct.Builder thisptr cdef public object _parent cdef _init(self, C_DynamicStruct.Builder other, object parent): - self.thisptr = new C_DynamicStruct.Builder(other) + self.thisptr = other self._parent = parent return self - def __dealloc__(self): - del self.thisptr - cdef _get(self, field) except +ValueError: return toPython(self.thisptr.get(field), self._parent) @@ -731,10 +725,10 @@ cdef class _DynamicStructBuilder: return list(self.schema.fieldnames) def __str__(self): - return printStructBuilder(deref(self.thisptr)).flatten().cStr() + return printStructBuilder(self.thisptr).flatten().cStr() def __repr__(self): - return '<%s builder %s>' % (self.schema.node.displayName, strStructBuilder(deref(self.thisptr)).cStr()) + return '<%s builder %s>' % (self.schema.node.displayName, strStructBuilder(self.thisptr).cStr()) cdef class _DynamicOrphan: cdef C_DynamicOrphan thisptr diff --git a/capnp/capnp_cpp.pxd b/capnp/capnp_cpp.pxd index aa495df..f00f2c6 100644 --- a/capnp/capnp_cpp.pxd +++ b/capnp/capnp_cpp.pxd @@ -99,6 +99,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": StructSchema getSchema() Maybe[StructSchema.Field] which() cppclass Builder: + Builder() Builder(Builder &) DynamicValueForward.Builder get(char *) bint has(char *) except +ValueError @@ -125,6 +126,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": DynamicValueForward.Reader operator[](uint) except +ValueError uint size() cppclass Builder: + Builder() Builder(Builder &) DynamicValueForward.Builder operator[](uint) uint size() @@ -167,6 +169,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": Data.Reader asData"as< ::capnp::Data>"() cppclass Builder: + Builder() Type getType() int64_t asInt"as"() uint64_t asUint"as"() From 1cfea9c846e2566657cd302e4af0e9432f880a6a Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 31 Aug 2013 18:19:02 -0700 Subject: [PATCH 6/6] Add shortcuts for reading from / writing to files. In Python, it doesn't make much sense to force people to muck around with MessageReaders and MessageBuilders since everything is landing on the heap anyway. Instead, let's make it easy: MyType.read[Packed]From(file) reads a file and returns a MyType reader. MyType.newMessage() returns a MyType builder representing the root of a new message. You can call this builder's write[Packed]To(file) method to write it to a file. --- capnp/capnp.pyx | 54 ++++++++++++++++++++++++++++++++++++++--- test/test_regression.py | 23 +++++++++++++----- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index 8ef44ca..12d10b9 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -525,10 +525,46 @@ cdef class _DynamicStructBuilder: """ cdef C_DynamicStruct.Builder thisptr cdef public object _parent - cdef _init(self, C_DynamicStruct.Builder other, object parent): + cdef bint _isRoot + cdef _init(self, C_DynamicStruct.Builder other, object parent, bint isRoot = False): self.thisptr = other self._parent = parent + self._isRoot = isRoot return self + + def writeTo(self, file): + """Writes the struct's containing message to the given file object in unpacked binary format. + + This is a shortcut for calling capnp.writeMessageToFd(). This can only be called on the + message's root struct. + + :type file: file + :param file: A file or socket object (or anything with a fileno() method), open for write. + + :rtype: void + + :Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct. + """ + if not self._isRoot: + raise ValueError("You can only call writeTo() on the message's root struct.") + writeMessageToFd(file.fileno(), self._parent) + + def writePackedTo(self, file): + """Writes the struct's containing message to the given file object in packed binary format. + + This is a shortcut for calling capnp.writePackedMessageToFd(). This can only be called on + the message's root struct. + + :type file: file + :param file: A file or socket object (or anything with a fileno() method), open for write. + + :rtype: void + + :Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct. + """ + if not self._isRoot: + raise ValueError("You can only call writeTo() on the message's root struct.") + writePackedMessageToFd(file.fileno(), self._parent) cdef _get(self, field) except +ValueError: return toPython(self.thisptr.get(field), self._parent) @@ -892,6 +928,18 @@ cdef class SchemaParser: proto = schema.getProto() if proto.isStruct: local_module.schema = schema.asStruct() + def readFrom(file): + reader = StreamFdMessageReader(file.fileno()) + return reader.getRoot(local_module) + def readPackedFrom(file): + reader = PackedFdMessageReader(file.fileno()) + return reader.getRoot(local_module) + def newMessage(): + builder = MallocMessageBuilder() + return builder.initRoot(local_module) + local_module.readFrom = readFrom + local_module.readPackedFrom = readPackedFrom + local_module.newMessage = newMessage elif proto.isConst: module.__dict__[node.name] = schema.asConstValue() @@ -943,7 +991,7 @@ cdef class MessageBuilder: s = schema.schema else: s = schema - return _DynamicStructBuilder()._init(self.thisptr.initRootDynamicStruct(s.thisptr), self) + return _DynamicStructBuilder()._init(self.thisptr.initRootDynamicStruct(s.thisptr), self, True) cpdef getRoot(self, schema): """A method for instantiating Cap'n Proto structs, from an already pre-written buffer @@ -968,7 +1016,7 @@ cdef class MessageBuilder: s = schema.schema else: s = schema - return _DynamicStructBuilder()._init(self.thisptr.getRootDynamicStruct(s.thisptr), self) + return _DynamicStructBuilder()._init(self.thisptr.getRootDynamicStruct(s.thisptr), self, True) cpdef setRoot(self, value): """A method for instantiating Cap'n Proto structs by copying from an existing struct diff --git a/test/test_regression.py b/test/test_regression.py index f19683e..1a896ad 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -266,16 +266,14 @@ def check_all_types(reader): check_list(reader.enumList, ["foo", "garply"]) def test_build(all_types): - builder = capnp.MallocMessageBuilder() - root = builder.getRoot(all_types.TestAllTypes) + root = all_types.TestAllTypes.newMessage() init_all_types(root) expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() assert str(root) + '\n' == expectedText def test_binary_read(all_types): f = open(os.path.join(this_dir, 'all-types.binary'), 'r') - message = capnp.StreamFdMessageReader(f.fileno()) - root = message.getRoot(all_types.TestAllTypes) + root = all_types.TestAllTypes.readFrom(f) check_all_types(root) expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() @@ -292,9 +290,22 @@ def test_binary_read(all_types): def test_packed_read(all_types): f = open(os.path.join(this_dir, 'all-types.packed'), 'r') - message = capnp.PackedFdMessageReader(f.fileno()) - root = message.getRoot(all_types.TestAllTypes) + root = all_types.TestAllTypes.readPackedFrom(f) check_all_types(root) expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() assert str(root) + '\n' == expectedText + +def test_binary_write(all_types): + root = all_types.TestAllTypes.newMessage() + init_all_types(root) + root.writeTo(open('example', 'w')) + + check_all_types(all_types.TestAllTypes.readFrom(open('example', 'r'))) + +def test_packed_write(all_types): + root = all_types.TestAllTypes.newMessage() + init_all_types(root) + root.writePackedTo(open('example', 'w')) + + check_all_types(all_types.TestAllTypes.readPackedFrom(open('example', 'r')))