IQ.Lvbs/pycapnp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add {_get,_set,_has,_init}_by_field methods for faster field access

Browse Source
This commit is contained in:
Jason Paryani
2014-09-03 15:20:26 -07:00
parent a0143260d6
commit ea1be42252
4 changed files with 180 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
capnp/includes/capnp_cpp.pxd
View File

@@ -43,7 +43,7 @@ cdef extern from "kj/exception.h" namespace " ::kj":
int getDurability()
StringPtr getDescription()
cdef extern from "kj/memory.h" namespace " ::kj":
cdef extern from "kj/memory.h" namespace " ::kj":
cdef cppclass Own[T]:
T& operator*()
Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side)
@@ -225,7 +225,9 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
cdef cppclass DynamicStruct:
cppclass Reader:
DynamicValueForward.Reader get(char *) except +reraise_kj_exception
DynamicValueForward.Reader getByField"get"(StructSchema.Field) except +reraise_kj_exception
bint has(char *) except +reraise_kj_exception
bint hasByField"has"(StructSchema.Field) except +reraise_kj_exception
StructSchema getSchema()
Maybe[StructSchema.Field] which()
MessageSize totalSize()
@@ -239,10 +241,15 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
DynamicStruct_Builder()
DynamicStruct_Builder(DynamicStruct_Builder &)
DynamicValueForward.Builder get(char *) except +reraise_kj_exception
DynamicValueForward.Builder getByField"get"(StructSchema.Field) except +reraise_kj_exception
bint has(char *) except +reraise_kj_exception
bint hasByField"has"(StructSchema.Field) except +reraise_kj_exception
void set(char *, DynamicValueForward.Reader) except +reraise_kj_exception
void setByField"set"(StructSchema.Field, DynamicValueForward.Reader) except +reraise_kj_exception
DynamicValueForward.Builder init(char *, uint size) except +reraise_kj_exception
DynamicValueForward.Builder init(char *) except +reraise_kj_exception
DynamicValueForward.Builder initByField"init"(StructSchema.Field, uint size) except +reraise_kj_exception
DynamicValueForward.Builder initByField"init"(StructSchema.Field) except +reraise_kj_exception
StructSchema getSchema()
Maybe[StructSchema.Field] which()
void adopt(char *, DynamicOrphan) except +reraise_kj_exception

137
capnp/lib/capnp.pyx
View File

@@ -101,7 +101,7 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_
setattr(results, arg_name, arg_val)
return NULL
cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_DynamicObject.Reader & _reader) except * with gil:
restorer = <object>_restorer
reader = _DynamicObjectReader()._init(_reader, None)
@@ -150,13 +150,13 @@ def _make_enum(enum_name, *sequential, **named):
enums['reverse_mapping'] = reverse
return type(enum_name, (), enums)
_Nature = _make_enum('_Nature',
_Nature = _make_enum('_Nature',
PRECONDITION = 0,
LOCAL_BUG = 1,
OS_ERROR = 2,
NETWORK_FAILURE = 3,
OTHER = 4)
_Durability = _make_enum('_Durability',
_Durability = _make_enum('_Durability',
PERMANENT = 0,
TEMPORARY = 1,
OVERLOADED = 2)
@@ -208,7 +208,7 @@ class KjException(Exception):
self.message = message
self.nature = nature
self.durability = durability
@property
def file(self):
return self.wrapper.file
@@ -247,7 +247,7 @@ cdef public object wrap_kj_exception(capnp.Exception & exception) with gil:
cdef public object wrap_kj_exception_for_reraise(capnp.Exception & exception) with gil:
wrapper = _KjExceptionWrapper()._init(exception)
wrapper_msg = str(wrapper)
nature = wrapper.nature
if wrapper.nature == 'PRECONDITION':
@@ -415,7 +415,7 @@ cdef class _DynamicResizableListBuilder:
person = addressbook.Person.new_message()
phones = person.init_resizable_list('phones') # This returns a _DynamicResizableListBuilder
phone = phones.add()
phone.number = 'foo'
phone = phones.add()
@@ -447,7 +447,7 @@ cdef class _DynamicResizableListBuilder:
orphan_val = orphan.get()
self._list.append((orphan, orphan_val))
return orphan_val
def __getitem__(self, index):
return self._list[index][1]
@@ -477,7 +477,7 @@ cdef class _DynamicListBuilder:
person = addressbook.Person.new_message()
phones = person.init('phones', 2) # This returns a _DynamicListBuilder
phone = phones[0]
phone.number = 'foo'
phone = phones[1]
@@ -872,9 +872,15 @@ cdef class _DynamicStructReader:
def __getattr__(self, field):
return to_python_reader(self.thisptr.get(field), self._parent)
def _get_by_field(self, _StructSchemaField field):
return to_python_reader(self.thisptr.getByField(field.thisptr), self._parent)
def _has(self, field):
return self.thisptr.has(field)
def _has_by_field(self, _StructSchemaField field):
return self.thisptr.hasByField(field.thisptr)
cpdef _which(self):
"""Returns the enum corresponding to the union in this struct
@@ -945,7 +951,7 @@ cdef class _DynamicStructBuilder:
This class is almost a 1 for 1 wrapping of the Cap'n Proto C++ DynamicStruct::Builder. The only difference is that instead of a `get`/`set` method, __getattr__/__setattr__ is overloaded and the field name is passed onto the C++ equivalent function. This means you just use . syntax to access or set any field. For field names that don't follow valid python naming convention for fields, use the global functions :py:func:`getattr`/:py:func:`setattr`::
person = addressbook.Person.new_message() # This returns a _DynamicStructBuilder
person.name = 'foo' # using . syntax
print person.name # using . syntax
@@ -971,15 +977,15 @@ cdef class _DynamicStructBuilder:
def write(self, file):
"""Writes the struct's containing message to the given file object in unpacked binary format.
This is a shortcut for calling capnp._write_message_to_fd(). This can only be called on the
message's root struct.
:type file: file
:param file: A file or socket object (or anything with a fileno() method), open for write.
:rtype: void
:Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct.
"""
self._check_write()
@@ -988,15 +994,15 @@ cdef class _DynamicStructBuilder:
def write_packed(self, file):
"""Writes the struct's containing message to the given file object in packed binary format.
This is a shortcut for calling capnp._write_packed_message_to_fd(). This can only be called on
the message's root struct.
:type file: file
:param file: A file or socket object (or anything with a fileno() method), open for write.
:rtype: void
:Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct.
"""
self._check_write()
@@ -1047,23 +1053,33 @@ cdef class _DynamicStructBuilder:
cdef C_DynamicValue.Builder value = self.thisptr.get(field)
return to_python_builder(value, self._parent)
def _get_by_field(self, _StructSchemaField field):
return to_python_builder(self.thisptr.getByField(field.thisptr), self._parent)
def __getattr__(self, field):
return self._get(field)
cdef _set(self, field, value):
_setDynamicField(self.thisptr, field, value, self._parent)
def _set_by_field(self, _StructSchemaField field, value):
# TODO: make this faster
_setDynamicField(self.thisptr, field.proto.name, value, self._parent)
def __setattr__(self, field, value):
self._set(field, value)
def _has(self, field):
return self.thisptr.has(field)
def _has_by_field(self, _StructSchemaField field):
return self.thisptr.hasByField(field.thisptr)
cpdef init(self, field, size=None):
"""Method for initializing fields that are of type union/struct/list
Typically, you don't have to worry about initializing structs/unions, so this method is mainly for lists.
Typically, you don't have to worry about initializing structs/unions, so this method is mainly for lists.
:type field: str
:param field: The field name to initialize
@@ -1080,10 +1096,30 @@ cdef class _DynamicStructBuilder:
else:
return to_python_builder(self.thisptr.init(field, size), self._parent)
def _init_by_field(self, _StructSchemaField field, size=None):
"""Method for initializing fields that are of type union/struct/list
Typically, you don't have to worry about initializing structs/unions, so this method is mainly for lists.
:type field: str
:param field: The field name to initialize
:type size: int
:param size: The size of the list to initiialize. This should be None for struct/union initialization.
:rtype: :class:`_DynamicStructBuilder` or :class:`_DynamicListBuilder`
:Raises: :exc:`exceptions.ValueError` if the field isn't in this struct
"""
if size is None:
return to_python_builder(self.thisptr.initByField(field.thisptr), self._parent)
else:
return to_python_builder(self.thisptr.initByField(field.thisptr, size), self._parent)
cpdef init_resizable_list(self, field):
"""Method for initializing fields that are of type list (of structs)
This version of init returns a :class:`_DynamicResizableListBuilder` that allows you to add members one at a time (ie. if you don't know the size for sure). This is only meant for lists of Cap'n Proto objects, since for primitive types you can just define a normal python list and fill it yourself.
This version of init returns a :class:`_DynamicResizableListBuilder` that allows you to add members one at a time (ie. if you don't know the size for sure). This is only meant for lists of Cap'n Proto objects, since for primitive types you can just define a normal python list and fill it yourself.
.. warning:: You need to call :meth:`_DynamicResizableListBuilder.finish` on the list object before serializing the Cap'n Proto message. Failure to do so will cause your objects not to be written out as well as leaking orphan structs into your message.
@@ -1583,7 +1619,7 @@ cdef class _VoidPromise:
cpdef cancel(self, numParents=1) except +reraise_kj_exception:
if numParents > 0 and hasattr(self._parent, 'cancel'):
self._parent.cancel(numParents - 1)
self.is_consumed = True
del self.thisptr
self.thisptr = NULL
@@ -1670,7 +1706,7 @@ cdef class _RemotePromise:
cpdef cancel(self, numParents=1) except +reraise_kj_exception:
if numParents > 0 and hasattr(self._parent, 'cancel'):
self._parent.cancel(numParents - 1)
self.is_consumed = True
del self.thisptr
self.thisptr = NULL
@@ -1962,7 +1998,7 @@ cdef class TwoPartyClient:
return sock
cpdef restore(self, objectId) except +reraise_kj_exception:
cdef _MessageBuilder builder
cdef _MessageBuilder builder
cdef _MessageReader reader
cdef _DynamicObjectBuilder object_builder
cdef _DynamicObjectReader object_reader
@@ -2129,12 +2165,13 @@ cdef class _Schema:
cdef class _StructSchema:
cdef C_StructSchema thisptr
cdef object __fieldnames, __union_fields, __non_union_fields
cdef object __fieldnames, __union_fields, __non_union_fields, __fields
cdef _init(self, C_StructSchema other):
self.thisptr = other
self.__fieldnames = None
self.__union_fields = None
self.__non_union_fields = None
self.__fields = None
return self
property fieldnames:
@@ -2170,6 +2207,17 @@ cdef class _StructSchema:
for i in xrange(nfields))
return self.__non_union_fields
property fields:
"""A tuple of the field names in the struct."""
def __get__(self):
if self.__fields is not None:
return self.__fields
fieldlist = self.thisptr.getFields()
nfields = fieldlist.size()
self.__fields = {<char*>fieldlist[i].getProto().getName().cStr() : _StructSchemaField()._init(fieldlist[i], self)
for i in xrange(nfields)}
return self.__fields
property node:
"""The raw schema node"""
def __get__(self):
@@ -2315,13 +2363,14 @@ class _StructModule(object):
for field in schema.node.struct.fields:
if field.which() == 'group':
name = field.name.capitalize()
union_schema = schema.get_dependency(field.group.typeId).node.struct
raw_schema = schema.get_dependency(field.group.typeId)
union_schema = raw_schema.node.struct
if union_schema.discriminantCount == 0:
continue
union_module = _StructModuleWhich()
setattr(union_module, 'schema', union_schema)
setattr(union_module, 'schema', raw_schema.as_struct())
for union_field in union_schema.fields:
setattr(union_module, union_field.name, union_field.discriminantValue)
setattr(self, name, union_module)
@@ -2331,7 +2380,7 @@ class _StructModule(object):
:type file: file
:param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
:type traversal_limit_in_words: int
:param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
@@ -2346,7 +2395,7 @@ class _StructModule(object):
:type file: file
:param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
:type traversal_limit_in_words: int
:param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
@@ -2361,7 +2410,7 @@ class _StructModule(object):
:type file: file
:param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
:type traversal_limit_in_words: int
:param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
@@ -2376,7 +2425,7 @@ class _StructModule(object):
:type file: file
:param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
:type traversal_limit_in_words: int
:param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
@@ -2391,7 +2440,7 @@ class _StructModule(object):
:type buf: buffer
:param buf: Any Python object that supports the buffer interface.
:type traversal_limit_in_words: int
:param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
@@ -2415,7 +2464,7 @@ class _StructModule(object):
:type buf: buffer
:param buf: Any Python object that supports the readable buffer interface.
:type traversal_limit_in_words: int
:param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
@@ -2497,7 +2546,7 @@ cdef class SchemaParser:
return ret
def load(self, file_name, display_name=None, imports=[]):
"""Load a Cap'n Proto schema from a file
"""Load a Cap'n Proto schema from a file
You will have to load a schema before you can begin doing anything
meaningful with this library. Loading a schema is much like loading
@@ -2663,7 +2712,7 @@ cdef class _MessageBuilder:
:return: An AnyPointer that you can set fields in
"""
return _DynamicObjectBuilder()._init(self.thisptr.getRootAnyPointer(), self)
cpdef set_root(self, value) except +reraise_kj_exception:
"""A method for instantiating Cap'n Proto structs by copying from an existing struct
@@ -2819,7 +2868,7 @@ cdef class _PackedMessageReader(_MessageReader):
opts.traversalLimitInWords = traversal_limit_in_words
if nesting_limit is not None:
opts.nestingLimit = nesting_limit
self.thisptr = new schema_cpp.PackedMessageReader(stream, opts)
return self
@@ -2836,13 +2885,13 @@ cdef class _PackedMessageReaderBytes(_MessageReader):
opts.traversalLimitInWords = traversal_limit_in_words
if nesting_limit is not None:
opts.nestingLimit = nesting_limit
cdef const void *ptr
cdef Py_ssize_t sz
PyObject_AsReadBuffer(buf, &ptr, &sz)
self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(<byte *>ptr, sz))
self.thisptr = new schema_cpp.PackedMessageReader(deref(self.stream), opts)
def __dealloc__(self):
@@ -2873,7 +2922,7 @@ cdef class _InputMessageReader(_MessageReader):
opts.traversalLimitInWords = traversal_limit_in_words
if nesting_limit is not None:
opts.nestingLimit = nesting_limit
self.thisptr = new schema_cpp.InputStreamMessageReader(stream, opts)
return self
@@ -2896,7 +2945,7 @@ cdef class _PackedFdMessageReader(_MessageReader):
opts.traversalLimitInWords = traversal_limit_in_words
if nesting_limit is not None:
opts.nestingLimit = nesting_limit
self.thisptr = new schema_cpp.PackedFdMessageReader(fd, opts)
cdef class _MultipleMessageReader:
@@ -2909,7 +2958,7 @@ cdef class _MultipleMessageReader:
self.schema = schema
self.traversal_limit_in_words = traversal_limit_in_words
self.nesting_limit = nesting_limit
self.stream = new schema_cpp.FdInputStream(fd)
self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream))
@@ -2940,7 +2989,7 @@ cdef class _MultiplePackedMessageReader:
self.schema = schema
self.traversal_limit_in_words = traversal_limit_in_words
self.nesting_limit = nesting_limit
self.stream = new schema_cpp.FdInputStream(fd)
self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream))
@@ -2971,7 +3020,7 @@ cdef class _FlatArrayMessageReader(_MessageReader):
opts.traversalLimitInWords = traversal_limit_in_words
if nesting_limit is not None:
opts.nestingLimit = nesting_limit
cdef const void *ptr
cdef Py_ssize_t sz
PyObject_AsReadBuffer(buf, &ptr, &sz)
@@ -3062,7 +3111,7 @@ def _write_packed_message_to_fd(int fd, _MessageBuilder message):
_global_schema_parser = None
def load(file_name, display_name=None, imports=[]):
"""Load a Cap'n Proto schema from a file
"""Load a Cap'n Proto schema from a file
You will have to load a schema before you can begin doing anything
meaningful with this library. Loading a schema is much like loading

71
test/test_regression.py
View File

@@ -156,7 +156,7 @@ def test_addressbook_resizable(addressbook):
bob.employment.unemployed = None
people.finish()
addresses.write(file)
@@ -192,6 +192,75 @@ def test_addressbook_resizable(addressbook):
f = open('example', 'r')
printAddressBook(f)
def test_addressbook_explicit_fields(addressbook):
def writeAddressBook(file):
addresses = addressbook.AddressBook.new_message()
address_fields = addressbook.AddressBook.schema.fields
person_fields = addressbook.Person.schema.fields
phone_fields = addressbook.Person.PhoneNumber.schema.fields
people = addresses._init_by_field(address_fields['people'], 2)
alice = people[0]
alice._set_by_field(person_fields['id'], 123)
alice._set_by_field(person_fields['name'], 'Alice')
alice._set_by_field(person_fields['email'], 'alice@example.com')
alicePhones = alice._init_by_field(person_fields['phones'], 1)
alicePhones[0]._set_by_field(phone_fields['number'], "555-1212")
alicePhones[0]._set_by_field(phone_fields['type'], 'mobile')
employment = alice._get_by_field(person_fields['employment'])
employment._set_by_field(addressbook.Person.Employment.schema.fields['school'], "MIT")
bob = people[1]
bob._set_by_field(person_fields['id'], 456)
bob._set_by_field(person_fields['name'], 'Bob')
bob._set_by_field(person_fields['email'], 'bob@example.com')
bobPhones = bob._init_by_field(person_fields['phones'], 2)
bobPhones[0]._set_by_field(phone_fields['number'], "555-4567")
bobPhones[0]._set_by_field(phone_fields['type'], 'home')
bobPhones[1]._set_by_field(phone_fields['number'], "555-7654")
bobPhones[1]._set_by_field(phone_fields['type'], 'work')
employment = bob._get_by_field(person_fields['employment'])
employment._set_by_field(addressbook.Person.Employment.schema.fields['unemployed'], None)
addresses.write(file)
def printAddressBook(file):
addresses = addressbook.AddressBook.read(file)
address_fields = addressbook.AddressBook.schema.fields
person_fields = addressbook.Person.schema.fields
phone_fields = addressbook.Person.PhoneNumber.schema.fields
people = addresses._get_by_field(address_fields['people'])
alice = people[0]
assert alice._get_by_field(person_fields['id']) == 123
assert alice._get_by_field(person_fields['name']) == 'Alice'
assert alice._get_by_field(person_fields['email']) == 'alice@example.com'
alicePhones = alice._get_by_field(person_fields['phones'])
assert alicePhones[0]._get_by_field(phone_fields['number']) == "555-1212"
assert alicePhones[0]._get_by_field(phone_fields['type']) == 'mobile'
employment = alice._get_by_field(person_fields['employment'])
employment._get_by_field(addressbook.Person.Employment.schema.fields['school']) == "MIT"
bob = people[1]
assert bob._get_by_field(person_fields['id']) == 456
assert bob._get_by_field(person_fields['name']) == 'Bob'
assert bob._get_by_field(person_fields['email']) == 'bob@example.com'
bobPhones = bob._get_by_field(person_fields['phones'])
assert bobPhones[0]._get_by_field(phone_fields['number']) == "555-4567"
assert bobPhones[0]._get_by_field(phone_fields['type']) == 'home'
assert bobPhones[1]._get_by_field(phone_fields['number']) == "555-7654"
assert bobPhones[1]._get_by_field(phone_fields['type']) == 'work'
employment = bob._get_by_field(person_fields['employment'])
employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) == None
f = open('example', 'w')
writeAddressBook(f)
f = open('example', 'r')
printAddressBook(f)
@pytest.fixture
def all_types():

9
test/test_struct.py
View File

@@ -187,3 +187,12 @@ def test_to_dict_enum(addressbook):
field = person.to_dict()['phones'][0]['type']
assert isstr(field)
assert field == 'mobile'
def test_explicit_field(addressbook):
person = addressbook.Person.new_message(**{'name': 'Test'})
name_field = addressbook.Person.schema.fields['name']
assert person.name == person._get_by_field(name_field)
assert person.name == person.as_reader()._get_by_field(name_field)