Simplify server interface

capnp/capnp.pyx

@@ -43,6 +43,7 @@ import imp as _imp
 from functools import partial as _partial
 import warnings as _warnings
 import inspect as _inspect
+from operator import attrgetter as _attrgetter
 
 # By making it public, we'll be able to call it from capabilityHelper.h
 cdef public object wrap_dynamic_struct_reader(C_DynamicStruct.Reader & reader):
@@ -55,23 +56,55 @@ cdef public void wrap_remote_call(PyObject * func, Response & r) except *:
     # TODO: decref func?
     func_obj(response)
 
+cdef _find_field_order(struct_node):
+    return [f.name for f in sorted(struct_node.fields, key=_attrgetter('codeOrder'))]
+
 cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_name, CallContext & _context) except *:
     server = <object>_server
     method_name = <object>_method_name
 
     context = _CallContext()._init(_context)
-    func = getattr(server, method_name)
+    func = getattr(server, method_name+'_context', None)
-    ret = func(context)
+    if func is not None:
+        ret = func(context)
+        if ret is not None:
+            if type(ret) is _VoidPromise:
+                return new VoidPromise(moveVoidPromise(deref((<_VoidPromise>ret).thisptr)))
+            else:
+                try:
+                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise: return = %s' % (method_name, str(ret))
+                except:
+                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise' % (method_name)
+                _warnings.warn_explicit(warning_msg, UserWarning, _inspect.getsourcefile(func), _inspect.getsourcelines(func)[1])
 
-    if ret is not None:
+        if ret is not None:
-        if type(ret) is _VoidPromise:
+            if type(ret) is _VoidPromise:
-            return new VoidPromise(moveVoidPromise(deref((<_VoidPromise>ret).thisptr)))
+                return new VoidPromise(moveVoidPromise(deref((<_VoidPromise>ret).thisptr)))
-        else:
+            else:
-            try:
+                try:
-                warning_msg = 'Server function (%s) returned a value that was not a VoidPromise: return = %s' % (method_name, str(ret))
+                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise: return = %s' % (method_name, str(ret))
-            except:
+                except:
-                warning_msg = 'Server function (%s) returned a value that was not a VoidPromise' % (method_name)
+                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise' % (method_name)
-            _warnings.warn_explicit(warning_msg, UserWarning, _inspect.getsourcefile(func), _inspect.getsourcelines(func)[1])
+                _warnings.warn_explicit(warning_msg, UserWarning, _inspect.getsourcefile(func), _inspect.getsourcelines(func)[1])
+    else:
+        func = getattr(server, method_name) # will raise if no function found
+        params = context.params
+        params_dict = {name : getattr(params, name) for name in params.schema.fieldnames}
+        params_dict['_results'] = context.results
+        ret = func(**params_dict)
+
+        if ret is not None:
+            if type(ret) is _VoidPromise:
+                return new VoidPromise(moveVoidPromise(deref((<_VoidPromise>ret).thisptr)))
+            if not isinstance(ret, tuple):
+                ret = (ret,)
+            names = _find_field_order(context.results.schema.node.struct)
+            if len(ret) > len(names):
+                raise ValueError('Too many values returned from `%s`. Expected %d and got %d' % (method_name, len(names), len(ret)))
+
+            results = context.results
+            for arg_name, arg_val in zip(names, ret):
+                setattr(results, arg_name, arg_val)
 
     return NULL
     
@@ -1388,15 +1421,18 @@ cdef class _DynamicCapabilityClient:
 
         params = s.get_dependency(meth.paramStructType).node
         if params.scopeId != 0:
-            raise ValueError("Cannot call method `%s` with positional args, since its param struct is not implicitly defined and thus does not have a set order of arguments")
+            raise ValueError("Cannot call method `%s` with positional args, since its param struct is not implicitly defined and thus does not have a set order of arguments" % method_name)
 
-        return [f.name for f in params.struct.fields]
+        return _find_field_order(params.struct)
 
     cpdef _send_helper(self, name, firstSegmentWordSize, args, kwargs) except +reraise_kj_exception:
         cdef Request * request = new Request(self.thisptr.newRequest(name, firstSegmentWordSize))
 
         if args is not None:
-            for arg_name, arg_val in zip(self._find_method_args(name), args):
+            arg_names = self._find_method_args(name)
+            if len(args) > len(arg_names):
+                raise ValueError('Too many arguments passed to `%s`. Expected %d and got %d' % (name, len(arg_names), len(args)))
+            for arg_name, arg_val in zip(arg_names, args):
                 _setDynamicFieldPtr(request, arg_name, arg_val, self)
 
         for key, val in kwargs.items():
@@ -1700,46 +1736,130 @@ class _StructABCMeta(type):
     def __instancecheck__(cls, obj):
         return isinstance(obj, cls.__base__) and obj.schema == cls._schema
 
+cdef _new_message(self, kwargs):
+    builder = _MallocMessageBuilder()
+    msg = builder.init_root(self.schema)
+    if kwargs is not None:
+        _from_dict(msg, kwargs)
+    return msg
+
 class _StructModule(object):
     def __init__(self, schema):
         self.schema = schema
 
     def read(self, file, traversal_limit_in_words = None, nesting_limit = None):
+        """Returns a Reader for the unpacked object read from file.
+
+        :type file: file
+        :param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
+        
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :rtype: :class:`_DynamicStructReader`"""
         reader = _StreamFdMessageReader(file.fileno(), traversal_limit_in_words, nesting_limit)
         return reader.get_root(self.schema)
     def read_multiple(self, file, traversal_limit_in_words = None, nesting_limit = None):
+        """Returns an iterable, that when traversed will return Readers for messages.
+
+        :type file: file
+        :param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
+        
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :rtype: Iterable with elements of :class:`_DynamicStructReader`"""
         reader = _MultipleMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit)
         return reader
     def read_packed(self, file, traversal_limit_in_words = None, nesting_limit = None):
+        """Returns a Reader for the packed object read from file.
+
+        :type file: file
+        :param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
+        
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :rtype: :class:`_DynamicStructReader`"""
         reader = _PackedFdMessageReader(file.fileno(), traversal_limit_in_words, nesting_limit)
         return reader.get_root(self.schema)
     def read_multiple_packed(self, file, traversal_limit_in_words = None, nesting_limit = None):
+        """Returns an iterable, that when traversed will return Readers for messages.
+
+        :type file: file
+        :param file: A python file-like object. It must be a "real" file, with a `fileno()` method.
+        
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :rtype: Iterable with elements of :class:`_DynamicStructReader`"""
         reader = _MultiplePackedMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit)
         return reader
     def from_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None, builder=False):
         """Returns a Reader for the unpacked object in buf.
 
         :type buf: buffer
-        :param buf: Any Python object that supports the readable buffer interface.  If buf is mutable, then changes to the object will be reflected in the returned Reader, which may be surprising.  If buf is an ordinary bytes object, then there should be no concern.
+        :param buf: Any Python object that supports the buffer interface.
-        :type bool: builder
+        
-        :param buf: If true, return a builder object. This will allow you to change the contents of `buf`, so do this with care."""
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :type builder: bool
+        :param builder: If true, return a builder object. This will allow you to change the contents of `buf`, so do this with care.
+
+        :rtype: :class:`_DynamicStructReader` or :class:`_DynamicStructBuilder`
+        """
         if builder:
             message = _FlatMessageBuilder(buf)
         else:
             message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit)
         return message.get_root(self.schema)
     def from_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None):
+        """Returns a Reader for the packed object in buf.
+
+        :type buf: buffer
+        :param buf: Any Python object that supports the readable buffer interface.
+        
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :rtype: :class:`_DynamicStructReader`
+        """
         return _PackedMessageReaderBytes(buf, traversal_limit_in_words, nesting_limit).get_root(self.schema)
-    def new_message(self):
+    def new_message(self, **kwargs):
-        builder = _MallocMessageBuilder()
+        """Returns a newly allocated builder message.
-        return builder.init_root(self.schema)
+
-    def from_dict(self, d):
+        :type kwargs: dict
-        builder = _MallocMessageBuilder()
+        :param kwargs: A list of fields and their values to initialize in the struct
-        msg = builder.init_root(self.schema)
+
-        _from_dict(msg, d)
+        :rtype: :class:`_DynamicStructBuilder`
-        return msg
+        """
+        return _new_message(self, kwargs)
+    def from_dict(self, kwargs):
+        '.. warning:: This method is deprecated and will be removed in the 0.5 release. Use the :meth:`new_message` function instead with **kwargs'
+        _warnings.warn('This method is deprecated and will be removed in the 0.5 release. Use the :meth:`new_message` function instead with **kwargs', UserWarning)
+        return _new_message(self, kwargs)
     def from_object(self, obj):
-        _warnings.warn('This method is deprecated and will be removed in the 0.5 release. Use the `as_builder` or `copy` functions instead', UserWarning)
+        '.. warning:: This method is deprecated and will be removed in the 0.5 release. Use the :meth:`_DynamicStructReader.as_builder` or :meth:`_DynamicStructBuilder.copy` functions instead'
+        _warnings.warn('This method is deprecated and will be removed in the 0.5 release. Use the :meth:`_DynamicStructReader.as_builder` or :meth:`_DynamicStructBuilder.copy` functions instead', UserWarning)
         builder = _MallocMessageBuilder()
         return builder.set_root(obj)
 

test/test_capability.capnp

@@ -26,6 +26,7 @@
 interface TestInterface {
   foo @0 (i :UInt32, j :Bool) -> (x: Text);
   bar @1 () -> ();
+  buz @2 (i: TestSturdyRefHostId) -> (x: Text);
   # baz @2 (s: TestAllTypes);
 }
 

test/test_capability.py

@@ -12,16 +12,22 @@ class Server:
     def __init__(self, val=1):
         self.val = val
 
-    def foo(self, context):
+    def foo(self, i, j, **kwargs):
-        context.results.x = str(context.params.i * 5 + self.val)
+        extra = 0
+        if j:
+            extra = 1
+        return str(i * 5 + extra + self.val)
+
+    def buz(self, i, **kwargs):
+        return i.host + '_test'
 
 class PipelineServer:
-    def getCap(self, context):
+    def getCap(self, n, inCap, _results, **kwargs):
         def _then(response):
-            context.results.s = response.x + '_foo'
+            _results.s = response.x + '_foo'
-            context.results.outBox.cap = capability().TestInterface.new_server(Server(100))
+            _results.outBox.cap = capability().TestInterface.new_server(Server(100))
 
-        return context.params.inCap.foo(i=context.params.n).then(_then)
+        return inCap.foo(i=n).then(_then)
 
 def test_client(capability):
     loop = capnp.EventLoop()
@@ -73,11 +79,37 @@ def test_simple_client(capability):
 
     assert response.x == '26'
     
+    remote = client.foo(i=5, j=True)
+    response = loop.wait(remote)
+
+    assert response.x == '27'
+
     remote = client.foo(5)
     response = loop.wait(remote)
 
     assert response.x == '26'
 
+    remote = client.foo(5, True)
+    response = loop.wait(remote)
+
+    assert response.x == '27'
+
+    remote = client.foo(5, j=True)
+    response = loop.wait(remote)
+
+    assert response.x == '27'
+
+    remote = client.buz(capability.TestSturdyRefHostId.new_message(host='localhost'))
+    response = loop.wait(remote)
+
+    assert response.x == 'localhost_test'
+
+    with pytest.raises(ValueError):
+        remote = client.foo(5, 10)
+
+    with pytest.raises(ValueError):
+        remote = client.foo(5, True, 100)
+
     with pytest.raises(ValueError):
         remote = client.foo(i='foo')
 
@@ -108,9 +140,11 @@ class BadServer:
     def __init__(self, val=1):
         self.val = val
 
-    def foo(self, context):
+    def foo(self, i, j, **kwargs):
-        context.results.x = str(context.params.i * 5 + self.val)
+        extra = 0
-        context.results.x2 = 5 # raises exception
+        if j:
+            extra = 1
+        return str(i * 5 + extra + self.val), 10 # returning too many args
 
 def test_exception_client(capability):
     loop = capnp.EventLoop()
@@ -122,14 +156,14 @@ def test_exception_client(capability):
         loop.wait(remote)
 
 class BadPipelineServer:
-    def getCap(self, context):
+    def getCap(self, n, inCap, _results, **kwargs):
         def _then(response):
-            context.results.s = response.x + '_foo'
+            _results.s = response.x + '_foo'
-            context.results.outBox.cap = capability().TestInterface.new_server(Server(100))
+            _results.outBox.cap = capability().TestInterface.new_server(Server(100))
         def _error(error):
             raise Exception('test was a success')
 
-        return context.params.inCap.foo(i=context.params.n).then(_then, _error)
+        return inCap.foo(i=n).then(_then, _error)
 
 def test_exception_chain(capability):
     loop = capnp.EventLoop()

test/test_capability_context.py

@@ -0,0 +1,204 @@
+import pytest
+import capnp
+import os
+
+this_dir = os.path.dirname(__file__)
+
+@pytest.fixture
+def capability():
+     return capnp.load(os.path.join(this_dir, 'test_capability.capnp'))
+
+class Server:
+    def __init__(self, val=1):
+        self.val = val
+
+    def foo_context(self, context):
+        extra = 0
+        if context.params.j:
+            extra = 1
+        context.results.x = str(context.params.i * 5 + extra + self.val)
+
+    def buz_context(self, context):
+        context.results.x = context.params.i.host + '_test'
+
+class PipelineServer:
+    def getCap_context(self, context):
+        def _then(response):
+            context.results.s = response.x + '_foo'
+            context.results.outBox.cap = capability().TestInterface.new_server(Server(100))
+
+        return context.params.inCap.foo(i=context.params.n).then(_then)
+
+def test_client(capability):
+    loop = capnp.EventLoop()
+
+    client = capability.TestInterface._new_client(Server(), loop)
+    
+    req = client._request('foo')
+    req.i = 5
+
+    remote = req.send()
+    response = loop.wait(remote)
+
+    assert response.x == '26'
+    
+    req = client.foo_request()
+    req.i = 5
+
+    remote = req.send()
+    response = loop.wait(remote)
+
+    assert response.x == '26'
+
+    with pytest.raises(ValueError):
+        client.foo2_request()
+
+    req = client.foo_request()
+
+    with pytest.raises(ValueError):
+        req.i = 'foo'
+
+    req = client.foo_request()
+
+    with pytest.raises(ValueError):
+        req.baz = 1
+
+def test_simple_client(capability):
+    loop = capnp.EventLoop()
+
+    client = capability.TestInterface._new_client(Server(), loop)
+    
+    remote = client._send('foo', i=5)
+    response = loop.wait(remote)
+
+    assert response.x == '26'
+
+    
+    remote = client.foo(i=5)
+    response = loop.wait(remote)
+
+    assert response.x == '26'
+    
+    remote = client.foo(i=5, j=True)
+    response = loop.wait(remote)
+
+    assert response.x == '27'
+
+    remote = client.foo(5)
+    response = loop.wait(remote)
+
+    assert response.x == '26'
+
+    remote = client.foo(5, True)
+    response = loop.wait(remote)
+
+    assert response.x == '27'
+
+    remote = client.foo(5, j=True)
+    response = loop.wait(remote)
+
+    assert response.x == '27'
+
+    remote = client.buz(capability.TestSturdyRefHostId.new_message(host='localhost'))
+    response = loop.wait(remote)
+
+    assert response.x == 'localhost_test'
+
+    with pytest.raises(ValueError):
+        remote = client.foo(5, 10)
+
+    with pytest.raises(ValueError):
+        remote = client.foo(5, True, 100)
+
+    with pytest.raises(ValueError):
+        remote = client.foo(i='foo')
+
+    with pytest.raises(ValueError):
+        remote = client.foo2(i=5)
+
+    with pytest.raises(ValueError):
+        remote = client.foo(baz=5)
+
+def test_pipeline(capability):
+    loop = capnp.EventLoop()
+
+    client = capability.TestPipeline._new_client(PipelineServer(), loop)
+    foo_client = capability.TestInterface._new_client(Server(), loop)
+
+    remote = client.getCap(n=5, inCap=foo_client)
+
+    outCap = remote.outBox.cap
+    pipelinePromise = outCap.foo(i=10)
+
+    response = loop.wait(pipelinePromise)
+    assert response.x == '150'
+
+    response = loop.wait(remote)
+    assert response.s == '26_foo'
+
+class BadServer:
+    def __init__(self, val=1):
+        self.val = val
+
+    def foo_context(self, context):
+        context.results.x = str(context.params.i * 5 + self.val)
+        context.results.x2 = 5 # raises exception
+
+def test_exception_client(capability):
+    loop = capnp.EventLoop()
+
+    client = capability.TestInterface._new_client(BadServer(), loop)
+    
+    remote = client._send('foo', i=5)
+    with pytest.raises(capnp.KjException):
+        loop.wait(remote)
+
+class BadPipelineServer:
+    def getCap_context(self, context):
+        def _then(response):
+            context.results.s = response.x + '_foo'
+            context.results.outBox.cap = capability().TestInterface.new_server(Server(100))
+        def _error(error):
+            raise Exception('test was a success')
+
+        return context.params.inCap.foo(i=context.params.n).then(_then, _error)
+
+def test_exception_chain(capability):
+    loop = capnp.EventLoop()
+
+    client = capability.TestPipeline._new_client(BadPipelineServer(), loop)
+    foo_client = capability.TestInterface._new_client(BadServer(), loop)
+
+    remote = client.getCap(n=5, inCap=foo_client)
+
+    try:
+        loop.wait(remote)
+    except Exception as e:
+        assert 'test was a success' in str(e)
+
+def test_pipeline_exception(capability):
+    loop = capnp.EventLoop()
+
+    client = capability.TestPipeline._new_client(BadPipelineServer(), loop)
+    foo_client = capability.TestInterface._new_client(BadServer(), loop)
+
+    remote = client.getCap(n=5, inCap=foo_client)
+
+    outCap = remote.outBox.cap
+    pipelinePromise = outCap.foo(i=10)
+
+    with pytest.raises(Exception):
+        loop.wait(pipelinePromise)
+
+    with pytest.raises(Exception):
+        loop.wait(remote)
+
+def test_casting(capability):
+    loop = capnp.EventLoop()
+
+    client = capability.TestExtends._new_client(Server(), loop)
+    client2 = client.upcast(capability.TestInterface)
+    client3 = client2.cast_as(capability.TestInterface)
+
+    with pytest.raises(Exception):
+        client.upcast(capability.TestPipeline)

test/test_rpc.py

@@ -13,8 +13,8 @@ class Server:
     def __init__(self, val=1):
         self.val = val
 
-    def foo(self, context):
+    def foo(self, i, j, **kwargs):
-        context.results.x = str(context.params.i * 5 + self.val)
+        return str(i * 5 + self.val)
 
 def test_simple_rpc(capability):
     def _restore(ref_id):