From 8c3c57aa325758fc44916a2dab93c9c52802369f Mon Sep 17 00:00:00 2001
From: Mike Lundy <mike@fluffypenguin.org>
Date: Thu, 3 Mar 2016 14:29:32 -0800
Subject: [PATCH] Make sure to encode to utf-8, not the default encoding

This allows text fields to take unicode strings under python 2.
---
 capnp/lib/capnp.pyx        |   4 ++--
 capnp/templates/module.pyx |   4 ++--
 test/all-types.binary      | Bin 2816 -> 2816 bytes
 test/all-types.packed      | Bin 831 -> 831 bytes
 test/all-types.txt         |   2 +-
 test/test_regression.py    |  21 +++++++++++++++++++--
 6 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx
index 290c4cf..55b3964 100644
--- a/capnp/lib/capnp.pyx
+++ b/capnp/lib/capnp.pyx
@@ -679,7 +679,7 @@ cdef _setBytes(_DynamicSetterClasses thisptr, field, value):
     thisptr.set(field, temp)
 
 cdef _setBaseString(_DynamicSetterClasses thisptr, field, value):
-    encoded_value = value.encode()
+    encoded_value = value.encode('utf-8')
     cdef capnp.StringPtr temp_string = capnp.StringPtr(<char*>encoded_value, len(encoded_value))
     cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string)
     thisptr.set(field, temp)
@@ -690,7 +690,7 @@ cdef _setBytesField(DynamicStruct_Builder thisptr, _StructSchemaField field, val
     thisptr.setByField(field.thisptr, temp)
 
 cdef _setBaseStringField(DynamicStruct_Builder thisptr, _StructSchemaField field, value):
-    encoded_value = value.encode()
+    encoded_value = value.encode('utf-8')
     cdef capnp.StringPtr temp_string = capnp.StringPtr(<char*>encoded_value, len(encoded_value))
     cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string)
     thisptr.setByField(field.thisptr, temp)
diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx
index df45582..9e51a7e 100644
--- a/capnp/templates/module.pyx
+++ b/capnp/templates/module.pyx
@@ -90,7 +90,7 @@ cpdef _set_{{field.name}}(self, value):
     if type(value) is bytes:
         temp_string = StringPtr(<char*>value, len(value))
     else:
-        encoded_value = value.encode()
+        encoded_value = value.encode('utf-8')
         temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
     self.thisptr_child.set{{field.c_name}}(temp_string)
     {% elif 'data' == field['type'] -%}
@@ -99,7 +99,7 @@ cpdef _set_{{field.name}}(self, value):
     if type(value) is bytes:
         temp_string = StringPtr(<char*>value, len(value))
     else:
-        encoded_value = value.encode()
+        encoded_value = value.encode('utf-8')
         temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
     self.thisptr_child.set{{field.c_name}}(ArrayPtr[byte](<byte *>temp_string.begin(), temp_string.size()))
     {% else -%}
diff --git a/test/all-types.binary b/test/all-types.binary
index ea39763774b2ed570407a3384a8865fbeaa79213..3381caad76714027a9f0768938dc79cd0314d8d4 100644
GIT binary patch
delta 26
fcmZn=YY^LTfRX#rjAjN9C@ig*tjJ`#h=~&ba}Ecb

delta 26
fcmZn=YY^LTfRQ^Xv5El%3QH>{D>9ibV&Vh<W!DDW

diff --git a/test/all-types.packed b/test/all-types.packed
index 8627833b2caa63e59b0a1c9ed6d317187bb1219f..acb6086e58e37b121e9ca54bf1f58ff98260eb2f 100644
GIT binary patch
delta 16
Xcmdnbwx4YSD<kux8O@tH7|odgG7<$g

delta 16
Xcmdnbwx4YSD<gAKV%262Msp?rEL#M2

diff --git a/test/all-types.txt b/test/all-types.txt
index 079ff8d..a9e3dcc 100644
--- a/test/all-types.txt
+++ b/test/all-types.txt
@@ -25,7 +25,7 @@
     uInt64Field = 345678901234567890,
     float32Field = -1.25e-10,
     float64Field = 345,
-    textField = "baz",
+    textField = "\xe2\x98\x83",
     dataField = "qux",
     structField = (
       voidField = void,
diff --git a/test/test_regression.py b/test/test_regression.py
index d21c2f4..78f3fec 100644
--- a/test/test_regression.py
+++ b/test/test_regression.py
@@ -1,10 +1,19 @@
+ # -*- coding: utf-8 -*-
+
 import pytest
 import capnp
 import os
 import math
+import sys
 
 this_dir = os.path.dirname(__file__)
 
+if sys.version_info[0] < 3:
+    EXPECT_BYTES = True
+else:
+    EXPECT_BYTES = False
+
+
 @pytest.fixture
 def addressbook():
      return capnp.load(os.path.join(this_dir, 'addressbook.capnp'))
@@ -300,7 +309,7 @@ def init_all_types(builder):
     subBuilder.uInt64Field = 345678901234567890
     subBuilder.float32Field = -1.25e-10
     subBuilder.float64Field = 345
-    subBuilder.textField = "baz"
+    subBuilder.textField = b"\xe2\x98\x83".decode('utf-8') # This is u"☃", but py3.2 doesn't support u
     subBuilder.dataField = b"qux"
     subSubBuilder = subBuilder.structField
     subSubBuilder.textField = "nested"
@@ -390,7 +399,15 @@ def check_all_types(reader):
     assert subReader.uInt64Field == 345678901234567890
     assert_almost(subReader.float32Field, -1.25e-10)
     assert subReader.float64Field == 345
-    assert subReader.textField == "baz"
+
+    assert subReader.textField == "☃"
+    # This assertion highlights the encoding we expect to see here, since
+    # otherwise this appears a bit magical...
+    if EXPECT_BYTES:
+        assert len(subReader.textField) == 3
+    else:
+        assert len(subReader.textField) == 1
+
     assert subReader.dataField == b"qux"
 
     subSubReader = subReader.structField