Truncate UTF-8 in a way that produces valid UTF-8.

This commit is contained in:
Sybren A. Stüvel
2018-03-08 14:47:41 +01:00
parent 898f79a951
commit 0dab872289
2 changed files with 43 additions and 3 deletions
+19 -3
View File
@@ -73,9 +73,25 @@ class EndianIO:
:returns: the number of bytes written.
"""
assert isinstance(astring, str)
# TODO: truncate the string on a UTF-8 character boundary to avoid creating invalid UTF-8.
encoded = astring.encode('utf-8')[:fieldlen-1] + b'\0'
return fileobj.write(encoded)
encoded = astring.encode('utf-8')
# Take into account we also need space for a trailing 0-byte.
maxlen = fieldlen - 1
if len(encoded) >= maxlen:
encoded = encoded[:maxlen]
# Keep stripping off the last byte until the string
# is valid UTF-8 again.
while True:
try:
encoded.decode('utf8')
except UnicodeDecodeError:
encoded = encoded[:-1]
else:
break
return fileobj.write(encoded + b'\0')
@classmethod
def write_bytes(cls, fileobj: typing.BinaryIO, data: bytes, fieldlen: int) -> int: