From 85fb6de7f4c1ea6260f98bc24401593e8c974bc7 Mon Sep 17 00:00:00 2001 From: Roger Meier Date: Fri, 2 Nov 2012 00:05:42 +0000 Subject: [PATCH] THRIFT-1745 Python JSON protocol TJSONProtocol.py: Frederic Delbos THRIFT-847 Test Framework harmonization across all languages Integration into py lib and test suite git-svn-id: https://svn.apache.org/repos/asf/thrift/trunk@1404838 13f79535-47bb-0310-9956-ffa450edef68 --- lib/py/src/protocol/TJSONProtocol.py | 451 +++++++++++++++++++++++++++ lib/py/src/protocol/__init__.py | 2 +- test/py/RunClientServer.py | 4 +- test/py/TestClient.py | 16 +- test/py/TestServer.py | 12 +- test/test.sh | 38 ++- 6 files changed, 510 insertions(+), 13 deletions(-) create mode 100644 lib/py/src/protocol/TJSONProtocol.py diff --git a/lib/py/src/protocol/TJSONProtocol.py b/lib/py/src/protocol/TJSONProtocol.py new file mode 100644 index 00000000..97bc7639 --- /dev/null +++ b/lib/py/src/protocol/TJSONProtocol.py @@ -0,0 +1,451 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TProtocol import * +import json, base64, sys + +__all__ = ['TJSONProtocol', 'TJSONProtocolFactory'] + +VERSION = 1 + +COMMA = ',' +COLON = ':' +LBRACE = '{' +RBRACE = '}' +LBRACKET = '[' +RBRACKET = ']' +QUOTE = '"' +BACKSLASH = '\\' +ZERO = '0' + +ESCSEQ = '\\u00' +ESCAPE_CHAR = '"\\bfnrt' +ESCAPE_CHAR_VALS = ['"', '\\', '\b', '\f', '\n', '\r', '\t'] +NUMERIC_CHAR = '+-.0123456789Ee' + +CTYPES = {TType.BOOL: 'tf', + TType.BYTE: 'i8', + TType.I16: 'i16', + TType.I32: 'i32', + TType.I64: 'i64', + TType.DOUBLE: 'dbl', + TType.STRING: 'str', + TType.STRUCT: 'rect', + TType.LIST: 'lst', + TType.SET: 'set', + TType.MAP: 'map'} + +JTYPES = {} +for key in CTYPES.keys(): + JTYPES[CTYPES[key]] = key + + +class JSONBaseContext(): + + def __init__(self, protocol): + self.protocol = protocol + self.first = True + + def doIO(self, function): + pass + + def write(self): + pass + + def read(self): + pass + + def escapeNum(self): + return False + + +class JSONListContext(JSONBaseContext): + + def doIO(self, function): + if self.first is True: + self.first = False + else: + function(COMMA) + + def write(self): + self.doIO(self.protocol.trans.write) + + def read(self): + self.doIO(self.protocol.readJSONSyntaxChar) + + +class JSONPairContext(JSONBaseContext): + colon = True + + def doIO(self, function): + if self.first is True: + self.first = False + self.colon = True + else: + function(COLON if self.colon == True else COMMA) + self.colon = not self.colon + + def write(self): + self.doIO(self.protocol.trans.write) + + def read(self): + self.doIO(self.protocol.readJSONSyntaxChar) + + def escapeNum(self): + return self.colon + + +class LookaheadReader(): + hasData = False + data = '' + + def __init__(self, protocol): + self.protocol = protocol + + def read(self): + if self.hasData is True: + self.hasData = False + else: + self.data = self.protocol.trans.read(1) + return self.data + + def peek(self): + if self.hasData is False: + self.data = self.protocol.trans.read(1) + self.hasData = True + return self.data + +class TJSONProtocolBase(TProtocolBase): + + def __init__(self, trans): + TProtocolBase.__init__(self, trans) + + def resetWriteContext(self): + self.contextStack = [] + self.context = JSONBaseContext(self) + + def resetReadContext(self): + self.resetWriteContext() + self.reader = LookaheadReader(self) + + def pushContext(self, ctx): + self.contextStack.append(ctx) + self.context = ctx + + def popContext(self): + self.contextStack.pop() + + def writeJSONString(self, string): + self.context.write() + self.trans.write(json.dumps(string)) + + def writeJSONNumber(self, number): + self.context.write() + jsNumber = str(number) + if self.context.escapeNum(): + jsNumber = "%s%s%s" % (QUOTE, jsNumber, QUOTE) + self.trans.write(jsNumber) + + def writeJSONBase64(self, binary): + self.context.write() + self.trans.write(QUOTE) + self.trans.write(base64.b64encode(binary)) + self.trans.write(QUOTE) + + def writeJSONObjectStart(self): + self.context.write() + self.trans.write(LBRACE) + self.pushContext(JSONPairContext(self)) + + def writeJSONObjectEnd(self): + self.popContext() + self.trans.write(RBRACE) + + def writeJSONArrayStart(self): + self.context.write() + self.trans.write(LBRACKET) + self.pushContext(JSONListContext(self)) + + def writeJSONArrayEnd(self): + self.popContext() + self.trans.write(RBRACKET) + + def readJSONSyntaxChar(self, character): + current = self.reader.read() + if character != current: + raise TProtocolException(TProtocolException.INVALID_DATA, + "Unexpected character: %s" % current) + + def readJSONString(self, skipContext): + string = [] + if skipContext is False: + self.context.read() + self.readJSONSyntaxChar(QUOTE) + while True: + character = self.reader.read() + if character == QUOTE: + break + if character == ESCSEQ[0]: + character = self.reader.read() + if character == ESCSEQ[1]: + self.readJSONSyntaxChar(ZERO) + self.readJSONSyntaxChar(ZERO) + character = json.JSONDecoder().decode('"\u00%s"' % self.trans.read(2)) + else: + off = ESCAPE_CHAR.find(char) + if off == -1: + raise TProtocolException(TProtocolException.INVALID_DATA, + "Expected control char") + character = ESCAPE_CHAR_VALS[off] + string.append(character) + return ''.join(string) + + def isJSONNumeric(self, character): + return (True if NUMERIC_CHAR.find(character) != - 1 else False) + + def readJSONQuotes(self): + if (self.context.escapeNum()): + self.readJSONSyntaxChar(QUOTE) + + def readJSONNumericChars(self): + numeric = [] + while True: + character = self.reader.peek() + if self.isJSONNumeric(character) is False: + break + numeric.append(self.reader.read()) + return ''.join(numeric) + + def readJSONInteger(self): + self.context.read() + self.readJSONQuotes() + numeric = self.readJSONNumericChars() + self.readJSONQuotes() + try: + return int(numeric) + except ValueError: + raise TProtocolException(TProtocolException.INVALID_DATA, + "Bad data encounted in numeric data") + + def readJSONDouble(self): + self.context.read() + if self.reader.peek() == QUOTE: + string = self.readJSONString(True) + try: + double = float(string) + if self.context.escapeNum is False and double != inf and double != nan: + raise TProtocolException(TProtocolException.INVALID_DATA, + "Numeric data unexpectedly quoted") + return double + except ValueError: + raise TProtocolException(TProtocolException.INVALID_DATA, + "Bad data encounted in numeric data") + else: + if self.context.escapeNum() is True: + self.readJSONSyntaxChar(QUOTE) + try: + return float(self.readJSONNumericChars()) + except ValueErro: + raise TProtocolException(TProtocolException.INVALID_DATA, + "Bad data encounted in numeric data") + + def readJSONBase64(self): + string = self.readJSONString(False) + return base64.b64decode(string) + + def readJSONObjectStart(self): + self.context.read() + self.readJSONSyntaxChar(LBRACE) + self.pushContext(JSONPairContext(self)) + + def readJSONObjectEnd(self): + self.readJSONSyntaxChar(RBRACE) + self.popContext() + + def readJSONArrayStart(self): + self.context.read() + self.readJSONSyntaxChar(LBRACKET) + self.pushContext(JSONListContext(self)) + + def readJSONArrayEnd(self): + self.readJSONSyntaxChar(RBRACKET) + self.popContext() + + +class TJSONProtocol(TJSONProtocolBase): + + def readMessageBegin(self): + self.resetReadContext() + self.readJSONArrayStart() + if self.readJSONInteger() != VERSION: + raise TProtocolException(TProtocolException.BAD_VERSION, + "Message contained bad version.") + name = self.readJSONString(False) + typen = self.readJSONInteger() + seqid = self.readJSONInteger() + return (name, typen, seqid) + + def readMessageEnd(self): + self.readJSONArrayEnd() + + def readStructBegin(self): + self.readJSONObjectStart() + + def readStructEnd(self): + self.readJSONObjectEnd() + + def readFieldBegin(self): + character = self.reader.peek() + type = 0 + id = 0 + if character == RBRACE: + type = TType.STOP + else: + id = self.readJSONInteger() + self.readJSONObjectStart() + type = JTYPES[self.readJSONString(False)] + return (None, type, id) + + def readFieldEnd(self): + self.readJSONObjectEnd() + + def readMapBegin(self): + self.readJSONArrayStart() + keyType = JTYPES[self.readJSONString(False)] + valueType = JTYPES[self.readJSONString(False)] + size = self.readJSONInteger() + self.readJSONObjectStart() + return (keyType, valueType, size) + + def readMapEnd(self): + self.readJSONObjectEnd() + self.readJSONArrayEnd() + + def readCollectionBegin(self): + self.readJSONArrayStart() + elemType = JTYPES[self.readJSONString(False)] + size = self.readJSONInteger() + return (type, size) + readListBegin = readCollectionBegin + readSetBegin = readCollectionBegin + + def readCollectionEnd(self): + self.readJSONArrayEnd() + readSetEnd = readCollectionEnd + readListEnd = readCollectionEnd + + def readBool(self): + return (False if self.readJSONInteger() == 0 else True) + + def readNumber(self): + return self.readJSONInteger() + readByte = readNumber + readI16 = readNumber + readI32 = readNumber + readI64 = readNumber + + def readDouble(self): + return self.readJSONDouble() + + def readString(self): + return self.readJSONString(False) + + def readBinary(self): + return self.readJSONBase64() + + def writeMessageBegin(self, name, request_type, seqid): + self.resetWriteContext() + self.writeJSONArrayStart() + self.writeJSONNumber(VERSION) + self.writeJSONString(name) + self.writeJSONNumber(request_type) + self.writeJSONNumber(seqid) + + def writeMessageEnd(self): + self.writeJSONArrayEnd() + + def writeStructBegin(self, name): + self.writeJSONObjectStart() + + def writeStructEnd(self): + self.writeJSONObjectEnd() + + def writeFieldBegin(self, name, type, id): + self.writeJSONNumber(id) + self.writeJSONObjectStart() + self.writeJSONString(CTYPES[type]) + + def writeFieldEnd(self): + self.writeJSONObjectEnd() + + def writeFieldStop(self): + pass + + def writeMapBegin(self, ktype, vtype, size): + self.writeJSONArrayStart() + self.writeJSONString(CTYPES[ktype]) + self.writeJSONString(CTYPES[vtype]) + self.writeJSONNumber(size) + self.writeJSONObjectStart() + + def writeMapEnd(self): + self.writeJSONObjectEnd() + self.writeJSONArrayEnd() + + def writeListBegin(self, etype, size): + self.writeJSONArrayStart() + self.writeJSONString(CTYPES[etype]) + self.writeJSONNumber(size) + + def writeListEnd(self): + self.writeJSONArrayEnd() + + def writeSetBegin(self, etype, size): + self.writeJSONArrayStart() + self.writeJSONString(CTYPES[etype]) + self.writeJSONNumber(size) + + def writeSetEnd(self): + self.writeJSONArrayEnd() + + def writeBool(self, boolean): + self.writeJSONNumber(1 if boolean is True else 0) + + def writeInteger(self, integer): + self.writeJSONNumber(integer) + writeByte = writeInteger + writeI16 = writeInteger + writeI32 = writeInteger + writeI64 = writeInteger + + def writeDouble(self, dbl): + self.writeJSONNumber(dbl) + + def writeString(self, string): + self.writeJSONString(string) + + def writeBinary(self, binary): + self.writeJSONBase64(binary) + +class TJSONProtocolFactory: + def __init__(self): + pass + + def getProtocol(self, trans): + return TJSONProtocol(trans) diff --git a/lib/py/src/protocol/__init__.py b/lib/py/src/protocol/__init__.py index d53359b2..7eefb458 100644 --- a/lib/py/src/protocol/__init__.py +++ b/lib/py/src/protocol/__init__.py @@ -17,4 +17,4 @@ # under the License. # -__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary', 'TBase'] +__all__ = ['fastbinary', 'TBase', 'TBinaryProtocol', 'TCompactProtocol', 'TJSONProtocol', 'TProtocol'] diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py index 8a7fda64..f9121c8b 100755 --- a/test/py/RunClientServer.py +++ b/test/py/RunClientServer.py @@ -55,7 +55,9 @@ EXTRA_DELAY = dict(TProcessPoolServer=3.5) PROTOS= [ 'accel', 'binary', - 'compact' ] + 'compact'] +# FIXME: add json +# disabled because json HTTP test hangs... why? SERVERS = [ "TSimpleServer", diff --git a/test/py/TestClient.py b/test/py/TestClient.py index 71001b1b..471e030d 100755 --- a/test/py/TestClient.py +++ b/test/py/TestClient.py @@ -19,8 +19,8 @@ # under the License. # -import sys, glob -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +import sys, glob, os +sys.path.insert(0, glob.glob(os.path.join(os.path.dirname(__file__),'../../lib/py/build/lib.*'))[0]) import unittest import time @@ -63,6 +63,7 @@ from thrift.transport import THttpClient from thrift.transport import TZlibTransport from thrift.protocol import TBinaryProtocol from thrift.protocol import TCompactProtocol +from thrift.protocol import TJSONProtocol class AbstractTest(unittest.TestCase): def setUp(self): @@ -125,7 +126,7 @@ class AbstractTest(unittest.TestCase): def testNest(self): inner = Xtruct(string_thing="Zero", byte_thing=1, i32_thing=-3, i64_thing=-5) - x = Xtruct2(struct_thing=inner) + x = Xtruct2(struct_thing=inner, byte_thing=0, i32_thing=0) y = self.client.testNest(x) self.assertEqual(y, x) @@ -163,13 +164,13 @@ class AbstractTest(unittest.TestCase): pass def testMulti(self): - xpected = Xtruct(byte_thing=74, i32_thing=0xff00ff, i64_thing=0xffffffffd0d0) + xpected = Xtruct(string_thing='Hello2', byte_thing=74, i32_thing=0xff00ff, i64_thing=0xffffffffd0d0) y = self.client.testMulti(xpected.byte_thing, xpected.i32_thing, xpected.i64_thing, { 0:'abc' }, Numberz.FIVE, - 0xf0f0f0) + 0xf0f0f0) self.assertEqual(y, xpected) def testException(self): @@ -208,6 +209,9 @@ class NormalBinaryTest(AbstractTest): class CompactTest(AbstractTest): protocol_factory = TCompactProtocol.TCompactProtocolFactory() +class JSONTest(AbstractTest): + protocol_factory = TJSONProtocol.TJSONProtocolFactory() + class AcceleratedBinaryTest(AbstractTest): protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory() @@ -220,6 +224,8 @@ def suite(): suite.addTest(loader.loadTestsFromTestCase(AcceleratedBinaryTest)) elif options.proto == 'compact': suite.addTest(loader.loadTestsFromTestCase(CompactTest)) + elif options.proto == 'json': + suite.addTest(loader.loadTestsFromTestCase(JSONTest)) else: raise AssertionError('Unknown protocol given with --proto: %s' % options.proto) return suite diff --git a/test/py/TestServer.py b/test/py/TestServer.py index 6f4af440..1eae0976 100755 --- a/test/py/TestServer.py +++ b/test/py/TestServer.py @@ -19,8 +19,8 @@ # under the License. # from __future__ import division -import sys, glob, time -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +import sys, glob, time, os +sys.path.insert(0, glob.glob(os.path.join(os.path.dirname(__file__),'../../lib/py/build/lib.*'))[0]) from optparse import OptionParser parser = OptionParser() @@ -40,7 +40,7 @@ parser.add_option('-q', '--quiet', action="store_const", dest="verbose", const=0, help="minimal output") parser.add_option('--proto', dest="proto", type="string", - help="protocol to use, one of: accel, binary, compact") + help="protocol to use, one of: accel, binary, compact, json") parser.set_defaults(port=9090, verbose=1, proto='binary') options, args = parser.parse_args() @@ -53,11 +53,13 @@ from thrift.transport import TSocket from thrift.transport import TZlibTransport from thrift.protocol import TBinaryProtocol from thrift.protocol import TCompactProtocol +from thrift.protocol import TJSONProtocol from thrift.server import TServer, TNonblockingServer, THttpServer PROT_FACTORIES = {'binary': TBinaryProtocol.TBinaryProtocolFactory, 'accel': TBinaryProtocol.TBinaryProtocolAcceleratedFactory, - 'compact': TCompactProtocol.TCompactProtocolFactory} + 'compact': TCompactProtocol.TCompactProtocolFactory, + 'json': TJSONProtocol.TJSONProtocolFactory} class TestHandler: @@ -156,7 +158,7 @@ class TestHandler: def testMulti(self, arg0, arg1, arg2, arg3, arg4, arg5): if options.verbose > 1: print 'testMulti(%s)' % [arg0, arg1, arg2, arg3, arg4, arg5] - x = Xtruct(byte_thing=arg0, i32_thing=arg1, i64_thing=arg2) + x = Xtruct(string_thing='Hello2', byte_thing=arg0, i32_thing=arg1, i64_thing=arg2) return x # set up the protocol factory form the --proto option diff --git a/test/test.sh b/test/test.sh index 30ca8381..36e57c30 100755 --- a/test/test.sh +++ b/test/test.sh @@ -55,6 +55,7 @@ do_test () { echo "=================== client message ===================" tail log/${testname}_client.log echo "======================================================" + echo "" print_header fi sleep 10 @@ -93,7 +94,42 @@ for proto in $protocols; do done; done; - +do_test "py-py" "binary" "buffered-ip" \ + "py/TestClient.py --proto=binary --port=9090 --host=localhost --genpydir=py/gen-py" \ + "py/TestServer.py --proto=binary --port=9090 --genpydir=py/gen-py TSimpleServer" \ + "10" +do_test "py-py" "json" "buffered-ip" \ + "py/TestClient.py --proto=json --port=9090 --host=localhost --genpydir=py/gen-py" \ + "py/TestServer.py --proto=json --port=9090 --genpydir=py/gen-py TSimpleServer" \ + "10" +do_test "py-cpp" "binary" "buffered-ip" \ + "py/TestClient.py --proto=binary --port=9090 --host=localhost --genpydir=py/gen-py" \ + "cpp/TestServer" \ + "10" +do_test "py-cpp" "json" "buffered-ip" \ + "py/TestClient.py --proto=json --port=9090 --host=localhost --genpydir=py/gen-py" \ + "cpp/TestServer --protocol=json" \ + "10" +do_test "cpp-py" "binary" "buffered-ip" \ + "cpp/TestClient --protocol=binary --port=9090" \ + "py/TestServer.py --proto=binary --port=9090 --genpydir=py/gen-py TSimpleServer" \ + "10" +do_test "cpp-py" "json" "buffered-ip" \ + "cpp/TestClient --protocol=json --port=9090" \ + "py/TestServer.py --proto=json --port=9090 --genpydir=py/gen-py TSimpleServer" \ + "10" +do_test "py-java" "binary" "buffered-ip" \ + "py/TestClient.py --proto=binary --port=9090 --host=localhost --genpydir=py/gen-py" \ + "ant -f ../lib/java/build.xml testserver" \ + "100" +do_test "py-java" "json" "buffered-ip" \ + "py/TestClient.py --proto=json --port=9090 --host=localhost --genpydir=py/gen-py" \ + "ant -f ../lib/java/build.xml testserver" \ + "100" +do_test "java-py" "binary" "buffered-ip" \ + "ant -f ../lib/java/build.xml testclient" \ + "py/TestServer.py --proto=binary --port=9090 --genpydir=py/gen-py TSimpleServer" \ + "10" do_test "java-java" "binary" "buffered-ip" \ "ant -f ../lib/java/build.xml testclient" \ "ant -f ../lib/java/build.xml testserver" \ -- 2.17.1